diff options
| -rw-r--r-- | Documentation/config/trace2.txt | 6 | ||||
| -rw-r--r-- | Documentation/technical/api-trace2.txt | 31 | ||||
| -rw-r--r-- | Documentation/trace2-target-values.txt | 4 | ||||
| -rwxr-xr-x | t/t0212-trace2-event.sh | 19 | ||||
| -rw-r--r-- | trace2/tr2_dst.c | 111 | ||||
| -rw-r--r-- | trace2/tr2_dst.h | 1 | ||||
| -rw-r--r-- | trace2/tr2_sysenv.c | 3 | ||||
| -rw-r--r-- | trace2/tr2_sysenv.h | 2 | ||||
| -rw-r--r-- | trace2/tr2_tgt_event.c | 31 | ||||
| -rw-r--r-- | trace2/tr2_tgt_normal.c | 2 | ||||
| -rw-r--r-- | trace2/tr2_tgt_perf.c | 2 |
11 files changed, 184 insertions, 28 deletions
diff --git a/Documentation/config/trace2.txt b/Documentation/config/trace2.txt index 2edbfb0..4ce0b9a 100644 --- a/Documentation/config/trace2.txt +++ b/Documentation/config/trace2.txt @@ -54,3 +54,9 @@ trace2.destinationDebug:: By default, these errors are suppressed and tracing is silently disabled. May be overridden by the `GIT_TRACE2_DST_DEBUG` environment variable. + +trace2.maxFiles:: + Integer. When writing trace files to a target directory, do not + write additional traces if we would exceed this many files. Instead, + write a sentinel file that will block further tracing to this + directory. Defaults to 0, which disables this check. diff --git a/Documentation/technical/api-trace2.txt b/Documentation/technical/api-trace2.txt index 71eb081..a045dbe 100644 --- a/Documentation/technical/api-trace2.txt +++ b/Documentation/technical/api-trace2.txt @@ -128,7 +128,7 @@ yields ------------ $ cat ~/log.event -{"event":"version","sid":"sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.620713Z","file":"common-main.c","line":38,"evt":"1","exe":"2.20.1.155.g426c96fcdb"} +{"event":"version","sid":"sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.620713Z","file":"common-main.c","line":38,"evt":"2","exe":"2.20.1.155.g426c96fcdb"} {"event":"start","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621027Z","file":"common-main.c","line":39,"t_abs":0.001173,"argv":["git","version"]} {"event":"cmd_name","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621122Z","file":"git.c","line":432,"name":"version","hierarchy":"version"} {"event":"exit","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621236Z","file":"git.c","line":662,"t_abs":0.001227,"code":0} @@ -142,10 +142,9 @@ system or global config value to one of the following: include::../trace2-target-values.txt[] -If the target already exists and is a directory, the traces will be -written to files (one per process) underneath the given directory. They -will be named according to the last component of the SID (optionally -followed by a counter to avoid filename collisions). +When trace files are written to a target directory, they will be named according +to the last component of the SID (optionally followed by a counter to avoid +filename collisions). == Trace2 API @@ -605,17 +604,35 @@ only present on the "start" and "atexit" events. ==== Event-Specific Key/Value Pairs `"version"`:: - This event gives the version of the executable and the EVENT format. + This event gives the version of the executable and the EVENT format. It + should always be the first event in a trace session. The EVENT format + version will be incremented if new event types are added, if existing + fields are removed, or if there are significant changes in + interpretation of existing events or fields. Smaller changes, such as + adding a new field to an existing event, will not require an increment + to the EVENT format version. + ------------ { "event":"version", ... - "evt":"1", # EVENT format version + "evt":"2", # EVENT format version "exe":"2.20.1.155.g426c96fcdb" # git version } ------------ +`"discard"`:: + This event is written to the git-trace2-discard sentinel file if there + are too many files in the target trace directory (see the + trace2.maxFiles config option). ++ +------------ +{ + "event":"discard", + ... +} +------------ + `"start"`:: This event contains the complete argv received by main(). + diff --git a/Documentation/trace2-target-values.txt b/Documentation/trace2-target-values.txt index 27d3c64..3985b6d 100644 --- a/Documentation/trace2-target-values.txt +++ b/Documentation/trace2-target-values.txt @@ -2,7 +2,9 @@ * `0` or `false` - Disables the target. * `1` or `true` - Writes to `STDERR`. * `[2-9]` - Writes to the already opened file descriptor. -* `<absolute-pathname>` - Writes to the file in append mode. +* `<absolute-pathname>` - Writes to the file in append mode. If the target +already exists and is a directory, the traces will be written to files (one +per process) underneath the given directory. * `af_unix:[<socket_type>:]<absolute-pathname>` - Write to a Unix DomainSocket (on platforms that support them). Socket type can be either `stream` or `dgram`; if omitted Git will diff --git a/t/t0212-trace2-event.sh b/t/t0212-trace2-event.sh index ff5b9cc..7065a1b 100755 --- a/t/t0212-trace2-event.sh +++ b/t/t0212-trace2-event.sh @@ -265,4 +265,23 @@ test_expect_success JSON_PP 'using global config, event stream, error event' ' test_cmp expect actual ' +test_expect_success 'discard traces when there are too many files' ' + mkdir trace_target_dir && + test_when_finished "rm -r trace_target_dir" && + ( + GIT_TRACE2_MAX_FILES=5 && + export GIT_TRACE2_MAX_FILES && + cd trace_target_dir && + test_seq $GIT_TRACE2_MAX_FILES >../expected_filenames.txt && + xargs touch <../expected_filenames.txt && + cd .. && + GIT_TRACE2_EVENT="$(pwd)/trace_target_dir" test-tool trace2 001return 0 + ) && + echo git-trace2-discard >>expected_filenames.txt && + ls trace_target_dir >ls_output.txt && + test_cmp expected_filenames.txt ls_output.txt && + head -n1 trace_target_dir/git-trace2-discard | grep \"event\":\"version\" && + head -n2 trace_target_dir/git-trace2-discard | tail -n1 | grep \"event\":\"too_many_files\" +' + test_done diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c index c698575..ae052a0 100644 --- a/trace2/tr2_dst.c +++ b/trace2/tr2_dst.c @@ -8,6 +8,19 @@ */ #define MAX_AUTO_ATTEMPTS 10 +/* + * Sentinel file used to detect when we should discard new traces to avoid + * writing too many trace files to a directory. + */ +#define DISCARD_SENTINEL_NAME "git-trace2-discard" + +/* + * When set to zero, disables directory file count checks. Otherwise, controls + * how many files we can write to a directory before entering discard mode. + * This can be overridden via the TR2_SYSENV_MAX_FILES setting. + */ +static int tr2env_max_files = 0; + static int tr2_dst_want_warning(void) { static int tr2env_dst_debug = -1; @@ -32,9 +45,75 @@ void tr2_dst_trace_disable(struct tr2_dst *dst) dst->need_close = 0; } +/* + * Check to make sure we're not overloading the target directory with too many + * files. First get the threshold (if present) from the config or envvar. If + * it's zero or unset, disable this check. Next check for the presence of a + * sentinel file, then check file count. + * + * Returns 0 if tracing should proceed as normal. Returns 1 if the sentinel file + * already exists, which means tracing should be disabled. Returns -1 if there + * are too many files but there was no sentinel file, which means we have + * created and should write traces to the sentinel file. + * + * We expect that some trace processing system is gradually collecting files + * from the target directory; after it removes the sentinel file we'll start + * writing traces again. + */ +static int tr2_dst_too_many_files(struct tr2_dst *dst, const char *tgt_prefix) +{ + int file_count = 0, max_files = 0, ret = 0; + const char *max_files_var; + DIR *dirp; + struct strbuf path = STRBUF_INIT, sentinel_path = STRBUF_INIT; + struct stat statbuf; + + /* Get the config or envvar and decide if we should continue this check */ + max_files_var = tr2_sysenv_get(TR2_SYSENV_MAX_FILES); + if (max_files_var && *max_files_var && ((max_files = atoi(max_files_var)) >= 0)) + tr2env_max_files = max_files; + + if (!tr2env_max_files) { + ret = 0; + goto cleanup; + } + + strbuf_addstr(&path, tgt_prefix); + if (!is_dir_sep(path.buf[path.len - 1])) { + strbuf_addch(&path, '/'); + } + + /* check sentinel */ + strbuf_addbuf(&sentinel_path, &path); + strbuf_addstr(&sentinel_path, DISCARD_SENTINEL_NAME); + if (!stat(sentinel_path.buf, &statbuf)) { + ret = 1; + goto cleanup; + } + + /* check file count */ + dirp = opendir(path.buf); + while (file_count < tr2env_max_files && dirp && readdir(dirp)) + file_count++; + if (dirp) + closedir(dirp); + + if (file_count >= tr2env_max_files) { + dst->too_many_files = 1; + dst->fd = open(sentinel_path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666); + ret = -1; + goto cleanup; + } + +cleanup: + strbuf_release(&path); + strbuf_release(&sentinel_path); + return ret; +} + static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix) { - int fd; + int too_many_files; const char *last_slash, *sid = tr2_sid_get(); struct strbuf path = STRBUF_INIT; size_t base_path_len; @@ -50,18 +129,29 @@ static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix) strbuf_addstr(&path, sid); base_path_len = path.len; - for (attempt_count = 0; attempt_count < MAX_AUTO_ATTEMPTS; attempt_count++) { - if (attempt_count > 0) { - strbuf_setlen(&path, base_path_len); - strbuf_addf(&path, ".%d", attempt_count); + too_many_files = tr2_dst_too_many_files(dst, tgt_prefix); + if (!too_many_files) { + for (attempt_count = 0; attempt_count < MAX_AUTO_ATTEMPTS; attempt_count++) { + if (attempt_count > 0) { + strbuf_setlen(&path, base_path_len); + strbuf_addf(&path, ".%d", attempt_count); + } + + dst->fd = open(path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666); + if (dst->fd != -1) + break; } - - fd = open(path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666); - if (fd != -1) - break; + } else if (too_many_files == 1) { + strbuf_release(&path); + if (tr2_dst_want_warning()) + warning("trace2: not opening %s trace file due to too " + "many files in target directory %s", + tr2_sysenv_display_name(dst->sysenv_var), + tgt_prefix); + return 0; } - if (fd == -1) { + if (dst->fd == -1) { if (tr2_dst_want_warning()) warning("trace2: could not open '%.*s' for '%s' tracing: %s", (int) base_path_len, path.buf, @@ -75,7 +165,6 @@ static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix) strbuf_release(&path); - dst->fd = fd; dst->need_close = 1; dst->initialized = 1; diff --git a/trace2/tr2_dst.h b/trace2/tr2_dst.h index 3adf3ba..b1a8c14 100644 --- a/trace2/tr2_dst.h +++ b/trace2/tr2_dst.h @@ -9,6 +9,7 @@ struct tr2_dst { int fd; unsigned int initialized : 1; unsigned int need_close : 1; + unsigned int too_many_files : 1; }; /* diff --git a/trace2/tr2_sysenv.c b/trace2/tr2_sysenv.c index 5958cfc..3c3792e 100644 --- a/trace2/tr2_sysenv.c +++ b/trace2/tr2_sysenv.c @@ -49,6 +49,9 @@ static struct tr2_sysenv_entry tr2_sysenv_settings[] = { "trace2.perftarget" }, [TR2_SYSENV_PERF_BRIEF] = { "GIT_TRACE2_PERF_BRIEF", "trace2.perfbrief" }, + + [TR2_SYSENV_MAX_FILES] = { "GIT_TRACE2_MAX_FILES", + "trace2.maxfiles" }, }; /* clang-format on */ diff --git a/trace2/tr2_sysenv.h b/trace2/tr2_sysenv.h index 8dd82a7..d4364a7 100644 --- a/trace2/tr2_sysenv.h +++ b/trace2/tr2_sysenv.h @@ -24,6 +24,8 @@ enum tr2_sysenv_variable { TR2_SYSENV_PERF, TR2_SYSENV_PERF_BRIEF, + TR2_SYSENV_MAX_FILES, + TR2_SYSENV_MUST_BE_LAST }; diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index 9bcac20..6353e8a 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -10,16 +10,17 @@ #include "trace2/tr2_tgt.h" #include "trace2/tr2_tls.h" -static struct tr2_dst tr2dst_event = { TR2_SYSENV_EVENT, 0, 0, 0 }; +static struct tr2_dst tr2dst_event = { TR2_SYSENV_EVENT, 0, 0, 0, 0 }; /* - * The version number of the JSON data generated by the EVENT target - * in this source file. Update this if you make a significant change - * to the JSON fields or message structure. You probably do not need - * to update this if you just add another call to one of the existing - * TRACE2 API methods. + * The version number of the JSON data generated by the EVENT target in this + * source file. The version should be incremented if new event types are added, + * if existing fields are removed, or if there are significant changes in + * interpretation of existing events or fields. Smaller changes, such as adding + * a new field to an existing event, do not require an increment to the EVENT + * format version. */ -#define TR2_EVENT_VERSION "1" +#define TR2_EVENT_VERSION "2" /* * Region nesting limit for messages written to the event target. @@ -107,6 +108,19 @@ static void event_fmt_prepare(const char *event_name, const char *file, jw_object_intmax(jw, "repo", repo->trace2_repo_id); } +static void fn_too_many_files_fl(const char *file, int line) +{ + const char *event_name = "too_many_files"; + struct json_writer jw = JSON_WRITER_INIT; + + jw_object_begin(&jw, 0); + event_fmt_prepare(event_name, file, line, NULL, &jw); + jw_end(&jw); + + tr2_dst_write_line(&tr2dst_event, &jw.json); + jw_release(&jw); +} + static void fn_version_fl(const char *file, int line) { const char *event_name = "version"; @@ -120,6 +134,9 @@ static void fn_version_fl(const char *file, int line) tr2_dst_write_line(&tr2dst_event, &jw.json); jw_release(&jw); + + if (tr2dst_event.too_many_files) + fn_too_many_files_fl(file, line); } static void fn_start_fl(const char *file, int line, diff --git a/trace2/tr2_tgt_normal.c b/trace2/tr2_tgt_normal.c index 438ed05..31b602c 100644 --- a/trace2/tr2_tgt_normal.c +++ b/trace2/tr2_tgt_normal.c @@ -9,7 +9,7 @@ #include "trace2/tr2_tgt.h" #include "trace2/tr2_tls.h" -static struct tr2_dst tr2dst_normal = { TR2_SYSENV_NORMAL, 0, 0, 0 }; +static struct tr2_dst tr2dst_normal = { TR2_SYSENV_NORMAL, 0, 0, 0, 0 }; /* * Use the TR2_SYSENV_NORMAL_BRIEF setting to omit the "<time> <file>:<line>" diff --git a/trace2/tr2_tgt_perf.c b/trace2/tr2_tgt_perf.c index fd979db..ffac802 100644 --- a/trace2/tr2_tgt_perf.c +++ b/trace2/tr2_tgt_perf.c @@ -11,7 +11,7 @@ #include "trace2/tr2_tgt.h" #include "trace2/tr2_tls.h" -static struct tr2_dst tr2dst_perf = { TR2_SYSENV_PERF, 0, 0, 0 }; +static struct tr2_dst tr2dst_perf = { TR2_SYSENV_PERF, 0, 0, 0, 0 }; /* * Use TR2_SYSENV_PERF_BRIEF to omit the "<time> <file>:<line>" |