summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJunio C Hamano <gitster@pobox.com>2019-10-15 04:48:01 (GMT)
committerJunio C Hamano <gitster@pobox.com>2019-10-15 04:48:01 (GMT)
commitd0ce4d9024882b5363141288979ea3689eab9998 (patch)
treecfd58c1a5cb0f735536ca96400329649477e8d7f
parent6ed610b96840349e9a29c40e60dc926ce03d982b (diff)
parent87db61a4363ad9f42cf7b0d90ad13763dc01e3af (diff)
downloadgit-d0ce4d9024882b5363141288979ea3689eab9998.zip
git-d0ce4d9024882b5363141288979ea3689eab9998.tar.gz
git-d0ce4d9024882b5363141288979ea3689eab9998.tar.bz2
Merge branch 'js/trace2-cap-max-output-files'
The trace2 output, when sending them to files in a designated directory, can populate the directory with too many files; a mechanism is introduced to set the maximum number of files and discard further logs when the maximum is reached. * js/trace2-cap-max-output-files: trace2: write discard message to sentinel files trace2: discard new traces if target directory has too many files docs: clarify trace2 version invariants docs: mention trace2 target-dir mode in git-config
-rw-r--r--Documentation/config/trace2.txt6
-rw-r--r--Documentation/technical/api-trace2.txt31
-rw-r--r--Documentation/trace2-target-values.txt4
-rwxr-xr-xt/t0212-trace2-event.sh19
-rw-r--r--trace2/tr2_dst.c111
-rw-r--r--trace2/tr2_dst.h1
-rw-r--r--trace2/tr2_sysenv.c3
-rw-r--r--trace2/tr2_sysenv.h2
-rw-r--r--trace2/tr2_tgt_event.c31
-rw-r--r--trace2/tr2_tgt_normal.c2
-rw-r--r--trace2/tr2_tgt_perf.c2
11 files changed, 184 insertions, 28 deletions
diff --git a/Documentation/config/trace2.txt b/Documentation/config/trace2.txt
index 2edbfb0..4ce0b9a 100644
--- a/Documentation/config/trace2.txt
+++ b/Documentation/config/trace2.txt
@@ -54,3 +54,9 @@ trace2.destinationDebug::
By default, these errors are suppressed and tracing is
silently disabled. May be overridden by the
`GIT_TRACE2_DST_DEBUG` environment variable.
+
+trace2.maxFiles::
+ Integer. When writing trace files to a target directory, do not
+ write additional traces if we would exceed this many files. Instead,
+ write a sentinel file that will block further tracing to this
+ directory. Defaults to 0, which disables this check.
diff --git a/Documentation/technical/api-trace2.txt b/Documentation/technical/api-trace2.txt
index 71eb081..a045dbe 100644
--- a/Documentation/technical/api-trace2.txt
+++ b/Documentation/technical/api-trace2.txt
@@ -128,7 +128,7 @@ yields
------------
$ cat ~/log.event
-{"event":"version","sid":"sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.620713Z","file":"common-main.c","line":38,"evt":"1","exe":"2.20.1.155.g426c96fcdb"}
+{"event":"version","sid":"sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.620713Z","file":"common-main.c","line":38,"evt":"2","exe":"2.20.1.155.g426c96fcdb"}
{"event":"start","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621027Z","file":"common-main.c","line":39,"t_abs":0.001173,"argv":["git","version"]}
{"event":"cmd_name","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621122Z","file":"git.c","line":432,"name":"version","hierarchy":"version"}
{"event":"exit","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621236Z","file":"git.c","line":662,"t_abs":0.001227,"code":0}
@@ -142,10 +142,9 @@ system or global config value to one of the following:
include::../trace2-target-values.txt[]
-If the target already exists and is a directory, the traces will be
-written to files (one per process) underneath the given directory. They
-will be named according to the last component of the SID (optionally
-followed by a counter to avoid filename collisions).
+When trace files are written to a target directory, they will be named according
+to the last component of the SID (optionally followed by a counter to avoid
+filename collisions).
== Trace2 API
@@ -605,17 +604,35 @@ only present on the "start" and "atexit" events.
==== Event-Specific Key/Value Pairs
`"version"`::
- This event gives the version of the executable and the EVENT format.
+ This event gives the version of the executable and the EVENT format. It
+ should always be the first event in a trace session. The EVENT format
+ version will be incremented if new event types are added, if existing
+ fields are removed, or if there are significant changes in
+ interpretation of existing events or fields. Smaller changes, such as
+ adding a new field to an existing event, will not require an increment
+ to the EVENT format version.
+
------------
{
"event":"version",
...
- "evt":"1", # EVENT format version
+ "evt":"2", # EVENT format version
"exe":"2.20.1.155.g426c96fcdb" # git version
}
------------
+`"discard"`::
+ This event is written to the git-trace2-discard sentinel file if there
+ are too many files in the target trace directory (see the
+ trace2.maxFiles config option).
++
+------------
+{
+ "event":"discard",
+ ...
+}
+------------
+
`"start"`::
This event contains the complete argv received by main().
+
diff --git a/Documentation/trace2-target-values.txt b/Documentation/trace2-target-values.txt
index 27d3c64..3985b6d 100644
--- a/Documentation/trace2-target-values.txt
+++ b/Documentation/trace2-target-values.txt
@@ -2,7 +2,9 @@
* `0` or `false` - Disables the target.
* `1` or `true` - Writes to `STDERR`.
* `[2-9]` - Writes to the already opened file descriptor.
-* `<absolute-pathname>` - Writes to the file in append mode.
+* `<absolute-pathname>` - Writes to the file in append mode. If the target
+already exists and is a directory, the traces will be written to files (one
+per process) underneath the given directory.
* `af_unix:[<socket_type>:]<absolute-pathname>` - Write to a
Unix DomainSocket (on platforms that support them). Socket
type can be either `stream` or `dgram`; if omitted Git will
diff --git a/t/t0212-trace2-event.sh b/t/t0212-trace2-event.sh
index ff5b9cc..7065a1b 100755
--- a/t/t0212-trace2-event.sh
+++ b/t/t0212-trace2-event.sh
@@ -265,4 +265,23 @@ test_expect_success JSON_PP 'using global config, event stream, error event' '
test_cmp expect actual
'
+test_expect_success 'discard traces when there are too many files' '
+ mkdir trace_target_dir &&
+ test_when_finished "rm -r trace_target_dir" &&
+ (
+ GIT_TRACE2_MAX_FILES=5 &&
+ export GIT_TRACE2_MAX_FILES &&
+ cd trace_target_dir &&
+ test_seq $GIT_TRACE2_MAX_FILES >../expected_filenames.txt &&
+ xargs touch <../expected_filenames.txt &&
+ cd .. &&
+ GIT_TRACE2_EVENT="$(pwd)/trace_target_dir" test-tool trace2 001return 0
+ ) &&
+ echo git-trace2-discard >>expected_filenames.txt &&
+ ls trace_target_dir >ls_output.txt &&
+ test_cmp expected_filenames.txt ls_output.txt &&
+ head -n1 trace_target_dir/git-trace2-discard | grep \"event\":\"version\" &&
+ head -n2 trace_target_dir/git-trace2-discard | tail -n1 | grep \"event\":\"too_many_files\"
+'
+
test_done
diff --git a/trace2/tr2_dst.c b/trace2/tr2_dst.c
index c698575..ae052a0 100644
--- a/trace2/tr2_dst.c
+++ b/trace2/tr2_dst.c
@@ -8,6 +8,19 @@
*/
#define MAX_AUTO_ATTEMPTS 10
+/*
+ * Sentinel file used to detect when we should discard new traces to avoid
+ * writing too many trace files to a directory.
+ */
+#define DISCARD_SENTINEL_NAME "git-trace2-discard"
+
+/*
+ * When set to zero, disables directory file count checks. Otherwise, controls
+ * how many files we can write to a directory before entering discard mode.
+ * This can be overridden via the TR2_SYSENV_MAX_FILES setting.
+ */
+static int tr2env_max_files = 0;
+
static int tr2_dst_want_warning(void)
{
static int tr2env_dst_debug = -1;
@@ -32,9 +45,75 @@ void tr2_dst_trace_disable(struct tr2_dst *dst)
dst->need_close = 0;
}
+/*
+ * Check to make sure we're not overloading the target directory with too many
+ * files. First get the threshold (if present) from the config or envvar. If
+ * it's zero or unset, disable this check. Next check for the presence of a
+ * sentinel file, then check file count.
+ *
+ * Returns 0 if tracing should proceed as normal. Returns 1 if the sentinel file
+ * already exists, which means tracing should be disabled. Returns -1 if there
+ * are too many files but there was no sentinel file, which means we have
+ * created and should write traces to the sentinel file.
+ *
+ * We expect that some trace processing system is gradually collecting files
+ * from the target directory; after it removes the sentinel file we'll start
+ * writing traces again.
+ */
+static int tr2_dst_too_many_files(struct tr2_dst *dst, const char *tgt_prefix)
+{
+ int file_count = 0, max_files = 0, ret = 0;
+ const char *max_files_var;
+ DIR *dirp;
+ struct strbuf path = STRBUF_INIT, sentinel_path = STRBUF_INIT;
+ struct stat statbuf;
+
+ /* Get the config or envvar and decide if we should continue this check */
+ max_files_var = tr2_sysenv_get(TR2_SYSENV_MAX_FILES);
+ if (max_files_var && *max_files_var && ((max_files = atoi(max_files_var)) >= 0))
+ tr2env_max_files = max_files;
+
+ if (!tr2env_max_files) {
+ ret = 0;
+ goto cleanup;
+ }
+
+ strbuf_addstr(&path, tgt_prefix);
+ if (!is_dir_sep(path.buf[path.len - 1])) {
+ strbuf_addch(&path, '/');
+ }
+
+ /* check sentinel */
+ strbuf_addbuf(&sentinel_path, &path);
+ strbuf_addstr(&sentinel_path, DISCARD_SENTINEL_NAME);
+ if (!stat(sentinel_path.buf, &statbuf)) {
+ ret = 1;
+ goto cleanup;
+ }
+
+ /* check file count */
+ dirp = opendir(path.buf);
+ while (file_count < tr2env_max_files && dirp && readdir(dirp))
+ file_count++;
+ if (dirp)
+ closedir(dirp);
+
+ if (file_count >= tr2env_max_files) {
+ dst->too_many_files = 1;
+ dst->fd = open(sentinel_path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666);
+ ret = -1;
+ goto cleanup;
+ }
+
+cleanup:
+ strbuf_release(&path);
+ strbuf_release(&sentinel_path);
+ return ret;
+}
+
static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix)
{
- int fd;
+ int too_many_files;
const char *last_slash, *sid = tr2_sid_get();
struct strbuf path = STRBUF_INIT;
size_t base_path_len;
@@ -50,18 +129,29 @@ static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix)
strbuf_addstr(&path, sid);
base_path_len = path.len;
- for (attempt_count = 0; attempt_count < MAX_AUTO_ATTEMPTS; attempt_count++) {
- if (attempt_count > 0) {
- strbuf_setlen(&path, base_path_len);
- strbuf_addf(&path, ".%d", attempt_count);
+ too_many_files = tr2_dst_too_many_files(dst, tgt_prefix);
+ if (!too_many_files) {
+ for (attempt_count = 0; attempt_count < MAX_AUTO_ATTEMPTS; attempt_count++) {
+ if (attempt_count > 0) {
+ strbuf_setlen(&path, base_path_len);
+ strbuf_addf(&path, ".%d", attempt_count);
+ }
+
+ dst->fd = open(path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666);
+ if (dst->fd != -1)
+ break;
}
-
- fd = open(path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666);
- if (fd != -1)
- break;
+ } else if (too_many_files == 1) {
+ strbuf_release(&path);
+ if (tr2_dst_want_warning())
+ warning("trace2: not opening %s trace file due to too "
+ "many files in target directory %s",
+ tr2_sysenv_display_name(dst->sysenv_var),
+ tgt_prefix);
+ return 0;
}
- if (fd == -1) {
+ if (dst->fd == -1) {
if (tr2_dst_want_warning())
warning("trace2: could not open '%.*s' for '%s' tracing: %s",
(int) base_path_len, path.buf,
@@ -75,7 +165,6 @@ static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix)
strbuf_release(&path);
- dst->fd = fd;
dst->need_close = 1;
dst->initialized = 1;
diff --git a/trace2/tr2_dst.h b/trace2/tr2_dst.h
index 3adf3ba..b1a8c14 100644
--- a/trace2/tr2_dst.h
+++ b/trace2/tr2_dst.h
@@ -9,6 +9,7 @@ struct tr2_dst {
int fd;
unsigned int initialized : 1;
unsigned int need_close : 1;
+ unsigned int too_many_files : 1;
};
/*
diff --git a/trace2/tr2_sysenv.c b/trace2/tr2_sysenv.c
index 5958cfc..3c3792e 100644
--- a/trace2/tr2_sysenv.c
+++ b/trace2/tr2_sysenv.c
@@ -49,6 +49,9 @@ static struct tr2_sysenv_entry tr2_sysenv_settings[] = {
"trace2.perftarget" },
[TR2_SYSENV_PERF_BRIEF] = { "GIT_TRACE2_PERF_BRIEF",
"trace2.perfbrief" },
+
+ [TR2_SYSENV_MAX_FILES] = { "GIT_TRACE2_MAX_FILES",
+ "trace2.maxfiles" },
};
/* clang-format on */
diff --git a/trace2/tr2_sysenv.h b/trace2/tr2_sysenv.h
index 8dd82a7..d4364a7 100644
--- a/trace2/tr2_sysenv.h
+++ b/trace2/tr2_sysenv.h
@@ -24,6 +24,8 @@ enum tr2_sysenv_variable {
TR2_SYSENV_PERF,
TR2_SYSENV_PERF_BRIEF,
+ TR2_SYSENV_MAX_FILES,
+
TR2_SYSENV_MUST_BE_LAST
};
diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c
index 9bcac20..6353e8a 100644
--- a/trace2/tr2_tgt_event.c
+++ b/trace2/tr2_tgt_event.c
@@ -10,16 +10,17 @@
#include "trace2/tr2_tgt.h"
#include "trace2/tr2_tls.h"
-static struct tr2_dst tr2dst_event = { TR2_SYSENV_EVENT, 0, 0, 0 };
+static struct tr2_dst tr2dst_event = { TR2_SYSENV_EVENT, 0, 0, 0, 0 };
/*
- * The version number of the JSON data generated by the EVENT target
- * in this source file. Update this if you make a significant change
- * to the JSON fields or message structure. You probably do not need
- * to update this if you just add another call to one of the existing
- * TRACE2 API methods.
+ * The version number of the JSON data generated by the EVENT target in this
+ * source file. The version should be incremented if new event types are added,
+ * if existing fields are removed, or if there are significant changes in
+ * interpretation of existing events or fields. Smaller changes, such as adding
+ * a new field to an existing event, do not require an increment to the EVENT
+ * format version.
*/
-#define TR2_EVENT_VERSION "1"
+#define TR2_EVENT_VERSION "2"
/*
* Region nesting limit for messages written to the event target.
@@ -107,6 +108,19 @@ static void event_fmt_prepare(const char *event_name, const char *file,
jw_object_intmax(jw, "repo", repo->trace2_repo_id);
}
+static void fn_too_many_files_fl(const char *file, int line)
+{
+ const char *event_name = "too_many_files";
+ struct json_writer jw = JSON_WRITER_INIT;
+
+ jw_object_begin(&jw, 0);
+ event_fmt_prepare(event_name, file, line, NULL, &jw);
+ jw_end(&jw);
+
+ tr2_dst_write_line(&tr2dst_event, &jw.json);
+ jw_release(&jw);
+}
+
static void fn_version_fl(const char *file, int line)
{
const char *event_name = "version";
@@ -120,6 +134,9 @@ static void fn_version_fl(const char *file, int line)
tr2_dst_write_line(&tr2dst_event, &jw.json);
jw_release(&jw);
+
+ if (tr2dst_event.too_many_files)
+ fn_too_many_files_fl(file, line);
}
static void fn_start_fl(const char *file, int line,
diff --git a/trace2/tr2_tgt_normal.c b/trace2/tr2_tgt_normal.c
index 438ed05..31b602c 100644
--- a/trace2/tr2_tgt_normal.c
+++ b/trace2/tr2_tgt_normal.c
@@ -9,7 +9,7 @@
#include "trace2/tr2_tgt.h"
#include "trace2/tr2_tls.h"
-static struct tr2_dst tr2dst_normal = { TR2_SYSENV_NORMAL, 0, 0, 0 };
+static struct tr2_dst tr2dst_normal = { TR2_SYSENV_NORMAL, 0, 0, 0, 0 };
/*
* Use the TR2_SYSENV_NORMAL_BRIEF setting to omit the "<time> <file>:<line>"
diff --git a/trace2/tr2_tgt_perf.c b/trace2/tr2_tgt_perf.c
index fd979db..ffac802 100644
--- a/trace2/tr2_tgt_perf.c
+++ b/trace2/tr2_tgt_perf.c
@@ -11,7 +11,7 @@
#include "trace2/tr2_tgt.h"
#include "trace2/tr2_tls.h"
-static struct tr2_dst tr2dst_perf = { TR2_SYSENV_PERF, 0, 0, 0 };
+static struct tr2_dst tr2dst_perf = { TR2_SYSENV_PERF, 0, 0, 0, 0 };
/*
* Use TR2_SYSENV_PERF_BRIEF to omit the "<time> <file>:<line>"