Skip to content

Commit d0ce4d9

Browse files
gitstergitster
committed
Merge branch 'js/trace2-cap-max-output-files'
The trace2 output, when sending them to files in a designated directory, can populate the directory with too many files; a mechanism is introduced to set the maximum number of files and discard further logs when the maximum is reached. * js/trace2-cap-max-output-files: trace2: write discard message to sentinel files trace2: discard new traces if target directory has too many files docs: clarify trace2 version invariants docs: mention trace2 target-dir mode in git-config
2 parents 6ed610b + 87db61a commit d0ce4d9

File tree

11 files changed

+184
-28
lines changed

11 files changed

+184
-28
lines changed

‎Documentation/config/trace2.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,9 @@ trace2.destinationDebug::
5454
By default, these errors are suppressed and tracing is
5555
silently disabled. May be overridden by the
5656
`GIT_TRACE2_DST_DEBUG` environment variable.
57+
58+
trace2.maxFiles::
59+
Integer. When writing trace files to a target directory, do not
60+
write additional traces if we would exceed this many files. Instead,
61+
write a sentinel file that will block further tracing to this
62+
directory. Defaults to 0, which disables this check.

‎Documentation/technical/api-trace2.txt‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ yields
128128

129129
------------
130130
$ cat ~/log.event
131-
{"event":"version","sid":"sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.620713Z","file":"common-main.c","line":38,"evt":"1","exe":"2.20.1.155.g426c96fcdb"}
131+
{"event":"version","sid":"sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.620713Z","file":"common-main.c","line":38,"evt":"2","exe":"2.20.1.155.g426c96fcdb"}
132132
{"event":"start","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621027Z","file":"common-main.c","line":39,"t_abs":0.001173,"argv":["git","version"]}
133133
{"event":"cmd_name","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621122Z","file":"git.c","line":432,"name":"version","hierarchy":"version"}
134134
{"event":"exit","sid":"20190408T191610.507018Z-H9b68c35f-P000059a8","thread":"main","time":"2019-01-16T17:28:42.621236Z","file":"git.c","line":662,"t_abs":0.001227,"code":0}
@@ -142,10 +142,9 @@ system or global config value to one of the following:
142142

143143
include::../trace2-target-values.txt[]
144144

145-
If the target already exists and is a directory, the traces will be
146-
written to files (one per process) underneath the given directory. They
147-
will be named according to the last component of the SID (optionally
148-
followed by a counter to avoid filename collisions).
145+
When trace files are written to a target directory, they will be named according
146+
to the last component of the SID (optionally followed by a counter to avoid
147+
filename collisions).
149148

150149
== Trace2 API
151150

@@ -605,17 +604,35 @@ only present on the "start" and "atexit" events.
605604
==== Event-Specific Key/Value Pairs
606605

607606
`"version"`::
608-
This event gives the version of the executable and the EVENT format.
607+
This event gives the version of the executable and the EVENT format. It
608+
should always be the first event in a trace session. The EVENT format
609+
version will be incremented if new event types are added, if existing
610+
fields are removed, or if there are significant changes in
611+
interpretation of existing events or fields. Smaller changes, such as
612+
adding a new field to an existing event, will not require an increment
613+
to the EVENT format version.
609614
+
610615
------------
611616
{
612617
"event":"version",
613618
...
614-
"evt":"1", # EVENT format version
619+
"evt":"2", # EVENT format version
615620
"exe":"2.20.1.155.g426c96fcdb" # git version
616621
}
617622
------------
618623

624+
`"discard"`::
625+
This event is written to the git-trace2-discard sentinel file if there
626+
are too many files in the target trace directory (see the
627+
trace2.maxFiles config option).
628+
+
629+
------------
630+
{
631+
"event":"discard",
632+
...
633+
}
634+
------------
635+
619636
`"start"`::
620637
This event contains the complete argv received by main().
621638
+

‎Documentation/trace2-target-values.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
* `0` or `false` - Disables the target.
33
* `1` or `true` - Writes to `STDERR`.
44
* `[2-9]` - Writes to the already opened file descriptor.
5-
* `<absolute-pathname>` - Writes to the file in append mode.
5+
* `<absolute-pathname>` - Writes to the file in append mode. If the target
6+
already exists and is a directory, the traces will be written to files (one
7+
per process) underneath the given directory.
68
* `af_unix:[<socket_type>:]<absolute-pathname>` - Write to a
79
Unix DomainSocket (on platforms that support them). Socket
810
type can be either `stream` or `dgram`; if omitted Git will

‎t/t0212-trace2-event.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,4 +265,23 @@ test_expect_success JSON_PP 'using global config, event stream, error event' '
265265
test_cmp expect actual
266266
'
267267

268+
test_expect_success 'discard traces when there are too many files' '
269+
mkdir trace_target_dir &&
270+
test_when_finished "rm -r trace_target_dir" &&
271+
(
272+
GIT_TRACE2_MAX_FILES=5 &&
273+
export GIT_TRACE2_MAX_FILES &&
274+
cd trace_target_dir &&
275+
test_seq $GIT_TRACE2_MAX_FILES >../expected_filenames.txt &&
276+
xargs touch <../expected_filenames.txt &&
277+
cd .. &&
278+
GIT_TRACE2_EVENT="$(pwd)/trace_target_dir" test-tool trace2 001return 0
279+
) &&
280+
echo git-trace2-discard >>expected_filenames.txt &&
281+
ls trace_target_dir >ls_output.txt &&
282+
test_cmp expected_filenames.txt ls_output.txt &&
283+
head -n1 trace_target_dir/git-trace2-discard | grep \"event\":\"version\" &&
284+
head -n2 trace_target_dir/git-trace2-discard | tail -n1 | grep \"event\":\"too_many_files\"
285+
'
286+
268287
test_done

‎trace2/tr2_dst.c‎

Lines changed: 100 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,19 @@
88
*/
99
#define MAX_AUTO_ATTEMPTS 10
1010

11+
/*
12+
* Sentinel file used to detect when we should discard new traces to avoid
13+
* writing too many trace files to a directory.
14+
*/
15+
#define DISCARD_SENTINEL_NAME "git-trace2-discard"
16+
17+
/*
18+
* When set to zero, disables directory file count checks. Otherwise, controls
19+
* how many files we can write to a directory before entering discard mode.
20+
* This can be overridden via the TR2_SYSENV_MAX_FILES setting.
21+
*/
22+
static int tr2env_max_files = 0;
23+
1124
static int tr2_dst_want_warning(void)
1225
{
1326
static int tr2env_dst_debug = -1;
@@ -32,9 +45,75 @@ void tr2_dst_trace_disable(struct tr2_dst *dst)
3245
dst->need_close = 0;
3346
}
3447

48+
/*
49+
* Check to make sure we're not overloading the target directory with too many
50+
* files. First get the threshold (if present) from the config or envvar. If
51+
* it's zero or unset, disable this check. Next check for the presence of a
52+
* sentinel file, then check file count.
53+
*
54+
* Returns 0 if tracing should proceed as normal. Returns 1 if the sentinel file
55+
* already exists, which means tracing should be disabled. Returns -1 if there
56+
* are too many files but there was no sentinel file, which means we have
57+
* created and should write traces to the sentinel file.
58+
*
59+
* We expect that some trace processing system is gradually collecting files
60+
* from the target directory; after it removes the sentinel file we'll start
61+
* writing traces again.
62+
*/
63+
static int tr2_dst_too_many_files(struct tr2_dst *dst, const char *tgt_prefix)
64+
{
65+
int file_count = 0, max_files = 0, ret = 0;
66+
const char *max_files_var;
67+
DIR *dirp;
68+
struct strbuf path = STRBUF_INIT, sentinel_path = STRBUF_INIT;
69+
struct stat statbuf;
70+
71+
/* Get the config or envvar and decide if we should continue this check */
72+
max_files_var = tr2_sysenv_get(TR2_SYSENV_MAX_FILES);
73+
if (max_files_var && *max_files_var && ((max_files = atoi(max_files_var)) >= 0))
74+
tr2env_max_files = max_files;
75+
76+
if (!tr2env_max_files) {
77+
ret = 0;
78+
goto cleanup;
79+
}
80+
81+
strbuf_addstr(&path, tgt_prefix);
82+
if (!is_dir_sep(path.buf[path.len - 1])) {
83+
strbuf_addch(&path, '/');
84+
}
85+
86+
/* check sentinel */
87+
strbuf_addbuf(&sentinel_path, &path);
88+
strbuf_addstr(&sentinel_path, DISCARD_SENTINEL_NAME);
89+
if (!stat(sentinel_path.buf, &statbuf)) {
90+
ret = 1;
91+
goto cleanup;
92+
}
93+
94+
/* check file count */
95+
dirp = opendir(path.buf);
96+
while (file_count < tr2env_max_files && dirp && readdir(dirp))
97+
file_count++;
98+
if (dirp)
99+
closedir(dirp);
100+
101+
if (file_count >= tr2env_max_files) {
102+
dst->too_many_files = 1;
103+
dst->fd = open(sentinel_path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666);
104+
ret = -1;
105+
goto cleanup;
106+
}
107+
108+
cleanup:
109+
strbuf_release(&path);
110+
strbuf_release(&sentinel_path);
111+
return ret;
112+
}
113+
35114
static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix)
36115
{
37-
int fd;
116+
int too_many_files;
38117
const char *last_slash, *sid = tr2_sid_get();
39118
struct strbuf path = STRBUF_INIT;
40119
size_t base_path_len;
@@ -50,18 +129,29 @@ static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix)
50129
strbuf_addstr(&path, sid);
51130
base_path_len = path.len;
52131

53-
for (attempt_count = 0; attempt_count < MAX_AUTO_ATTEMPTS; attempt_count++) {
54-
if (attempt_count > 0) {
55-
strbuf_setlen(&path, base_path_len);
56-
strbuf_addf(&path, ".%d", attempt_count);
132+
too_many_files = tr2_dst_too_many_files(dst, tgt_prefix);
133+
if (!too_many_files) {
134+
for (attempt_count = 0; attempt_count < MAX_AUTO_ATTEMPTS; attempt_count++) {
135+
if (attempt_count > 0) {
136+
strbuf_setlen(&path, base_path_len);
137+
strbuf_addf(&path, ".%d", attempt_count);
138+
}
139+
140+
dst->fd = open(path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666);
141+
if (dst->fd != -1)
142+
break;
57143
}
58-
59-
fd = open(path.buf, O_WRONLY | O_CREAT | O_EXCL, 0666);
60-
if (fd != -1)
61-
break;
144+
} else if (too_many_files == 1) {
145+
strbuf_release(&path);
146+
if (tr2_dst_want_warning())
147+
warning("trace2: not opening %s trace file due to too "
148+
"many files in target directory %s",
149+
tr2_sysenv_display_name(dst->sysenv_var),
150+
tgt_prefix);
151+
return 0;
62152
}
63153

64-
if (fd == -1) {
154+
if (dst->fd == -1) {
65155
if (tr2_dst_want_warning())
66156
warning("trace2: could not open '%.*s' for '%s' tracing: %s",
67157
(int) base_path_len, path.buf,
@@ -75,7 +165,6 @@ static int tr2_dst_try_auto_path(struct tr2_dst *dst, const char *tgt_prefix)
75165

76166
strbuf_release(&path);
77167

78-
dst->fd = fd;
79168
dst->need_close = 1;
80169
dst->initialized = 1;
81170

‎trace2/tr2_dst.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ struct tr2_dst {
99
int fd;
1010
unsigned int initialized : 1;
1111
unsigned int need_close : 1;
12+
unsigned int too_many_files : 1;
1213
};
1314

1415
/*

‎trace2/tr2_sysenv.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ static struct tr2_sysenv_entry tr2_sysenv_settings[] = {
4949
"trace2.perftarget" },
5050
[TR2_SYSENV_PERF_BRIEF] = { "GIT_TRACE2_PERF_BRIEF",
5151
"trace2.perfbrief" },
52+
53+
[TR2_SYSENV_MAX_FILES] = { "GIT_TRACE2_MAX_FILES",
54+
"trace2.maxfiles" },
5255
};
5356
/* clang-format on */
5457

‎trace2/tr2_sysenv.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ enum tr2_sysenv_variable {
2424
TR2_SYSENV_PERF,
2525
TR2_SYSENV_PERF_BRIEF,
2626

27+
TR2_SYSENV_MAX_FILES,
28+
2729
TR2_SYSENV_MUST_BE_LAST
2830
};
2931

‎trace2/tr2_tgt_event.c‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,17 @@
1010
#include "trace2/tr2_tgt.h"
1111
#include "trace2/tr2_tls.h"
1212

13-
static struct tr2_dst tr2dst_event = { TR2_SYSENV_EVENT, 0, 0, 0 };
13+
static struct tr2_dst tr2dst_event = { TR2_SYSENV_EVENT, 0, 0, 0, 0 };
1414

1515
/*
16-
* The version number of the JSON data generated by the EVENT target
17-
* in this source file. Update this if you make a significant change
18-
* to the JSON fields or message structure. You probably do not need
19-
* to update this if you just add another call to one of the existing
20-
* TRACE2 API methods.
16+
* The version number of the JSON data generated by the EVENT target in this
17+
* source file. The version should be incremented if new event types are added,
18+
* if existing fields are removed, or if there are significant changes in
19+
* interpretation of existing events or fields. Smaller changes, such as adding
20+
* a new field to an existing event, do not require an increment to the EVENT
21+
* format version.
2122
*/
22-
#define TR2_EVENT_VERSION "1"
23+
#define TR2_EVENT_VERSION "2"
2324

2425
/*
2526
* Region nesting limit for messages written to the event target.
@@ -107,6 +108,19 @@ static void event_fmt_prepare(const char *event_name, const char *file,
107108
jw_object_intmax(jw, "repo", repo->trace2_repo_id);
108109
}
109110

111+
static void fn_too_many_files_fl(const char *file, int line)
112+
{
113+
const char *event_name = "too_many_files";
114+
struct json_writer jw = JSON_WRITER_INIT;
115+
116+
jw_object_begin(&jw, 0);
117+
event_fmt_prepare(event_name, file, line, NULL, &jw);
118+
jw_end(&jw);
119+
120+
tr2_dst_write_line(&tr2dst_event, &jw.json);
121+
jw_release(&jw);
122+
}
123+
110124
static void fn_version_fl(const char *file, int line)
111125
{
112126
const char *event_name = "version";
@@ -120,6 +134,9 @@ static void fn_version_fl(const char *file, int line)
120134

121135
tr2_dst_write_line(&tr2dst_event, &jw.json);
122136
jw_release(&jw);
137+
138+
if (tr2dst_event.too_many_files)
139+
fn_too_many_files_fl(file, line);
123140
}
124141

125142
static void fn_start_fl(const char *file, int line,

‎trace2/tr2_tgt_normal.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
#include "trace2/tr2_tgt.h"
1010
#include "trace2/tr2_tls.h"
1111

12-
static struct tr2_dst tr2dst_normal = { TR2_SYSENV_NORMAL, 0, 0, 0 };
12+
static struct tr2_dst tr2dst_normal = { TR2_SYSENV_NORMAL, 0, 0, 0, 0 };
1313

1414
/*
1515
* Use the TR2_SYSENV_NORMAL_BRIEF setting to omit the "<time> <file>:<line>"

0 commit comments

Comments
 (0)