Skip to content

Commit 187c0d3

Browse files
gitstergitster
committed
Merge branch 'sb/submodule-parallel-fetch'
Add a framework to spawn a group of processes in parallel, and use it to run "git fetch --recurse-submodules" in parallel. Rerolled and this seems to be a lot cleaner. The merge of the earlier one to 'next' has been reverted. * sb/submodule-parallel-fetch: submodules: allow parallel fetching, add tests and documentation fetch_populated_submodules: use new parallel job processing run-command: add an asynchronous parallel child processor sigchain: add command to pop all common signals strbuf: add strbuf_read_once to read without blocking xread: poll on non blocking fds submodule.c: write "Fetching submodule <foo>" to stderr
2 parents 7b9d1b9 + 62104ba commit 187c0d3

File tree

15 files changed

+731
-74
lines changed

15 files changed

+731
-74
lines changed

‎Documentation/fetch-options.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,13 @@ ifndef::git-pull[]
100100
reference to a commit that isn't already in the local submodule
101101
clone.
102102

103+
-j::
104+
--jobs=<n>::
105+
Number of parallel children to be used for fetching submodules.
106+
Each will fetch from different submodules, such that fetching many
107+
submodules will be faster. By default submodules will be fetched
108+
one at a time.
109+
103110
--no-recurse-submodules::
104111
Disable recursive fetching of submodules (this has the same effect as
105112
using the '--recurse-submodules=no' option).

‎builtin/fetch.c‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ static int prune = -1; /* unspecified */
3737
static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
3838
static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
3939
static int tags = TAGS_DEFAULT, unshallow, update_shallow;
40+
static int max_children = 1;
4041
static const char *depth;
4142
static const char *upload_pack;
4243
static struct strbuf default_rla = STRBUF_INIT;
@@ -99,6 +100,8 @@ static struct option builtin_fetch_options[] = {
99100
N_("fetch all tags and associated objects"), TAGS_SET),
100101
OPT_SET_INT('n', NULL, &tags,
101102
N_("do not fetch all tags (--no-tags)"), TAGS_UNSET),
103+
OPT_INTEGER('j', "jobs", &max_children,
104+
N_("number of submodules fetched in parallel")),
102105
OPT_BOOL('p', "prune", &prune,
103106
N_("prune remote-tracking branches no longer on remote")),
104107
{ OPTION_CALLBACK, 0, "recurse-submodules", NULL, N_("on-demand"),
@@ -1213,7 +1216,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
12131216
result = fetch_populated_submodules(&options,
12141217
submodule_prefix,
12151218
recurse_submodules,
1216-
verbosity < 0);
1219+
verbosity < 0,
1220+
max_children);
12171221
argv_array_clear(&options);
12181222
}
12191223

‎builtin/pull.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ static int opt_force;
9595
static char *opt_tags;
9696
static char *opt_prune;
9797
static char *opt_recurse_submodules;
98+
static char *max_children;
9899
static int opt_dry_run;
99100
static char *opt_keep;
100101
static char *opt_depth;
@@ -178,6 +179,9 @@ static struct option pull_options[] = {
178179
N_("on-demand"),
179180
N_("control recursive fetching of submodules"),
180181
PARSE_OPT_OPTARG),
182+
OPT_PASSTHRU('j', "jobs", &max_children, N_("n"),
183+
N_("number of submodules pulled in parallel"),
184+
PARSE_OPT_OPTARG),
181185
OPT_BOOL(0, "dry-run", &opt_dry_run,
182186
N_("dry run")),
183187
OPT_PASSTHRU('k', "keep", &opt_keep, NULL,
@@ -525,6 +529,8 @@ static int run_fetch(const char *repo, const char **refspecs)
525529
argv_array_push(&args, opt_prune);
526530
if (opt_recurse_submodules)
527531
argv_array_push(&args, opt_recurse_submodules);
532+
if (max_children)
533+
argv_array_push(&args, max_children);
528534
if (opt_dry_run)
529535
argv_array_push(&args, "--dry-run");
530536
if (opt_keep)

0 commit comments

Comments
 (0)