Skip to content

Commit 6bed209

Browse files
gitstergitster
committed
Merge branch 'jh/partial-clone'
The machinery to clone & fetch, which in turn involves packing and unpacking objects, have been told how to omit certain objects using the filtering mechanism introduced by the jh/object-filtering topic, and also mark the resulting pack as a promisor pack to tolerate missing objects, taking advantage of the mechanism introduced by the jh/fsck-promisors topic. * jh/partial-clone: t5616: test bulk prefetch after partial fetch fetch: inherit filter-spec from partial clone t5616: end-to-end tests for partial clone fetch-pack: restore save_commit_buffer after use unpack-trees: batch fetching of missing blobs clone: partial clone partial-clone: define partial clone settings in config fetch: support filters fetch: refactor calculation of remote list fetch-pack: test support excluding large blobs fetch-pack: add --no-filter fetch-pack, index-pack, transport: partial clone upload-pack: add object filtering for partial clone
2 parents f3d618d + 3aa6694 commit 6bed209

26 files changed

+657
-30
lines changed

‎Documentation/config.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3343,6 +3343,10 @@ uploadpack.packObjectsHook::
33433343
was run. I.e., `upload-pack` will feed input intended for
33443344
`pack-objects` to the hook, and expects a completed packfile on
33453345
stdout.
3346+
3347+
uploadpack.allowFilter::
3348+
If this option is set, `upload-pack` will advertise partial
3349+
clone and partial fetch object filtering.
33463350
+
33473351
Note that this configuration variable is ignored if it is seen in the
33483352
repository-level config (this is a safety measure against fetching from

‎Documentation/technical/pack-protocol.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@ out of what the server said it could do with the first 'want' line.
241241
upload-request = want-list
242242
*shallow-line
243243
*1depth-request
244+
[filter-request]
244245
flush-pkt
245246

246247
want-list = first-want
@@ -256,6 +257,8 @@ out of what the server said it could do with the first 'want' line.
256257
additional-want = PKT-LINE("want" SP obj-id)
257258

258259
depth = 1*DIGIT
260+
261+
filter-request = PKT-LINE("filter" SP filter-spec)
259262
----
260263

261264
Clients MUST send all the obj-ids it wants from the reference
@@ -278,6 +281,11 @@ complete those commits. Commits whose parents are not received as a
278281
result are defined as shallow and marked as such in the server. This
279282
information is sent back to the client in the next step.
280283

284+
The client can optionally request that pack-objects omit various
285+
objects from the packfile using one of several filtering techniques.
286+
These are intended for use with partial clone and partial fetch
287+
operations. See `rev-list` for possible "filter-spec" values.
288+
281289
Once all the 'want's and 'shallow's (and optional 'deepen') are
282290
transferred, clients MUST send a flush-pkt, to tell the server side
283291
that it is done sending the list.

‎Documentation/technical/protocol-capabilities.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,3 +309,11 @@ to accept a signed push certificate, and asks the <nonce> to be
309309
included in the push certificate. A send-pack client MUST NOT
310310
send a push-cert packet unless the receive-pack server advertises
311311
this capability.
312+
313+
filter
314+
------
315+
316+
If the upload-pack server advertises the 'filter' capability,
317+
fetch-pack may send "filter" commands to request a partial clone
318+
or partial fetch and request that the server omit various objects
319+
from the packfile.

‎builtin/clone.c‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "run-command.h"
2727
#include "connected.h"
2828
#include "packfile.h"
29+
#include "list-objects-filter-options.h"
2930

3031
/*
3132
* Overall FIXMEs:
@@ -60,6 +61,7 @@ static struct string_list option_optional_reference = STRING_LIST_INIT_NODUP;
6061
static int option_dissociate;
6162
static int max_jobs = -1;
6263
static struct string_list option_recurse_submodules = STRING_LIST_INIT_NODUP;
64+
static struct list_objects_filter_options filter_options;
6365

6466
static int recurse_submodules_cb(const struct option *opt,
6567
const char *arg, int unset)
@@ -135,6 +137,7 @@ static struct option builtin_clone_options[] = {
135137
TRANSPORT_FAMILY_IPV4),
136138
OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"),
137139
TRANSPORT_FAMILY_IPV6),
140+
OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
138141
OPT_END()
139142
};
140143

@@ -893,6 +896,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
893896
struct refspec *refspec;
894897
const char *fetch_pattern;
895898

899+
fetch_if_missing = 0;
900+
896901
packet_trace_identity("clone");
897902
argc = parse_options(argc, argv, prefix, builtin_clone_options,
898903
builtin_clone_usage, 0);
@@ -1090,6 +1095,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
10901095
warning(_("--shallow-since is ignored in local clones; use file:// instead."));
10911096
if (option_not.nr)
10921097
warning(_("--shallow-exclude is ignored in local clones; use file:// instead."));
1098+
if (filter_options.choice)
1099+
warning(_("--filter is ignored in local clones; use file:// instead."));
10931100
if (!access(mkpath("%s/shallow", path), F_OK)) {
10941101
if (option_local > 0)
10951102
warning(_("source repository is shallow, ignoring --local"));
@@ -1118,7 +1125,13 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
11181125
transport_set_option(transport, TRANS_OPT_UPLOADPACK,
11191126
option_upload_pack);
11201127

1121-
if (transport->smart_options && !deepen)
1128+
if (filter_options.choice) {
1129+
transport_set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER,
1130+
filter_options.filter_spec);
1131+
transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
1132+
}
1133+
1134+
if (transport->smart_options && !deepen && !filter_options.choice)
11221135
transport->smart_options->check_self_contained_and_connected = 1;
11231136

11241137
refs = transport_get_remote_refs(transport);
@@ -1178,13 +1191,17 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
11781191
write_refspec_config(src_ref_prefix, our_head_points_at,
11791192
remote_head_points_at, &branch_top);
11801193

1194+
if (filter_options.choice)
1195+
partial_clone_register("origin", &filter_options);
1196+
11811197
if (is_local)
11821198
clone_local(path, git_dir);
11831199
else if (refs && complete_refs_before_fetch)
11841200
transport_fetch_refs(transport, mapped_refs);
11851201

11861202
update_remote_refs(refs, mapped_refs, remote_head_points_at,
1187-
branch_top.buf, reflog_msg.buf, transport, !is_local);
1203+
branch_top.buf, reflog_msg.buf, transport,
1204+
!is_local && !filter_options.choice);
11881205

11891206
update_head(our_head_points_at, remote_head, reflog_msg.buf);
11901207

@@ -1205,6 +1222,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
12051222
}
12061223

12071224
junk_mode = JUNK_LEAVE_REPO;
1225+
fetch_if_missing = 1;
12081226
err = checkout(submodule_progress);
12091227

12101228
strbuf_release(&reflog_msg);

‎builtin/fetch-pack.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,14 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
153153
args.no_dependents = 1;
154154
continue;
155155
}
156+
if (skip_prefix(arg, ("--" CL_ARG__FILTER "="), &arg)) {
157+
parse_list_objects_filter(&args.filter_options, arg);
158+
continue;
159+
}
160+
if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) {
161+
list_objects_filter_set_no_filter(&args.filter_options);
162+
continue;
163+
}
156164
usage(fetch_pack_usage);
157165
}
158166
if (deepen_not.nr)

‎builtin/fetch.c‎

Lines changed: 77 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "argv-array.h"
2020
#include "utf8.h"
2121
#include "packfile.h"
22+
#include "list-objects-filter-options.h"
2223

2324
static const char * const builtin_fetch_usage[] = {
2425
N_("git fetch [<options>] [<repository> [<refspec>...]]"),
@@ -56,6 +57,7 @@ static int recurse_submodules_default = RECURSE_SUBMODULES_ON_DEMAND;
5657
static int shown_url = 0;
5758
static int refmap_alloc, refmap_nr;
5859
static const char **refmap_array;
60+
static struct list_objects_filter_options filter_options;
5961

6062
static int git_fetch_config(const char *k, const char *v, void *cb)
6163
{
@@ -161,6 +163,7 @@ static struct option builtin_fetch_options[] = {
161163
TRANSPORT_FAMILY_IPV4),
162164
OPT_SET_INT('6', "ipv6", &family, N_("use IPv6 addresses only"),
163165
TRANSPORT_FAMILY_IPV6),
166+
OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options),
164167
OPT_END()
165168
};
166169

@@ -1045,6 +1048,11 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
10451048
set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, "yes");
10461049
if (update_shallow)
10471050
set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes");
1051+
if (filter_options.choice) {
1052+
set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER,
1053+
filter_options.filter_spec);
1054+
set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");
1055+
}
10481056
return transport;
10491057
}
10501058

@@ -1265,6 +1273,56 @@ static int fetch_multiple(struct string_list *list)
12651273
return result;
12661274
}
12671275

1276+
/*
1277+
* Fetching from the promisor remote should use the given filter-spec
1278+
* or inherit the default filter-spec from the config.
1279+
*/
1280+
static inline void fetch_one_setup_partial(struct remote *remote)
1281+
{
1282+
/*
1283+
* Explicit --no-filter argument overrides everything, regardless
1284+
* of any prior partial clones and fetches.
1285+
*/
1286+
if (filter_options.no_filter)
1287+
return;
1288+
1289+
/*
1290+
* If no prior partial clone/fetch and the current fetch DID NOT
1291+
* request a partial-fetch, do a normal fetch.
1292+
*/
1293+
if (!repository_format_partial_clone && !filter_options.choice)
1294+
return;
1295+
1296+
/*
1297+
* If this is the FIRST partial-fetch request, we enable partial
1298+
* on this repo and remember the given filter-spec as the default
1299+
* for subsequent fetches to this remote.
1300+
*/
1301+
if (!repository_format_partial_clone && filter_options.choice) {
1302+
partial_clone_register(remote->name, &filter_options);
1303+
return;
1304+
}
1305+
1306+
/*
1307+
* We are currently limited to only ONE promisor remote and only
1308+
* allow partial-fetches from the promisor remote.
1309+
*/
1310+
if (strcmp(remote->name, repository_format_partial_clone)) {
1311+
if (filter_options.choice)
1312+
die(_("--filter can only be used with the remote configured in core.partialClone"));
1313+
return;
1314+
}
1315+
1316+
/*
1317+
* Do a partial-fetch from the promisor remote using either the
1318+
* explicitly given filter-spec or inherit the filter-spec from
1319+
* the config.
1320+
*/
1321+
if (!filter_options.choice)
1322+
partial_clone_get_default_filter_spec(&filter_options);
1323+
return;
1324+
}
1325+
12681326
static int fetch_one(struct remote *remote, int argc, const char **argv)
12691327
{
12701328
static const char **refs = NULL;
@@ -1320,12 +1378,14 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
13201378
{
13211379
int i;
13221380
struct string_list list = STRING_LIST_INIT_DUP;
1323-
struct remote *remote;
1381+
struct remote *remote = NULL;
13241382
int result = 0;
13251383
struct argv_array argv_gc_auto = ARGV_ARRAY_INIT;
13261384

13271385
packet_trace_identity("fetch");
13281386

1387+
fetch_if_missing = 0;
1388+
13291389
/* Record the command line for the reflog */
13301390
strbuf_addstr(&default_rla, "fetch");
13311391
for (i = 1; i < argc; i++)
@@ -1359,38 +1419,49 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
13591419
if (depth || deepen_since || deepen_not.nr)
13601420
deepen = 1;
13611421

1422+
if (filter_options.choice && !repository_format_partial_clone)
1423+
die("--filter can only be used when extensions.partialClone is set");
1424+
13621425
if (all) {
13631426
if (argc == 1)
13641427
die(_("fetch --all does not take a repository argument"));
13651428
else if (argc > 1)
13661429
die(_("fetch --all does not make sense with refspecs"));
13671430
(void) for_each_remote(get_one_remote_for_fetch, &list);
1368-
result = fetch_multiple(&list);
13691431
} else if (argc == 0) {
13701432
/* No arguments -- use default remote */
13711433
remote = remote_get(NULL);
1372-
result = fetch_one(remote, argc, argv);
13731434
} else if (multiple) {
13741435
/* All arguments are assumed to be remotes or groups */
13751436
for (i = 0; i < argc; i++)
13761437
if (!add_remote_or_group(argv[i], &list))
13771438
die(_("No such remote or remote group: %s"), argv[i]);
1378-
result = fetch_multiple(&list);
13791439
} else {
13801440
/* Single remote or group */
13811441
(void) add_remote_or_group(argv[0], &list);
13821442
if (list.nr > 1) {
13831443
/* More than one remote */
13841444
if (argc > 1)
13851445
die(_("Fetching a group and specifying refspecs does not make sense"));
1386-
result = fetch_multiple(&list);
13871446
} else {
13881447
/* Zero or one remotes */
13891448
remote = remote_get(argv[0]);
1390-
result = fetch_one(remote, argc-1, argv+1);
1449+
argc--;
1450+
argv++;
13911451
}
13921452
}
13931453

1454+
if (remote) {
1455+
if (filter_options.choice || repository_format_partial_clone)
1456+
fetch_one_setup_partial(remote);
1457+
result = fetch_one(remote, argc, argv);
1458+
} else {
1459+
if (filter_options.choice)
1460+
die(_("--filter can only be used with the remote configured in core.partialClone"));
1461+
/* TODO should this also die if we have a previous partial-clone? */
1462+
result = fetch_multiple(&list);
1463+
}
1464+
13941465
if (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) {
13951466
struct argv_array options = ARGV_ARRAY_INIT;
13961467

‎builtin/rev-list.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,7 @@ int cmd_rev_list(int argc, const char **argv, const char *prefix)
460460
continue;
461461
}
462462
if (!strcmp(arg, ("--no-" CL_ARG__FILTER))) {
463-
list_objects_filter_release(&filter_options);
463+
list_objects_filter_set_no_filter(&filter_options);
464464
continue;
465465
}
466466
if (!strcmp(arg, "--filter-print-omitted")) {

‎cache.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,7 @@ extern int grafts_replace_parents;
915915
#define GIT_REPO_VERSION_READ 1
916916
extern int repository_format_precious_objects;
917917
extern char *repository_format_partial_clone;
918+
extern const char *core_partial_clone_filter_default;
918919

919920
struct repository_format {
920921
int version;

‎config.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,6 +1251,11 @@ static int git_default_core_config(const char *var, const char *value)
12511251
return 0;
12521252
}
12531253

1254+
if (!strcmp(var, "core.partialclonefilter")) {
1255+
return git_config_string(&core_partial_clone_filter_default,
1256+
var, value);
1257+
}
1258+
12541259
/* Add other config variables here and to Documentation/config.txt. */
12551260
return 0;
12561261
}

‎connected.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
5656
argv_array_push(&rev_list.args,"rev-list");
5757
argv_array_push(&rev_list.args, "--objects");
5858
argv_array_push(&rev_list.args, "--stdin");
59+
if (repository_format_partial_clone)
60+
argv_array_push(&rev_list.args, "--exclude-promisor-objects");
5961
argv_array_push(&rev_list.args, "--not");
6062
argv_array_push(&rev_list.args, "--all");
6163
argv_array_push(&rev_list.args, "--quiet");

0 commit comments

Comments
 (0)