Skip to content

Commit dd4b732

Browse files
jonathantanmygitster
jonathantanmy
authored and
gitster
committed
upload-pack: send part of packfile response as uri
Teach upload-pack to send part of its packfile response as URIs. An administrator may configure a repository with one or more "uploadpack.blobpackfileuri" lines, each line containing an OID, a pack hash, and a URI. A client may configure fetch.uriprotocols to be a comma-separated list of protocols that it is willing to use to fetch additional packfiles - this list will be sent to the server. Whenever an object with one of those OIDs would appear in the packfile transmitted by upload-pack, the server may exclude that object, and instead send the URI. The client will then download the packs referred to by those URIs before performing the connectivity check. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
1 parent 9da69a6 commit dd4b732

File tree

4 files changed

+343
-10
lines changed

4 files changed

+343
-10
lines changed

‎builtin/pack-objects.c‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,8 @@ static unsigned long window_memory_limit = 0;
117117

118118
static struct list_objects_filter_options filter_options;
119119

120+
static struct string_list uri_protocols = STRING_LIST_INIT_NODUP;
121+
120122
enum missing_action {
121123
MA_ERROR = 0, /* fail if any missing objects are encountered */
122124
MA_ALLOW_ANY, /* silently allow ALL missing objects */
@@ -125,6 +127,15 @@ enum missing_action {
125127
static enum missing_action arg_missing_action;
126128
static show_object_fn fn_show_object;
127129

130+
struct configured_exclusion {
131+
struct oidmap_entry e;
132+
char *pack_hash_hex;
133+
char *uri;
134+
};
135+
static struct oidmap configured_exclusions;
136+
137+
static struct oidset excluded_by_config;
138+
128139
/*
129140
* stats
130141
*/
@@ -969,6 +980,25 @@ static void write_reused_pack(struct hashfile *f)
969980
unuse_pack(&w_curs);
970981
}
971982

983+
static void write_excluded_by_configs(void)
984+
{
985+
struct oidset_iter iter;
986+
const struct object_id *oid;
987+
988+
oidset_iter_init(&excluded_by_config, &iter);
989+
while ((oid = oidset_iter_next(&iter))) {
990+
struct configured_exclusion *ex =
991+
oidmap_get(&configured_exclusions, oid);
992+
993+
if (!ex)
994+
BUG("configured exclusion wasn't configured");
995+
write_in_full(1, ex->pack_hash_hex, strlen(ex->pack_hash_hex));
996+
write_in_full(1, " ", 1);
997+
write_in_full(1, ex->uri, strlen(ex->uri));
998+
write_in_full(1, "\n", 1);
999+
}
1000+
}
1001+
9721002
static const char no_split_warning[] = N_(
9731003
"disabling bitmap writing, packs are split due to pack.packSizeLimit"
9741004
);
@@ -1266,6 +1296,25 @@ static int want_object_in_pack(const struct object_id *oid,
12661296
}
12671297
}
12681298

1299+
if (uri_protocols.nr) {
1300+
struct configured_exclusion *ex =
1301+
oidmap_get(&configured_exclusions, oid);
1302+
int i;
1303+
const char *p;
1304+
1305+
if (ex) {
1306+
for (i = 0; i < uri_protocols.nr; i++) {
1307+
if (skip_prefix(ex->uri,
1308+
uri_protocols.items[i].string,
1309+
&p) &&
1310+
*p == ':') {
1311+
oidset_insert(&excluded_by_config, oid);
1312+
return 0;
1313+
}
1314+
}
1315+
}
1316+
}
1317+
12691318
return 1;
12701319
}
12711320

@@ -2864,6 +2913,29 @@ static int git_pack_config(const char *k, const char *v, void *cb)
28642913
pack_idx_opts.version);
28652914
return 0;
28662915
}
2916+
if (!strcmp(k, "uploadpack.blobpackfileuri")) {
2917+
struct configured_exclusion *ex = xmalloc(sizeof(*ex));
2918+
const char *oid_end, *pack_end;
2919+
/*
2920+
* Stores the pack hash. This is not a true object ID, but is
2921+
* of the same form.
2922+
*/
2923+
struct object_id pack_hash;
2924+
2925+
if (parse_oid_hex(v, &ex->e.oid, &oid_end) ||
2926+
*oid_end != ' ' ||
2927+
parse_oid_hex(oid_end + 1, &pack_hash, &pack_end) ||
2928+
*pack_end != ' ')
2929+
die(_("value of uploadpack.blobpackfileuri must be "
2930+
"of the form '<object-hash> <pack-hash> <uri>' (got '%s')"), v);
2931+
if (oidmap_get(&configured_exclusions, &ex->e.oid))
2932+
die(_("object already configured in another "
2933+
"uploadpack.blobpackfileuri (got '%s')"), v);
2934+
ex->pack_hash_hex = xcalloc(1, pack_end - oid_end);
2935+
memcpy(ex->pack_hash_hex, oid_end + 1, pack_end - oid_end - 1);
2936+
ex->uri = xstrdup(pack_end + 1);
2937+
oidmap_put(&configured_exclusions, ex);
2938+
}
28672939
return git_default_config(k, v, cb);
28682940
}
28692941

@@ -3462,6 +3534,9 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
34623534
N_("do not pack objects in promisor packfiles")),
34633535
OPT_BOOL(0, "delta-islands", &use_delta_islands,
34643536
N_("respect islands during delta compression")),
3537+
OPT_STRING_LIST(0, "uri-protocol", &uri_protocols,
3538+
N_("protocol"),
3539+
N_("exclude any configured uploadpack.blobpackfileuri with this protocol")),
34653540
OPT_END(),
34663541
};
34673542

@@ -3650,6 +3725,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
36503725
}
36513726

36523727
trace2_region_enter("pack-objects", "write-pack-file", the_repository);
3728+
write_excluded_by_configs();
36533729
write_pack_file();
36543730
trace2_region_leave("pack-objects", "write-pack-file", the_repository);
36553731

‎fetch-pack.c‎

Lines changed: 108 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ static int server_supports_filtering;
3838
static struct shallow_lock shallow_lock;
3939
static const char *alternate_shallow_file;
4040
static struct strbuf fsck_msg_types = STRBUF_INIT;
41+
static struct string_list uri_protocols = STRING_LIST_INIT_DUP;
4142

4243
/* Remember to update object flag allocation in object.h */
4344
#define COMPLETE (1U << 0)
@@ -795,6 +796,7 @@ static void write_promisor_file(const char *keep_name,
795796

796797
static int get_pack(struct fetch_pack_args *args,
797798
int xd[2], struct string_list *pack_lockfiles,
799+
int only_packfile,
798800
struct ref **sought, int nr_sought)
799801
{
800802
struct async demux;
@@ -855,8 +857,15 @@ static int get_pack(struct fetch_pack_args *args,
855857
"--keep=fetch-pack %"PRIuMAX " on %s",
856858
(uintmax_t)getpid(), hostname);
857859
}
858-
if (args->check_self_contained_and_connected)
860+
if (only_packfile && args->check_self_contained_and_connected)
859861
argv_array_push(&cmd.args, "--check-self-contained-and-connected");
862+
else
863+
/*
864+
* We cannot perform any connectivity checks because
865+
* not all packs have been downloaded; let the caller
866+
* have this responsibility.
867+
*/
868+
args->check_self_contained_and_connected = 0;
860869
/*
861870
* If we're obtaining the filename of a lockfile, we'll use
862871
* that filename to write a .promisor file with more
@@ -1068,7 +1077,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
10681077
alternate_shallow_file = setup_temporary_shallow(si->shallow);
10691078
else
10701079
alternate_shallow_file = NULL;
1071-
if (get_pack(args, fd, pack_lockfiles, sought, nr_sought))
1080+
if (get_pack(args, fd, pack_lockfiles, 1, sought, nr_sought))
10721081
die(_("git fetch-pack: fetch failed."));
10731082

10741083
all_done:
@@ -1222,6 +1231,26 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
12221231
warning("filtering not recognized by server, ignoring");
12231232
}
12241233

1234+
if (server_supports_feature("fetch", "packfile-uris", 0)) {
1235+
int i;
1236+
struct strbuf to_send = STRBUF_INIT;
1237+
1238+
for (i = 0; i < uri_protocols.nr; i++) {
1239+
const char *s = uri_protocols.items[i].string;
1240+
1241+
if (!strcmp(s, "https") || !strcmp(s, "http")) {
1242+
if (to_send.len)
1243+
strbuf_addch(&to_send, ',');
1244+
strbuf_addstr(&to_send, s);
1245+
}
1246+
}
1247+
if (to_send.len) {
1248+
packet_buf_write(&req_buf, "packfile-uris %s",
1249+
to_send.buf);
1250+
strbuf_release(&to_send);
1251+
}
1252+
}
1253+
12251254
/* add wants */
12261255
add_wants(args->no_dependents, wants, &req_buf);
12271256

@@ -1444,6 +1473,21 @@ static void receive_wanted_refs(struct packet_reader *reader,
14441473
die(_("error processing wanted refs: %d"), reader->status);
14451474
}
14461475

1476+
static void receive_packfile_uris(struct packet_reader *reader,
1477+
struct string_list *uris)
1478+
{
1479+
process_section_header(reader, "packfile-uris", 0);
1480+
while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1481+
if (reader->pktlen < the_hash_algo->hexsz ||
1482+
reader->line[the_hash_algo->hexsz] != ' ')
1483+
die("expected '<hash> <uri>', got: %s\n", reader->line);
1484+
1485+
string_list_append(uris, reader->line);
1486+
}
1487+
if (reader->status != PACKET_READ_DELIM)
1488+
die("expected DELIM");
1489+
}
1490+
14471491
enum fetch_state {
14481492
FETCH_CHECK_LOCAL = 0,
14491493
FETCH_SEND_REQUEST,
@@ -1470,6 +1514,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
14701514
struct fetch_negotiator negotiator_alloc;
14711515
struct fetch_negotiator *negotiator;
14721516
int seen_ack = 0;
1517+
struct string_list packfile_uris = STRING_LIST_INIT_DUP;
1518+
int i;
14731519

14741520
if (args->no_dependents) {
14751521
negotiator = NULL;
@@ -1558,9 +1604,12 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
15581604
if (process_section_header(&reader, "wanted-refs", 1))
15591605
receive_wanted_refs(&reader, sought, nr_sought);
15601606

1561-
/* get the pack */
1607+
/* get the pack(s) */
1608+
if (process_section_header(&reader, "packfile-uris", 1))
1609+
receive_packfile_uris(&reader, &packfile_uris);
15621610
process_section_header(&reader, "packfile", 0);
1563-
if (get_pack(args, fd, pack_lockfiles, sought, nr_sought))
1611+
if (get_pack(args, fd, pack_lockfiles,
1612+
!packfile_uris.nr, sought, nr_sought))
15641613
die(_("git fetch-pack: fetch failed."));
15651614

15661615
state = FETCH_DONE;
@@ -1570,8 +1619,55 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
15701619
}
15711620
}
15721621

1622+
for (i = 0; i < packfile_uris.nr; i++) {
1623+
struct child_process cmd = CHILD_PROCESS_INIT;
1624+
char packname[GIT_MAX_HEXSZ + 1];
1625+
const char *uri = packfile_uris.items[i].string +
1626+
the_hash_algo->hexsz + 1;
1627+
1628+
argv_array_push(&cmd.args, "http-fetch");
1629+
argv_array_pushf(&cmd.args, "--packfile=%.*s",
1630+
(int) the_hash_algo->hexsz,
1631+
packfile_uris.items[i].string);
1632+
argv_array_push(&cmd.args, uri);
1633+
cmd.git_cmd = 1;
1634+
cmd.no_stdin = 1;
1635+
cmd.out = -1;
1636+
if (start_command(&cmd))
1637+
die("fetch-pack: unable to spawn http-fetch");
1638+
1639+
if (read_in_full(cmd.out, packname, 5) < 0 ||
1640+
memcmp(packname, "keep\t", 5))
1641+
die("fetch-pack: expected keep then TAB at start of http-fetch output");
1642+
1643+
if (read_in_full(cmd.out, packname,
1644+
the_hash_algo->hexsz + 1) < 0 ||
1645+
packname[the_hash_algo->hexsz] != '\n')
1646+
die("fetch-pack: expected hash then LF at end of http-fetch output");
1647+
1648+
packname[the_hash_algo->hexsz] = '\0';
1649+
1650+
close(cmd.out);
1651+
1652+
if (finish_command(&cmd))
1653+
die("fetch-pack: unable to finish http-fetch");
1654+
1655+
if (memcmp(packfile_uris.items[i].string, packname,
1656+
the_hash_algo->hexsz))
1657+
die("fetch-pack: pack downloaded from %s does not match expected hash %.*s",
1658+
uri, (int) the_hash_algo->hexsz,
1659+
packfile_uris.items[i].string);
1660+
1661+
string_list_append_nodup(pack_lockfiles,
1662+
xstrfmt("%s/pack/pack-%s.keep",
1663+
get_object_directory(),
1664+
packname));
1665+
}
1666+
string_list_clear(&packfile_uris, 0);
1667+
15731668
if (negotiator)
15741669
negotiator->release(negotiator);
1670+
15751671
oidset_clear(&common);
15761672
return ref;
15771673
}
@@ -1608,6 +1704,14 @@ static void fetch_pack_config(void)
16081704
git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
16091705
git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
16101706
git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1707+
if (!uri_protocols.nr) {
1708+
char *str;
1709+
1710+
if (!git_config_get_string("fetch.uriprotocols", &str) && str) {
1711+
string_list_split(&uri_protocols, str, ',', -1);
1712+
free(str);
1713+
}
1714+
}
16111715

16121716
git_config(fetch_pack_config_cb, NULL);
16131717
}

0 commit comments

Comments
 (0)