Skip to content

Commit 59d94bc

Browse files
gitstergitster
committed
Merge branch 'np/maint-safer-pack' into np/pack
* np/maint-safer-pack: fixup_pack_header_footer(): use nicely aligned buffer sizes index-pack: use fixup_pack_header_footer()'s validation mode pack-objects: use fixup_pack_header_footer()'s validation mode improve reliability of fixup_pack_header_footer() pack-objects: improve returned information from write_one()
2 parents 7f31456 + d35825d commit 59d94bc

File tree

6 files changed

+112
-55
lines changed

6 files changed

+112
-55
lines changed

‎builtin-pack-objects.c‎

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -410,35 +410,33 @@ static unsigned long write_object(struct sha1file *f,
410410
return hdrlen + datalen;
411411
}
412412

413-
static off_t write_one(struct sha1file *f,
413+
static int write_one(struct sha1file *f,
414414
struct object_entry *e,
415-
off_t offset)
415+
off_t *offset)
416416
{
417417
unsigned long size;
418418

419419
/* offset is non zero if object is written already. */
420420
if (e->idx.offset || e->preferred_base)
421-
return offset;
421+
return 1;
422422

423423
/* if we are deltified, write out base object first. */
424-
if (e->delta) {
425-
offset = write_one(f, e->delta, offset);
426-
if (!offset)
427-
return 0;
428-
}
424+
if (e->delta && !write_one(f, e->delta, offset))
425+
return 0;
429426

430-
e->idx.offset = offset;
431-
size = write_object(f, e, offset);
427+
e->idx.offset = *offset;
428+
size = write_object(f, e, *offset);
432429
if (!size) {
433430
e->idx.offset = 0;
434431
return 0;
435432
}
436433
written_list[nr_written++] = &e->idx;
437434

438435
/* make sure off_t is sufficiently large not to wrap */
439-
if (offset > offset + size)
436+
if (*offset > *offset + size)
440437
die("pack too large for current definition of off_t");
441-
return offset + size;
438+
*offset += size;
439+
return 1;
442440
}
443441

444442
/* forward declaration for write_pack_file */
@@ -448,7 +446,7 @@ static void write_pack_file(void)
448446
{
449447
uint32_t i = 0, j;
450448
struct sha1file *f;
451-
off_t offset, offset_one, last_obj_offset = 0;
449+
off_t offset;
452450
struct pack_header hdr;
453451
uint32_t nr_remaining = nr_result;
454452
time_t last_mtime = 0;
@@ -480,11 +478,8 @@ static void write_pack_file(void)
480478
offset = sizeof(hdr);
481479
nr_written = 0;
482480
for (; i < nr_objects; i++) {
483-
last_obj_offset = offset;
484-
offset_one = write_one(f, objects + i, offset);
485-
if (!offset_one)
481+
if (!write_one(f, objects + i, &offset))
486482
break;
487-
offset = offset_one;
488483
display_progress(progress_state, written);
489484
}
490485

@@ -497,8 +492,9 @@ static void write_pack_file(void)
497492
} else if (nr_written == nr_remaining) {
498493
sha1close(f, sha1, CSUM_FSYNC);
499494
} else {
500-
int fd = sha1close(f, NULL, 0);
501-
fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written);
495+
int fd = sha1close(f, sha1, 0);
496+
fixup_pack_header_footer(fd, sha1, pack_tmp_name,
497+
nr_written, sha1, offset);
502498
close(fd);
503499
}
504500

‎csum-file.c‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ int sha1close(struct sha1file *f, unsigned char *result, unsigned int flags)
4242
sha1flush(f, offset);
4343
f->offset = 0;
4444
}
45+
SHA1_Final(f->buffer, &f->ctx);
46+
if (result)
47+
hashcpy(result, f->buffer);
4548
if (flags & (CSUM_CLOSE | CSUM_FSYNC)) {
4649
/* write checksum and close fd */
47-
SHA1_Final(f->buffer, &f->ctx);
48-
if (result)
49-
hashcpy(result, f->buffer);
5050
sha1flush(f, 20);
5151
if (flags & CSUM_FSYNC)
5252
fsync_or_die(f->fd, f->name);

‎fast-import.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -951,7 +951,8 @@ static void end_packfile(void)
951951

952952
close_pack_windows(pack_data);
953953
fixup_pack_header_footer(pack_data->pack_fd, pack_data->sha1,
954-
pack_data->pack_name, object_count);
954+
pack_data->pack_name, object_count,
955+
NULL, 0);
955956
close(pack_data->pack_fd);
956957
idx_name = keep_pack(create_index());
957958

‎index-pack.c‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -654,7 +654,7 @@ static void parse_pack_objects(unsigned char *sha1)
654654
}
655655
}
656656

657-
static int write_compressed(int fd, void *in, unsigned int size, uint32_t *obj_crc)
657+
static int write_compressed(struct sha1file *f, void *in, unsigned int size)
658658
{
659659
z_stream stream;
660660
unsigned long maxsize;
@@ -674,13 +674,12 @@ static int write_compressed(int fd, void *in, unsigned int size, uint32_t *obj_c
674674
deflateEnd(&stream);
675675

676676
size = stream.total_out;
677-
write_or_die(fd, out, size);
678-
*obj_crc = crc32(*obj_crc, out, size);
677+
sha1write(f, out, size);
679678
free(out);
680679
return size;
681680
}
682681

683-
static struct object_entry *append_obj_to_pack(
682+
static struct object_entry *append_obj_to_pack(struct sha1file *f,
684683
const unsigned char *sha1, void *buf,
685684
unsigned long size, enum object_type type)
686685
{
@@ -696,15 +695,15 @@ static struct object_entry *append_obj_to_pack(
696695
s >>= 7;
697696
}
698697
header[n++] = c;
699-
write_or_die(output_fd, header, n);
700-
obj[0].idx.crc32 = crc32(0, Z_NULL, 0);
701-
obj[0].idx.crc32 = crc32(obj[0].idx.crc32, header, n);
698+
crc32_begin(f);
699+
sha1write(f, header, n);
702700
obj[0].size = size;
703701
obj[0].hdr_size = n;
704702
obj[0].type = type;
705703
obj[0].real_type = type;
706704
obj[1].idx.offset = obj[0].idx.offset + n;
707-
obj[1].idx.offset += write_compressed(output_fd, buf, size, &obj[0].idx.crc32);
705+
obj[1].idx.offset += write_compressed(f, buf, size);
706+
obj[0].idx.crc32 = crc32_end(f);
708707
hashcpy(obj->idx.sha1, sha1);
709708
return obj;
710709
}
@@ -716,7 +715,7 @@ static int delta_pos_compare(const void *_a, const void *_b)
716715
return a->obj_no - b->obj_no;
717716
}
718717

719-
static void fix_unresolved_deltas(int nr_unresolved)
718+
static void fix_unresolved_deltas(struct sha1file *f, int nr_unresolved)
720719
{
721720
struct delta_entry **sorted_by_pos;
722721
int i, n = 0;
@@ -754,8 +753,8 @@ static void fix_unresolved_deltas(int nr_unresolved)
754753
if (check_sha1_signature(d->base.sha1, base_obj.data,
755754
base_obj.size, typename(type)))
756755
die("local object %s is corrupt", sha1_to_hex(d->base.sha1));
757-
base_obj.obj = append_obj_to_pack(d->base.sha1, base_obj.data,
758-
base_obj.size, type);
756+
base_obj.obj = append_obj_to_pack(f, d->base.sha1,
757+
base_obj.data, base_obj.size, type);
759758
link_base_data(NULL, &base_obj);
760759

761760
find_delta_children(&d->base, &first, &last);
@@ -875,7 +874,7 @@ int main(int argc, char **argv)
875874
const char *keep_name = NULL, *keep_msg = NULL;
876875
char *index_name_buf = NULL, *keep_name_buf = NULL;
877876
struct pack_idx_entry **idx_objects;
878-
unsigned char sha1[20];
877+
unsigned char pack_sha1[20];
879878
int nongit = 0;
880879

881880
setup_git_directory_gently(&nongit);
@@ -962,13 +961,15 @@ int main(int argc, char **argv)
962961
parse_pack_header();
963962
objects = xmalloc((nr_objects + 1) * sizeof(struct object_entry));
964963
deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
965-
parse_pack_objects(sha1);
964+
parse_pack_objects(pack_sha1);
966965
if (nr_deltas == nr_resolved_deltas) {
967966
stop_progress(&progress);
968967
/* Flush remaining pack final 20-byte SHA1. */
969968
flush();
970969
} else {
971970
if (fix_thin_pack) {
971+
struct sha1file *f;
972+
unsigned char read_sha1[20], tail_sha1[20];
972973
char msg[48];
973974
int nr_unresolved = nr_deltas - nr_resolved_deltas;
974975
int nr_objects_initial = nr_objects;
@@ -977,12 +978,19 @@ int main(int argc, char **argv)
977978
objects = xrealloc(objects,
978979
(nr_objects + nr_unresolved + 1)
979980
* sizeof(*objects));
980-
fix_unresolved_deltas(nr_unresolved);
981+
f = sha1fd(output_fd, curr_pack);
982+
fix_unresolved_deltas(f, nr_unresolved);
981983
sprintf(msg, "completed with %d local objects",
982984
nr_objects - nr_objects_initial);
983985
stop_progress_msg(&progress, msg);
984-
fixup_pack_header_footer(output_fd, sha1,
985-
curr_pack, nr_objects);
986+
sha1close(f, tail_sha1, 0);
987+
hashcpy(read_sha1, pack_sha1);
988+
fixup_pack_header_footer(output_fd, pack_sha1,
989+
curr_pack, nr_objects,
990+
read_sha1, consumed_bytes-20);
991+
if (hashcmp(read_sha1, tail_sha1) != 0)
992+
die("Unexpected tail checksum for %s "
993+
"(disk corruption?)", curr_pack);
986994
}
987995
if (nr_deltas != nr_resolved_deltas)
988996
die("pack has %d unresolved deltas",
@@ -995,13 +1003,13 @@ int main(int argc, char **argv)
9951003
idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *));
9961004
for (i = 0; i < nr_objects; i++)
9971005
idx_objects[i] = &objects[i].idx;
998-
curr_index = write_idx_file(index_name, idx_objects, nr_objects, sha1);
1006+
curr_index = write_idx_file(index_name, idx_objects, nr_objects, pack_sha1);
9991007
free(idx_objects);
10001008

10011009
final(pack_name, curr_pack,
10021010
index_name, curr_index,
10031011
keep_name, keep_msg,
1004-
sha1);
1012+
pack_sha1);
10051013
free(objects);
10061014
free(index_name_buf);
10071015
free(keep_name_buf);

‎pack-write.c‎

Lines changed: 65 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -144,41 +144,93 @@ char *write_idx_file(char *index_name, struct pack_idx_entry **objects,
144144
return index_name;
145145
}
146146

147+
/*
148+
* Update pack header with object_count and compute new SHA1 for pack data
149+
* associated to pack_fd, and write that SHA1 at the end. That new SHA1
150+
* is also returned in new_pack_sha1.
151+
*
152+
* If partial_pack_sha1 is non null, then the SHA1 of the existing pack
153+
* (without the header update) is computed and validated against the
154+
* one provided in partial_pack_sha1. The validation is performed at
155+
* partial_pack_offset bytes in the pack file. The SHA1 of the remaining
156+
* data (i.e. from partial_pack_offset to the end) is then computed and
157+
* returned in partial_pack_sha1.
158+
*
159+
* Note that new_pack_sha1 is updated last, so both new_pack_sha1 and
160+
* partial_pack_sha1 can refer to the same buffer if the caller is not
161+
* interested in the resulting SHA1 of pack data above partial_pack_offset.
162+
*/
147163
void fixup_pack_header_footer(int pack_fd,
148-
unsigned char *pack_file_sha1,
164+
unsigned char *new_pack_sha1,
149165
const char *pack_name,
150-
uint32_t object_count)
166+
uint32_t object_count,
167+
unsigned char *partial_pack_sha1,
168+
off_t partial_pack_offset)
151169
{
152-
static const int buf_sz = 128 * 1024;
153-
SHA_CTX c;
170+
int aligned_sz, buf_sz = 8 * 1024;
171+
SHA_CTX old_sha1_ctx, new_sha1_ctx;
154172
struct pack_header hdr;
155173
char *buf;
156174

175+
SHA1_Init(&old_sha1_ctx);
176+
SHA1_Init(&new_sha1_ctx);
177+
157178
if (lseek(pack_fd, 0, SEEK_SET) != 0)
158-
die("Failed seeking to start: %s", strerror(errno));
179+
die("Failed seeking to start of %s: %s", pack_name, strerror(errno));
159180
if (read_in_full(pack_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
160181
die("Unable to reread header of %s: %s", pack_name, strerror(errno));
161182
if (lseek(pack_fd, 0, SEEK_SET) != 0)
162-
die("Failed seeking to start: %s", strerror(errno));
183+
die("Failed seeking to start of %s: %s", pack_name, strerror(errno));
184+
SHA1_Update(&old_sha1_ctx, &hdr, sizeof(hdr));
163185
hdr.hdr_entries = htonl(object_count);
186+
SHA1_Update(&new_sha1_ctx, &hdr, sizeof(hdr));
164187
write_or_die(pack_fd, &hdr, sizeof(hdr));
165-
166-
SHA1_Init(&c);
167-
SHA1_Update(&c, &hdr, sizeof(hdr));
188+
partial_pack_offset -= sizeof(hdr);
168189

169190
buf = xmalloc(buf_sz);
191+
aligned_sz = buf_sz - sizeof(hdr);
170192
for (;;) {
171-
ssize_t n = xread(pack_fd, buf, buf_sz);
193+
ssize_t m, n;
194+
m = (partial_pack_sha1 && partial_pack_offset < aligned_sz) ?
195+
partial_pack_offset : aligned_sz;
196+
n = xread(pack_fd, buf, m);
172197
if (!n)
173198
break;
174199
if (n < 0)
175200
die("Failed to checksum %s: %s", pack_name, strerror(errno));
176-
SHA1_Update(&c, buf, n);
201+
SHA1_Update(&new_sha1_ctx, buf, n);
202+
203+
aligned_sz -= n;
204+
if (!aligned_sz)
205+
aligned_sz = buf_sz;
206+
207+
if (!partial_pack_sha1)
208+
continue;
209+
210+
SHA1_Update(&old_sha1_ctx, buf, n);
211+
partial_pack_offset -= n;
212+
if (partial_pack_offset == 0) {
213+
unsigned char sha1[20];
214+
SHA1_Final(sha1, &old_sha1_ctx);
215+
if (hashcmp(sha1, partial_pack_sha1) != 0)
216+
die("Unexpected checksum for %s "
217+
"(disk corruption?)", pack_name);
218+
/*
219+
* Now let's compute the SHA1 of the remainder of the
220+
* pack, which also means making partial_pack_offset
221+
* big enough not to matter anymore.
222+
*/
223+
SHA1_Init(&old_sha1_ctx);
224+
partial_pack_offset = ~partial_pack_offset;
225+
partial_pack_offset -= MSB(partial_pack_offset, 1);
226+
}
177227
}
178228
free(buf);
179229

180-
SHA1_Final(pack_file_sha1, &c);
181-
write_or_die(pack_fd, pack_file_sha1, 20);
230+
if (partial_pack_sha1)
231+
SHA1_Final(partial_pack_sha1, &old_sha1_ctx);
232+
SHA1_Final(new_pack_sha1, &new_sha1_ctx);
233+
write_or_die(pack_fd, new_pack_sha1, 20);
182234
fsync_or_die(pack_fd, pack_name);
183235
}
184236

‎pack.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ struct pack_idx_entry {
5858
extern char *write_idx_file(char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
5959
extern int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, off_t offset, off_t len, unsigned int nr);
6060
extern int verify_pack(struct packed_git *);
61-
extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t);
61+
extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t, unsigned char *, off_t);
6262
extern char *index_pack_lockfile(int fd);
6363

6464
#define PH_ERROR_EOF (-1)

0 commit comments

Comments
 (0)