Skip to content

Commit c38138c

Browse files
author
Linus Torvalds
committed
git-pack-objects: write the pack files with a SHA1 csum
We want to be able to check their integrity later, and putting the sha1-sum of the contents at the end is a good thing. The writing routines are generic, so we could try to re-use them for the index file, instead of having the same logic duplicated. Update unpack-objects to know about the extra 20 bytes at the end of the index.
1 parent 9b66ec0 commit c38138c

File tree

5 files changed

+158
-62
lines changed

5 files changed

+158
-62
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ install: $(PROG) $(SCRIPTS)
4545

4646
LIB_OBJS=read-cache.o sha1_file.o usage.o object.o commit.o tree.o blob.o \
4747
tag.o delta.o date.o index.o diff-delta.o patch-delta.o entry.o \
48-
epoch.o refs.o
48+
epoch.o refs.o csum-file.o
4949
LIB_FILE=libgit.a
50-
LIB_H=cache.h object.h blob.h tree.h commit.h tag.h delta.h epoch.h
50+
LIB_H=cache.h object.h blob.h tree.h commit.h tag.h delta.h epoch.h csum-file.h
5151

5252
LIB_H += strbuf.h
5353
LIB_OBJS += strbuf.o

‎csum-file.c‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
/*
2+
* csum-file.c
3+
*
4+
* Copyright (C) 2005 Linus Torvalds
5+
*
6+
* Simple file write infrastructure for writing SHA1-summed
7+
* files. Useful when you write a file that you want to be
8+
* able to verify hasn't been messed with afterwards.
9+
*/
10+
#include "cache.h"
11+
#include "csum-file.h"
12+
13+
static int sha1flush(struct sha1file *f, unsigned int count)
14+
{
15+
void *buf = f->buffer;
16+
17+
for (;;) {
18+
int ret = write(f->fd, buf, count);
19+
if (ret > 0) {
20+
buf += ret;
21+
count -= ret;
22+
if (count)
23+
continue;
24+
return 0;
25+
}
26+
if (!ret)
27+
die("sha1 file write error. Out of diskspace");
28+
if (errno == EAGAIN || errno == EINTR)
29+
continue;
30+
die("sha1 file write error (%s)", strerror(errno));
31+
}
32+
}
33+
34+
int sha1close(struct sha1file *f)
35+
{
36+
unsigned offset = f->offset;
37+
if (offset) {
38+
SHA1_Update(&f->ctx, f->buffer, offset);
39+
sha1flush(f, offset);
40+
}
41+
SHA1_Final(f->buffer, &f->ctx);
42+
sha1flush(f, 20);
43+
return 0;
44+
}
45+
46+
int sha1write(struct sha1file *f, void *buf, unsigned int count)
47+
{
48+
while (count) {
49+
unsigned offset = f->offset;
50+
unsigned left = sizeof(f->buffer) - offset;
51+
unsigned nr = count > left ? left : count;
52+
53+
memcpy(f->buffer + offset, buf, nr);
54+
count -= nr;
55+
offset += nr;
56+
left -= nr;
57+
if (!left) {
58+
SHA1_Update(&f->ctx, f->buffer, offset);
59+
sha1flush(f, offset);
60+
offset = 0;
61+
}
62+
f->offset = offset;
63+
}
64+
return 0;
65+
}
66+
67+
struct sha1file *sha1create(const char *fmt, ...)
68+
{
69+
static char filename[PATH_MAX];
70+
struct sha1file *f;
71+
unsigned len;
72+
va_list arg;
73+
int fd;
74+
75+
va_start(arg, fmt);
76+
len = vsnprintf(filename, PATH_MAX, fmt, arg);
77+
va_end(arg);
78+
79+
if (len >= PATH_MAX)
80+
die("you wascally wabbit, you");
81+
fd = open(filename, O_CREAT | O_EXCL | O_WRONLY, 0644);
82+
if (fd < 0)
83+
die("unable to open %s (%s)", filename, strerror(errno));
84+
f = xmalloc(sizeof(*f));
85+
f->fd = fd;
86+
f->error = 0;
87+
f->offset = 0;
88+
SHA1_Init(&f->ctx);
89+
return f;
90+
}
91+
92+
int sha1write_compressed(struct sha1file *f, void *in, unsigned int size)
93+
{
94+
z_stream stream;
95+
unsigned long maxsize;
96+
void *out;
97+
98+
memset(&stream, 0, sizeof(stream));
99+
deflateInit(&stream, Z_DEFAULT_COMPRESSION);
100+
maxsize = deflateBound(&stream, size);
101+
out = xmalloc(maxsize);
102+
103+
/* Compress it */
104+
stream.next_in = in;
105+
stream.avail_in = size;
106+
107+
stream.next_out = out;
108+
stream.avail_out = maxsize;
109+
110+
while (deflate(&stream, Z_FINISH) == Z_OK)
111+
/* nothing */;
112+
deflateEnd(&stream);
113+
114+
size = stream.total_out;
115+
sha1write(f, out, size);
116+
free(out);
117+
return size;
118+
}
119+
120+

‎csum-file.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#ifndef CSUM_FILE_H
2+
#define CSUM_FILE_H
3+
4+
/* A SHA1-protected file */
5+
struct sha1file {
6+
int fd, error;
7+
unsigned long offset;
8+
SHA_CTX ctx;
9+
unsigned char buffer[8192];
10+
};
11+
12+
extern struct sha1file *sha1create(const char *fmt, ...);
13+
extern int sha1close(struct sha1file *);
14+
extern int sha1write(struct sha1file *, void *, unsigned int);
15+
extern int sha1write_compressed(struct sha1file *, void *, unsigned int);
16+
17+
#endif

‎pack-objects.c‎

Lines changed: 11 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include "cache.h"
33
#include "object.h"
44
#include "delta.h"
5+
#include "csum-file.h"
56

67
static const char pack_usage[] = "git-pack-objects [--window=N] [--depth=N] base-name < object-list";
78

@@ -29,51 +30,6 @@ static struct object_entry *objects = NULL;
2930
static int nr_objects = 0, nr_alloc = 0;
3031
static const char *base_name;
3132

32-
struct myfile {
33-
int fd;
34-
unsigned long chars;
35-
unsigned char buffer[8192];
36-
};
37-
38-
static FILE *create_file(const char *suffix)
39-
{
40-
static char filename[PATH_MAX];
41-
unsigned len;
42-
43-
len = snprintf(filename, PATH_MAX, "%s.%s", base_name, suffix);
44-
if (len >= PATH_MAX)
45-
die("you wascally wabbit, you");
46-
return fopen(filename, "w");
47-
}
48-
49-
static unsigned long fwrite_compressed(void *in, unsigned long size, FILE *f)
50-
{
51-
z_stream stream;
52-
unsigned long maxsize;
53-
void *out;
54-
55-
memset(&stream, 0, sizeof(stream));
56-
deflateInit(&stream, Z_DEFAULT_COMPRESSION);
57-
maxsize = deflateBound(&stream, size);
58-
out = xmalloc(maxsize);
59-
60-
/* Compress it */
61-
stream.next_in = in;
62-
stream.avail_in = size;
63-
64-
stream.next_out = out;
65-
stream.avail_out = maxsize;
66-
67-
while (deflate(&stream, Z_FINISH) == Z_OK)
68-
/* nothing */;
69-
deflateEnd(&stream);
70-
71-
size = stream.total_out;
72-
fwrite(out, size, 1, f);
73-
free(out);
74-
return size;
75-
}
76-
7733
static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
7834
{
7935
unsigned long othersize, delta_size;
@@ -92,7 +48,7 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
9248
return delta_buf;
9349
}
9450

95-
static unsigned long write_object(FILE *f, struct object_entry *entry)
51+
static unsigned long write_object(struct sha1file *f, struct object_entry *entry)
9652
{
9753
unsigned long size;
9854
char type[10];
@@ -121,16 +77,16 @@ static unsigned long write_object(FILE *f, struct object_entry *entry)
12177
}
12278
datalen = htonl(size);
12379
memcpy(header+1, &datalen, 4);
124-
fwrite(header, hdrlen, 1, f);
125-
datalen = fwrite_compressed(buf, size, f);
80+
sha1write(f, header, hdrlen);
81+
datalen = sha1write_compressed(f, buf, size);
12682
free(buf);
12783
return hdrlen + datalen;
12884
}
12985

13086
static void write_pack_file(void)
13187
{
13288
int i;
133-
FILE *f = create_file("pack");
89+
struct sha1file *f = sha1create("%s.%s", base_name, "pack");
13490
unsigned long offset = 0;
13591
unsigned long mb;
13692

@@ -139,15 +95,15 @@ static void write_pack_file(void)
13995
entry->offset = offset;
14096
offset += write_object(f, entry);
14197
}
142-
fclose(f);
98+
sha1close(f);
14399
mb = offset >> 20;
144100
offset &= 0xfffff;
145101
}
146102

147103
static void write_index_file(void)
148104
{
149105
int i;
150-
FILE *f = create_file("idx");
106+
struct sha1file *f = sha1create("%s.%s", base_name, "idx");
151107
struct object_entry **list = sorted_by_sha;
152108
struct object_entry **last = list + nr_objects;
153109
unsigned int array[256];
@@ -168,7 +124,7 @@ static void write_index_file(void)
168124
array[i] = htonl(next - sorted_by_sha);
169125
list = next;
170126
}
171-
fwrite(array, 256, sizeof(int), f);
127+
sha1write(f, array, 256 * sizeof(int));
172128

173129
/*
174130
* Write the actual SHA1 entries..
@@ -177,10 +133,10 @@ static void write_index_file(void)
177133
for (i = 0; i < nr_objects; i++) {
178134
struct object_entry *entry = *list++;
179135
unsigned int offset = htonl(entry->offset);
180-
fwrite(&offset, 4, 1, f);
181-
fwrite(entry->sha1, 20, 1, f);
136+
sha1write(f, &offset, 4);
137+
sha1write(f, entry->sha1, 20);
182138
}
183-
fclose(f);
139+
sha1close(f);
184140
}
185141

186142
static void add_object_entry(unsigned char *sha1, unsigned int hash)

‎unpack-objects.c‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ static int check_index(void)
6161
unsigned int nr;
6262
int i;
6363

64-
if (index_size < 4*256)
64+
if (index_size < 4*256 + 20)
6565
return error("index file too small");
6666
nr = 0;
6767
for (i = 0; i < 256; i++) {
@@ -70,11 +70,14 @@ static int check_index(void)
7070
return error("non-monotonic index");
7171
nr = n;
7272
}
73-
if (index_size != 4*256 + nr * 24) {
74-
printf("index_size=%lu, expected %u (%u)\n",
75-
index_size, 4*256 + nr * 24, nr);
73+
/*
74+
* Total size:
75+
* - 256 index entries 4 bytes each
76+
* - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
77+
* - 20-byte SHA1 file checksum
78+
*/
79+
if (index_size != 4*256 + nr * 24 + 20)
7680
return error("wrong index file size");
77-
}
7881

7982
nr_entries = nr;
8083
pack_list = xmalloc(nr * sizeof(struct pack_entry *));

0 commit comments

Comments
 (0)