Skip to content

Commit 1a2e1a7

Browse files
gitstergitster
committed
Merge branch 'mh/mmap-packed-refs'
Operations that do not touch (majority of) packed refs have been optimized by making accesses to packed-refs file lazy; we no longer pre-parse everything, and an access to a single ref in the packed-refs does not touch majority of irrelevant refs, either. * mh/mmap-packed-refs: (21 commits) packed-backend.c: rename a bunch of things and update comments mmapped_ref_iterator: inline into `packed_ref_iterator` ref_cache: remove support for storing peeled values packed_ref_store: get rid of the `ref_cache` entirely ref_store: implement `refs_peel_ref()` generically packed_read_raw_ref(): read the reference from the mmapped buffer packed_ref_iterator_begin(): iterate using `mmapped_ref_iterator` read_packed_refs(): ensure that references are ordered when read packed_ref_cache: keep the `packed-refs` file mmapped if possible packed-backend.c: reorder some definitions mmapped_ref_iterator_advance(): no peeled value for broken refs mmapped_ref_iterator: add iterator over a packed-refs file packed_ref_cache: remember the file-wide peeling state read_packed_refs(): read references with minimal copying read_packed_refs(): make parsing of the header line more robust read_packed_refs(): only check for a header at the top of the file read_packed_refs(): use mmap to read the `packed-refs` file die_unterminated_line(), die_invalid_line(): new functions packed_ref_cache: add a backlink to the associated `packed_ref_store` prefix_ref_iterator: break when we leave the prefix ...
2 parents 9124cca + cff28ca commit 1a2e1a7

File tree

9 files changed

+848
-369
lines changed

9 files changed

+848
-369
lines changed

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,9 @@ all::
205205
#
206206
# Define NO_MMAP if you want to avoid mmap.
207207
#
208+
# Define MMAP_PREVENTS_DELETE if a file that is currently mmapped cannot be
209+
# deleted or cannot be replaced using rename().
210+
#
208211
# Define NO_SYS_POLL_H if you don't have sys/poll.h.
209212
#
210213
# Define NO_POLL if you do not have or don't want to use poll().
@@ -1391,6 +1394,9 @@ else
13911394
COMPAT_OBJS += compat/win32mmap.o
13921395
endif
13931396
endif
1397+
ifdef MMAP_PREVENTS_DELETE
1398+
BASIC_CFLAGS += -DMMAP_PREVENTS_DELETE
1399+
endif
13941400
ifdef OBJECT_CREATION_USES_RENAMES
13951401
COMPAT_CFLAGS += -DOBJECT_CREATION_MODE=1
13961402
endif

‎config.mak.uname‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,7 @@ ifeq ($(uname_O),Cygwin)
184184
UNRELIABLE_FSTAT = UnfortunatelyYes
185185
SPARSE_FLAGS = -isystem /usr/include/w32api -Wno-one-bit-signed-bitfield
186186
OBJECT_CREATION_USES_RENAMES = UnfortunatelyNeedsTo
187+
MMAP_PREVENTS_DELETE = UnfortunatelyYes
187188
COMPAT_OBJS += compat/cygwin.o
188189
FREAD_READS_DIRECTORIES = UnfortunatelyYes
189190
endif
@@ -353,6 +354,7 @@ ifeq ($(uname_S),Windows)
353354
NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease
354355
NO_NSEC = YesPlease
355356
USE_WIN32_MMAP = YesPlease
357+
MMAP_PREVENTS_DELETE = UnfortunatelyYes
356358
# USE_NED_ALLOCATOR = YesPlease
357359
UNRELIABLE_FSTAT = UnfortunatelyYes
358360
OBJECT_CREATION_USES_RENAMES = UnfortunatelyNeedsTo
@@ -501,6 +503,7 @@ ifneq (,$(findstring MINGW,$(uname_S)))
501503
NO_ST_BLOCKS_IN_STRUCT_STAT = YesPlease
502504
NO_NSEC = YesPlease
503505
USE_WIN32_MMAP = YesPlease
506+
MMAP_PREVENTS_DELETE = UnfortunatelyYes
504507
USE_NED_ALLOCATOR = YesPlease
505508
UNRELIABLE_FSTAT = UnfortunatelyYes
506509
OBJECT_CREATION_USES_RENAMES = UnfortunatelyNeedsTo

‎refs.c‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1285,6 +1285,10 @@ struct ref_iterator *refs_ref_iterator_begin(
12851285
if (trim)
12861286
iter = prefix_ref_iterator_begin(iter, "", trim);
12871287

1288+
/* Sanity check for subclasses: */
1289+
if (!iter->ordered)
1290+
BUG("reference iterator is not ordered");
1291+
12881292
return iter;
12891293
}
12901294

@@ -1686,7 +1690,23 @@ int refs_pack_refs(struct ref_store *refs, unsigned int flags)
16861690
int refs_peel_ref(struct ref_store *refs, const char *refname,
16871691
unsigned char *sha1)
16881692
{
1689-
return refs->be->peel_ref(refs, refname, sha1);
1693+
int flag;
1694+
unsigned char base[20];
1695+
1696+
if (current_ref_iter && current_ref_iter->refname == refname) {
1697+
struct object_id peeled;
1698+
1699+
if (ref_iterator_peel(current_ref_iter, &peeled))
1700+
return -1;
1701+
hashcpy(sha1, peeled.hash);
1702+
return 0;
1703+
}
1704+
1705+
if (refs_read_ref_full(refs, refname,
1706+
RESOLVE_REF_READING, base, &flag))
1707+
return -1;
1708+
1709+
return peel_object(base, sha1);
16901710
}
16911711

16921712
int peel_ref(const char *refname, unsigned char *sha1)

‎refs/files-backend.c‎

Lines changed: 10 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -641,43 +641,6 @@ static int lock_raw_ref(struct files_ref_store *refs,
641641
return ret;
642642
}
643643

644-
static int files_peel_ref(struct ref_store *ref_store,
645-
const char *refname, unsigned char *sha1)
646-
{
647-
struct files_ref_store *refs =
648-
files_downcast(ref_store, REF_STORE_READ | REF_STORE_ODB,
649-
"peel_ref");
650-
int flag;
651-
unsigned char base[20];
652-
653-
if (current_ref_iter && current_ref_iter->refname == refname) {
654-
struct object_id peeled;
655-
656-
if (ref_iterator_peel(current_ref_iter, &peeled))
657-
return -1;
658-
hashcpy(sha1, peeled.hash);
659-
return 0;
660-
}
661-
662-
if (refs_read_ref_full(ref_store, refname,
663-
RESOLVE_REF_READING, base, &flag))
664-
return -1;
665-
666-
/*
667-
* If the reference is packed, read its ref_entry from the
668-
* cache in the hope that we already know its peeled value.
669-
* We only try this optimization on packed references because
670-
* (a) forcing the filling of the loose reference cache could
671-
* be expensive and (b) loose references anyway usually do not
672-
* have REF_KNOWS_PEELED.
673-
*/
674-
if (flag & REF_ISPACKED &&
675-
!refs_peel_ref(refs->packed_ref_store, refname, sha1))
676-
return 0;
677-
678-
return peel_object(base, sha1);
679-
}
680-
681644
struct files_ref_iterator {
682645
struct ref_iterator base;
683646

@@ -748,7 +711,7 @@ static struct ref_iterator *files_ref_iterator_begin(
748711
const char *prefix, unsigned int flags)
749712
{
750713
struct files_ref_store *refs;
751-
struct ref_iterator *loose_iter, *packed_iter;
714+
struct ref_iterator *loose_iter, *packed_iter, *overlay_iter;
752715
struct files_ref_iterator *iter;
753716
struct ref_iterator *ref_iterator;
754717
unsigned int required_flags = REF_STORE_READ;
@@ -758,10 +721,6 @@ static struct ref_iterator *files_ref_iterator_begin(
758721

759722
refs = files_downcast(ref_store, required_flags, "ref_iterator_begin");
760723

761-
iter = xcalloc(1, sizeof(*iter));
762-
ref_iterator = &iter->base;
763-
base_ref_iterator_init(ref_iterator, &files_ref_iterator_vtable);
764-
765724
/*
766725
* We must make sure that all loose refs are read before
767726
* accessing the packed-refs file; this avoids a race
@@ -797,7 +756,13 @@ static struct ref_iterator *files_ref_iterator_begin(
797756
refs->packed_ref_store, prefix, 0,
798757
DO_FOR_EACH_INCLUDE_BROKEN);
799758

800-
iter->iter0 = overlay_ref_iterator_begin(loose_iter, packed_iter);
759+
overlay_iter = overlay_ref_iterator_begin(loose_iter, packed_iter);
760+
761+
iter = xcalloc(1, sizeof(*iter));
762+
ref_iterator = &iter->base;
763+
base_ref_iterator_init(ref_iterator, &files_ref_iterator_vtable,
764+
overlay_iter->ordered);
765+
iter->iter0 = overlay_iter;
801766
iter->flags = flags;
802767

803768
return ref_iterator;
@@ -2094,7 +2059,7 @@ static struct ref_iterator *reflog_iterator_begin(struct ref_store *ref_store,
20942059
struct ref_iterator *ref_iterator = &iter->base;
20952060
struct strbuf sb = STRBUF_INIT;
20962061

2097-
base_ref_iterator_init(ref_iterator, &files_reflog_iterator_vtable);
2062+
base_ref_iterator_init(ref_iterator, &files_reflog_iterator_vtable, 0);
20982063
strbuf_addf(&sb, "%s/logs", gitdir);
20992064
iter->dir_iterator = dir_iterator_begin(sb.buf);
21002065
iter->ref_store = ref_store;
@@ -2138,6 +2103,7 @@ static struct ref_iterator *files_reflog_iterator_begin(struct ref_store *ref_st
21382103
return reflog_iterator_begin(ref_store, refs->gitcommondir);
21392104
} else {
21402105
return merge_ref_iterator_begin(
2106+
0,
21412107
reflog_iterator_begin(ref_store, refs->gitdir),
21422108
reflog_iterator_begin(ref_store, refs->gitcommondir),
21432109
reflog_iterator_select, refs);
@@ -3089,7 +3055,6 @@ struct ref_storage_be refs_be_files = {
30893055
files_initial_transaction_commit,
30903056

30913057
files_pack_refs,
3092-
files_peel_ref,
30933058
files_create_symref,
30943059
files_delete_refs,
30953060
files_rename_ref,

‎refs/iterator.c‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,11 @@ int ref_iterator_abort(struct ref_iterator *ref_iterator)
2525
}
2626

2727
void base_ref_iterator_init(struct ref_iterator *iter,
28-
struct ref_iterator_vtable *vtable)
28+
struct ref_iterator_vtable *vtable,
29+
int ordered)
2930
{
3031
iter->vtable = vtable;
32+
iter->ordered = !!ordered;
3133
iter->refname = NULL;
3234
iter->oid = NULL;
3335
iter->flags = 0;
@@ -72,7 +74,7 @@ struct ref_iterator *empty_ref_iterator_begin(void)
7274
struct empty_ref_iterator *iter = xcalloc(1, sizeof(*iter));
7375
struct ref_iterator *ref_iterator = &iter->base;
7476

75-
base_ref_iterator_init(ref_iterator, &empty_ref_iterator_vtable);
77+
base_ref_iterator_init(ref_iterator, &empty_ref_iterator_vtable, 1);
7678
return ref_iterator;
7779
}
7880

@@ -205,6 +207,7 @@ static struct ref_iterator_vtable merge_ref_iterator_vtable = {
205207
};
206208

207209
struct ref_iterator *merge_ref_iterator_begin(
210+
int ordered,
208211
struct ref_iterator *iter0, struct ref_iterator *iter1,
209212
ref_iterator_select_fn *select, void *cb_data)
210213
{
@@ -219,7 +222,7 @@ struct ref_iterator *merge_ref_iterator_begin(
219222
* references through only if they exist in both iterators.
220223
*/
221224

222-
base_ref_iterator_init(ref_iterator, &merge_ref_iterator_vtable);
225+
base_ref_iterator_init(ref_iterator, &merge_ref_iterator_vtable, ordered);
223226
iter->iter0 = iter0;
224227
iter->iter1 = iter1;
225228
iter->select = select;
@@ -268,9 +271,11 @@ struct ref_iterator *overlay_ref_iterator_begin(
268271
} else if (is_empty_ref_iterator(back)) {
269272
ref_iterator_abort(back);
270273
return front;
274+
} else if (!front->ordered || !back->ordered) {
275+
BUG("overlay_ref_iterator requires ordered inputs");
271276
}
272277

273-
return merge_ref_iterator_begin(front, back,
278+
return merge_ref_iterator_begin(1, front, back,
274279
overlay_iterator_select, NULL);
275280
}
276281

@@ -282,16 +287,46 @@ struct prefix_ref_iterator {
282287
int trim;
283288
};
284289

290+
/* Return -1, 0, 1 if refname is before, inside, or after the prefix. */
291+
static int compare_prefix(const char *refname, const char *prefix)
292+
{
293+
while (*prefix) {
294+
if (*refname != *prefix)
295+
return ((unsigned char)*refname < (unsigned char)*prefix) ? -1 : +1;
296+
297+
refname++;
298+
prefix++;
299+
}
300+
301+
return 0;
302+
}
303+
285304
static int prefix_ref_iterator_advance(struct ref_iterator *ref_iterator)
286305
{
287306
struct prefix_ref_iterator *iter =
288307
(struct prefix_ref_iterator *)ref_iterator;
289308
int ok;
290309

291310
while ((ok = ref_iterator_advance(iter->iter0)) == ITER_OK) {
292-
if (!starts_with(iter->iter0->refname, iter->prefix))
311+
int cmp = compare_prefix(iter->iter0->refname, iter->prefix);
312+
313+
if (cmp < 0)
293314
continue;
294315

316+
if (cmp > 0) {
317+
/*
318+
* If the source iterator is ordered, then we
319+
* can stop the iteration as soon as we see a
320+
* refname that comes after the prefix:
321+
*/
322+
if (iter->iter0->ordered) {
323+
ok = ref_iterator_abort(iter->iter0);
324+
break;
325+
} else {
326+
continue;
327+
}
328+
}
329+
295330
if (iter->trim) {
296331
/*
297332
* It is nonsense to trim off characters that
@@ -361,7 +396,7 @@ struct ref_iterator *prefix_ref_iterator_begin(struct ref_iterator *iter0,
361396
iter = xcalloc(1, sizeof(*iter));
362397
ref_iterator = &iter->base;
363398

364-
base_ref_iterator_init(ref_iterator, &prefix_ref_iterator_vtable);
399+
base_ref_iterator_init(ref_iterator, &prefix_ref_iterator_vtable, iter0->ordered);
365400

366401
iter->iter0 = iter0;
367402
iter->prefix = xstrdup(prefix);

0 commit comments

Comments
 (0)