Skip to content

Commit 96c47d1

Browse files
jamillgitster
jamill
authored and
gitster
committed
fast-import: introduce mem_pool type
Introduce the mem_pool type which encapsulates all the information necessary to manage a pool of memory. This change moves the existing variables in fast-import used to support the global memory pool to use this structure. It also renames variables that are no longer used by memory pools to reflect their more scoped usage. These changes allow for the multiple instances of a memory pool to exist and be reused outside of fast-import. In a future commit the mem_pool type will be moved to its own file. Signed-off-by: Jameson Miller <jamill@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
1 parent 7dc0656 commit 96c47d1

File tree

1 file changed

+52
-29
lines changed

1 file changed

+52
-29
lines changed

‎fast-import.c‎

Lines changed: 52 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,19 @@ struct mp_block {
216216
uintmax_t space[FLEX_ARRAY]; /* more */
217217
};
218218

219+
struct mem_pool {
220+
struct mp_block *mp_block;
221+
222+
/*
223+
* The amount of available memory to grow the pool by.
224+
* This size does not include the overhead for the mp_block.
225+
*/
226+
size_t block_alloc;
227+
228+
/* The total amount of memory allocated by the pool. */
229+
size_t pool_alloc;
230+
};
231+
219232
struct atom_str {
220233
struct atom_str *next_atom;
221234
unsigned short str_len;
@@ -304,9 +317,8 @@ static int global_argc;
304317
static const char **global_argv;
305318

306319
/* Memory pools */
307-
static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mp_block);
308-
static size_t total_allocd;
309-
static struct mp_block *mp_block_head;
320+
static struct mem_pool fi_mem_pool = {NULL, 2*1024*1024 -
321+
sizeof(struct mp_block), 0 };
310322

311323
/* Atom management */
312324
static unsigned int atom_table_sz = 4451;
@@ -341,6 +353,7 @@ static unsigned int tree_entry_alloc = 1000;
341353
static void *avail_tree_entry;
342354
static unsigned int avail_tree_table_sz = 100;
343355
static struct avail_tree_content **avail_tree_table;
356+
static size_t tree_entry_allocd;
344357
static struct strbuf old_tree = STRBUF_INIT;
345358
static struct strbuf new_tree = STRBUF_INIT;
346359

@@ -634,7 +647,21 @@ static unsigned int hc_str(const char *s, size_t len)
634647
return r;
635648
}
636649

637-
static void *pool_alloc(size_t len)
650+
static struct mp_block *mem_pool_alloc_block(struct mem_pool *mem_pool, size_t block_alloc)
651+
{
652+
struct mp_block *p;
653+
654+
mem_pool->pool_alloc += sizeof(struct mp_block) + block_alloc;
655+
p = xmalloc(st_add(sizeof(struct mp_block), block_alloc));
656+
p->next_block = mem_pool->mp_block;
657+
p->next_free = (char *)p->space;
658+
p->end = p->next_free + block_alloc;
659+
mem_pool->mp_block = p;
660+
661+
return p;
662+
}
663+
664+
static void *mem_pool_alloc(struct mem_pool *mem_pool, size_t len)
638665
{
639666
struct mp_block *p;
640667
void *r;
@@ -643,40 +670,36 @@ static void *pool_alloc(size_t len)
643670
if (len & (sizeof(uintmax_t) - 1))
644671
len += sizeof(uintmax_t) - (len & (sizeof(uintmax_t) - 1));
645672

646-
for (p = mp_block_head; p; p = p->next_block)
647-
if ((p->end - p->next_free >= len))
673+
for (p = mem_pool->mp_block; p; p = p->next_block)
674+
if (p->end - p->next_free >= len)
648675
break;
649676

650677
if (!p) {
651-
if (len >= (mem_pool_alloc/2)) {
652-
total_allocd += len;
678+
if (len >= (mem_pool->block_alloc / 2)) {
679+
mem_pool->pool_alloc += len;
653680
return xmalloc(len);
654681
}
655-
total_allocd += sizeof(struct mp_block) + mem_pool_alloc;
656-
p = xmalloc(st_add(sizeof(struct mp_block), mem_pool_alloc));
657-
p->next_block = mp_block_head;
658-
p->next_free = (char *) p->space;
659-
p->end = p->next_free + mem_pool_alloc;
660-
mp_block_head = p;
682+
683+
p = mem_pool_alloc_block(mem_pool, mem_pool->block_alloc);
661684
}
662685

663686
r = p->next_free;
664687
p->next_free += len;
665688
return r;
666689
}
667690

668-
static void *pool_calloc(size_t count, size_t size)
691+
static void *mem_pool_calloc(struct mem_pool *mem_pool, size_t count, size_t size)
669692
{
670-
size_t len = count * size;
671-
void *r = pool_alloc(len);
693+
size_t len = st_mult(count, size);
694+
void *r = mem_pool_alloc(mem_pool, len);
672695
memset(r, 0, len);
673696
return r;
674697
}
675698

676699
static char *pool_strdup(const char *s)
677700
{
678701
size_t len = strlen(s) + 1;
679-
char *r = pool_alloc(len);
702+
char *r = mem_pool_alloc(&fi_mem_pool, len);
680703
memcpy(r, s, len);
681704
return r;
682705
}
@@ -685,7 +708,7 @@ static void insert_mark(uintmax_t idnum, struct object_entry *oe)
685708
{
686709
struct mark_set *s = marks;
687710
while ((idnum >> s->shift) >= 1024) {
688-
s = pool_calloc(1, sizeof(struct mark_set));
711+
s = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
689712
s->shift = marks->shift + 10;
690713
s->data.sets[0] = marks;
691714
marks = s;
@@ -694,7 +717,7 @@ static void insert_mark(uintmax_t idnum, struct object_entry *oe)
694717
uintmax_t i = idnum >> s->shift;
695718
idnum -= i << s->shift;
696719
if (!s->data.sets[i]) {
697-
s->data.sets[i] = pool_calloc(1, sizeof(struct mark_set));
720+
s->data.sets[i] = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
698721
s->data.sets[i]->shift = s->shift - 10;
699722
}
700723
s = s->data.sets[i];
@@ -732,7 +755,7 @@ static struct atom_str *to_atom(const char *s, unsigned short len)
732755
if (c->str_len == len && !strncmp(s, c->str_dat, len))
733756
return c;
734757

735-
c = pool_alloc(sizeof(struct atom_str) + len + 1);
758+
c = mem_pool_alloc(&fi_mem_pool, sizeof(struct atom_str) + len + 1);
736759
c->str_len = len;
737760
memcpy(c->str_dat, s, len);
738761
c->str_dat[len] = 0;
@@ -763,7 +786,7 @@ static struct branch *new_branch(const char *name)
763786
if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL))
764787
die("Branch name doesn't conform to GIT standards: %s", name);
765788

766-
b = pool_calloc(1, sizeof(struct branch));
789+
b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch));
767790
b->name = pool_strdup(name);
768791
b->table_next_branch = branch_table[hc];
769792
b->branch_tree.versions[0].mode = S_IFDIR;
@@ -799,7 +822,7 @@ static struct tree_content *new_tree_content(unsigned int cnt)
799822
avail_tree_table[hc] = f->next_avail;
800823
} else {
801824
cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
802-
f = pool_alloc(sizeof(*t) + sizeof(t->entries[0]) * cnt);
825+
f = mem_pool_alloc(&fi_mem_pool, sizeof(*t) + sizeof(t->entries[0]) * cnt);
803826
f->entry_capacity = cnt;
804827
}
805828

@@ -844,7 +867,7 @@ static struct tree_entry *new_tree_entry(void)
844867

845868
if (!avail_tree_entry) {
846869
unsigned int n = tree_entry_alloc;
847-
total_allocd += n * sizeof(struct tree_entry);
870+
tree_entry_allocd += n * sizeof(struct tree_entry);
848871
ALLOC_ARRAY(e, n);
849872
avail_tree_entry = e;
850873
while (n-- > 1) {
@@ -2862,7 +2885,7 @@ static void parse_new_tag(const char *arg)
28622885
enum object_type type;
28632886
const char *v;
28642887

2865-
t = pool_alloc(sizeof(struct tag));
2888+
t = mem_pool_alloc(&fi_mem_pool, sizeof(struct tag));
28662889
memset(t, 0, sizeof(struct tag));
28672890
t->name = pool_strdup(arg);
28682891
if (last_tag)
@@ -3461,12 +3484,12 @@ int cmd_main(int argc, const char **argv)
34613484
atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));
34623485
branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));
34633486
avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
3464-
marks = pool_calloc(1, sizeof(struct mark_set));
3487+
marks = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));
34653488

34663489
global_argc = argc;
34673490
global_argv = argv;
34683491

3469-
rc_free = pool_alloc(cmd_save * sizeof(*rc_free));
3492+
rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free));
34703493
for (i = 0; i < (cmd_save - 1); i++)
34713494
rc_free[i].next = &rc_free[i + 1];
34723495
rc_free[cmd_save - 1].next = NULL;
@@ -3541,8 +3564,8 @@ int cmd_main(int argc, const char **argv)
35413564
fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
35423565
fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
35433566
fprintf(stderr, " atoms: %10u\n", atom_cnt);
3544-
fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
3545-
fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)(total_allocd/1024));
3567+
fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (tree_entry_allocd + fi_mem_pool.pool_alloc + alloc_count*sizeof(struct object_entry))/1024);
3568+
fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)((tree_entry_allocd + fi_mem_pool.pool_alloc) /1024));
35463569
fprintf(stderr, " objects: %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
35473570
fprintf(stderr, "---------------------------------------------------------------------\n");
35483571
pack_report();

0 commit comments

Comments
 (0)