Skip to content

Commit ed591fe

Browse files
garimasi514gitster
garimasi514
authored and
gitster
committed
bloom.c: core Bloom filter implementation for changed paths.
Add the core implementation for computing Bloom filters for the paths changed between a commit and it's first parent. We fill the Bloom filters as (const char *data, int len) pairs as `struct bloom_filters" within a commit slab. Filters for commits with no changes and more than 512 changes, is represented with a filter of length zero. There is no gain in distinguishing between a computed filter of length zero for a commit with no changes, and an uncomputed filter for new commits or for commits with more than 512 changes. The effect on `git log -- path` is the same in both cases. We will fall back to the normal diffing algorithm when we can't benefit from the existence of Bloom filters. Helped-by: Jeff King <peff@peff.net> Helped-by: Derrick Stolee <dstolee@microsoft.com> Reviewed-by: Jakub Narębski <jnareb@gmail.com> Signed-off-by: Garima Singh <garima.singh@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
1 parent f1294ea commit ed591fe

File tree

4 files changed

+172
-0
lines changed

4 files changed

+172
-0
lines changed

‎bloom.c‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
#include "git-compat-util.h"
22
#include "bloom.h"
3+
#include "diff.h"
4+
#include "diffcore.h"
5+
#include "revision.h"
6+
#include "hashmap.h"
7+
8+
define_commit_slab(bloom_filter_slab, struct bloom_filter);
9+
10+
struct bloom_filter_slab bloom_filters;
11+
12+
struct pathmap_hash_entry {
13+
struct hashmap_entry entry;
14+
const char path[FLEX_ARRAY];
15+
};
316

417
static uint32_t rotate_left(uint32_t value, int32_t count)
518
{
@@ -107,3 +120,87 @@ void add_key_to_filter(const struct bloom_key *key,
107120
filter->data[block_pos] |= get_bitmask(hash_mod);
108121
}
109122
}
123+
124+
void init_bloom_filters(void)
125+
{
126+
init_bloom_filter_slab(&bloom_filters);
127+
}
128+
129+
struct bloom_filter *get_bloom_filter(struct repository *r,
130+
struct commit *c)
131+
{
132+
struct bloom_filter *filter;
133+
struct bloom_filter_settings settings = DEFAULT_BLOOM_FILTER_SETTINGS;
134+
int i;
135+
struct diff_options diffopt;
136+
137+
if (bloom_filters.slab_size == 0)
138+
return NULL;
139+
140+
filter = bloom_filter_slab_at(&bloom_filters, c);
141+
142+
repo_diff_setup(r, &diffopt);
143+
diffopt.flags.recursive = 1;
144+
diff_setup_done(&diffopt);
145+
146+
if (c->parents)
147+
diff_tree_oid(&c->parents->item->object.oid, &c->object.oid, "", &diffopt);
148+
else
149+
diff_tree_oid(NULL, &c->object.oid, "", &diffopt);
150+
diffcore_std(&diffopt);
151+
152+
if (diff_queued_diff.nr <= 512) {
153+
struct hashmap pathmap;
154+
struct pathmap_hash_entry *e;
155+
struct hashmap_iter iter;
156+
hashmap_init(&pathmap, NULL, NULL, 0);
157+
158+
for (i = 0; i < diff_queued_diff.nr; i++) {
159+
const char *path = diff_queued_diff.queue[i]->two->path;
160+
161+
/*
162+
* Add each leading directory of the changed file, i.e. for
163+
* 'dir/subdir/file' add 'dir' and 'dir/subdir' as well, so
164+
* the Bloom filter could be used to speed up commands like
165+
* 'git log dir/subdir', too.
166+
*
167+
* Note that directories are added without the trailing '/'.
168+
*/
169+
do {
170+
char *last_slash = strrchr(path, '/');
171+
172+
FLEX_ALLOC_STR(e, path, path);
173+
hashmap_entry_init(&e->entry, strhash(path));
174+
hashmap_add(&pathmap, &e->entry);
175+
176+
if (!last_slash)
177+
last_slash = (char*)path;
178+
*last_slash = '\0';
179+
180+
} while (*path);
181+
182+
diff_free_filepair(diff_queued_diff.queue[i]);
183+
}
184+
185+
filter->len = (hashmap_get_size(&pathmap) * settings.bits_per_entry + BITS_PER_WORD - 1) / BITS_PER_WORD;
186+
filter->data = xcalloc(filter->len, sizeof(unsigned char));
187+
188+
hashmap_for_each_entry(&pathmap, &iter, e, entry) {
189+
struct bloom_key key;
190+
fill_bloom_key(e->path, strlen(e->path), &key, &settings);
191+
add_key_to_filter(&key, filter, &settings);
192+
}
193+
194+
hashmap_free_entries(&pathmap, struct pathmap_hash_entry, entry);
195+
} else {
196+
for (i = 0; i < diff_queued_diff.nr; i++)
197+
diff_free_filepair(diff_queued_diff.queue[i]);
198+
filter->data = NULL;
199+
filter->len = 0;
200+
}
201+
202+
free(diff_queued_diff.queue);
203+
DIFF_QUEUE_CLEAR(&diff_queued_diff);
204+
205+
return filter;
206+
}

‎bloom.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
#ifndef BLOOM_H
22
#define BLOOM_H
33

4+
struct commit;
5+
struct repository;
6+
47
struct bloom_filter_settings {
58
/*
69
* The version of the hashing technique being used.
@@ -73,4 +76,9 @@ void add_key_to_filter(const struct bloom_key *key,
7376
struct bloom_filter *filter,
7477
const struct bloom_filter_settings *settings);
7578

79+
void init_bloom_filters(void);
80+
81+
struct bloom_filter *get_bloom_filter(struct repository *r,
82+
struct commit *c);
83+
7684
#endif

‎t/helper/test-bloom.c‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#include "git-compat-util.h"
22
#include "bloom.h"
33
#include "test-tool.h"
4+
#include "commit.h"
45

56
struct bloom_filter_settings settings = DEFAULT_BLOOM_FILTER_SETTINGS;
67

@@ -32,6 +33,16 @@ static void print_bloom_filter(struct bloom_filter *filter) {
3233
printf("\n");
3334
}
3435

36+
static void get_bloom_filter_for_commit(const struct object_id *commit_oid)
37+
{
38+
struct commit *c;
39+
struct bloom_filter *filter;
40+
setup_git_directory();
41+
c = lookup_commit(the_repository, commit_oid);
42+
filter = get_bloom_filter(the_repository, c);
43+
print_bloom_filter(filter);
44+
}
45+
3546
int cmd__bloom(int argc, const char **argv)
3647
{
3748
if (!strcmp(argv[1], "get_murmur3")) {
@@ -57,5 +68,14 @@ int cmd__bloom(int argc, const char **argv)
5768
print_bloom_filter(&filter);
5869
}
5970

71+
if (!strcmp(argv[1], "get_filter_for_commit")) {
72+
struct object_id oid;
73+
const char *end;
74+
if (parse_oid_hex(argv[2], &oid, &end))
75+
die("cannot parse oid '%s'", argv[2]);
76+
init_bloom_filters();
77+
get_bloom_filter_for_commit(&oid);
78+
}
79+
6080
return 0;
6181
}

‎t/t0095-bloom.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,51 @@ test_expect_success 'compute bloom key for test string 2' '
6767
test_cmp expect actual
6868
'
6969

70+
test_expect_success 'get bloom filters for commit with no changes' '
71+
git init &&
72+
git commit --allow-empty -m "c0" &&
73+
cat >expect <<-\EOF &&
74+
Filter_Length:0
75+
Filter_Data:
76+
EOF
77+
test-tool bloom get_filter_for_commit "$(git rev-parse HEAD)" >actual &&
78+
test_cmp expect actual
79+
'
80+
81+
test_expect_success 'get bloom filter for commit with 10 changes' '
82+
rm actual &&
83+
rm expect &&
84+
mkdir smallDir &&
85+
for i in $(test_seq 0 9)
86+
do
87+
echo $i >smallDir/$i
88+
done &&
89+
git add smallDir &&
90+
git commit -m "commit with 10 changes" &&
91+
cat >expect <<-\EOF &&
92+
Filter_Length:25
93+
Filter_Data:82|a0|65|47|0c|92|90|c0|a1|40|02|a0|e2|40|e0|04|0a|9a|66|cf|80|19|85|42|23|
94+
EOF
95+
test-tool bloom get_filter_for_commit "$(git rev-parse HEAD)" >actual &&
96+
test_cmp expect actual
97+
'
98+
99+
test_expect_success EXPENSIVE 'get bloom filter for commit with 513 changes' '
100+
rm actual &&
101+
rm expect &&
102+
mkdir bigDir &&
103+
for i in $(test_seq 0 512)
104+
do
105+
echo $i >bigDir/$i
106+
done &&
107+
git add bigDir &&
108+
git commit -m "commit with 513 changes" &&
109+
cat >expect <<-\EOF &&
110+
Filter_Length:0
111+
Filter_Data:
112+
EOF
113+
test-tool bloom get_filter_for_commit "$(git rev-parse HEAD)" >actual &&
114+
test_cmp expect actual
115+
'
116+
70117
test_done

0 commit comments

Comments
 (0)