Repository navigation
Expand file tree
/
Copy pathBloomFilter.cpp
More file actions
151 lines (116 loc) · 3.6 KB
/
Copy pathBloomFilter.cpp
File metadata and controls
151 lines (116 loc) · 3.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
#include "BloomFilter.h"
#include <cmath>
using namespace std;
BloomFilter::BloomFilter(size_t elementCount, double fpr){
size_t bits = BloomFilter::computeTheoryBits(elementCount,fpr);
size_t hashes = BloomFilter::optimalHashes(bits,elementCount);
bitSize = bits;
hashCount = hashes;
size_t words = (bits+63)/64;
filterArray.resize(words,0);
}
size_t BloomFilter::computeTheoryBits(size_t elementCount, double fpr){
const double ln2 = log(2.0);
double bits = -(elementCount * log(fpr)) / (ln2 * ln2);
return static_cast<size_t>(ceil(bits));
}
size_t BloomFilter::optimalHashes(size_t bits, size_t elements){
return round((bits / (double)elements) * log(2));
}
uint64_t BloomFilter::rotl(uint64_t x, int r){
return (x << r) | (x >> (64 - r));
}
uint64_t BloomFilter::xxhash64(const std::string& s, uint64_t seed = 0){
const uint64_t PRIME1 = 11400714785074694791ULL;
const uint64_t PRIME2 = 14029467366897019727ULL;
const uint64_t PRIME3 = 1609587929392839161ULL;
const uint64_t PRIME4 = 9650029242287828579ULL;
const uint64_t PRIME5 = 2870177450012600261ULL;
const uint8_t* p = (const uint8_t*)s.data();
const uint8_t* end = p + s.size();
uint64_t h;
if (s.size() >= 32) {
uint64_t v1 = seed + PRIME1 + PRIME2;
uint64_t v2 = seed + PRIME2;
uint64_t v3 = seed + 0;
uint64_t v4 = seed - PRIME1;
const uint8_t* limit = end - 32;
while (p <= limit) {
uint64_t k1 = *(uint64_t*)p; p += 8;
v1 += k1 * PRIME2;
v1 = rotl(v1,31);
v1 *= PRIME1;
uint64_t k2 = *(uint64_t*)p; p += 8;
v2 += k2 * PRIME2;
v2 = rotl(v2,31);
v2 *= PRIME1;
uint64_t k3 = *(uint64_t*)p; p += 8;
v3 += k3 * PRIME2;
v3 = rotl(v3,31);
v3 *= PRIME1;
uint64_t k4 = *(uint64_t*)p; p += 8;
v4 += k4 * PRIME2;
v4 = rotl(v4,31);
v4 *= PRIME1;
}
h = rotl(v1,1) + rotl(v2,7) + rotl(v3,12) + rotl(v4,18);
} else {
h = seed + PRIME5;
}
h += s.size();
while (p + 8 <= end) {
uint64_t k1 = *(uint64_t*)p;
k1 *= PRIME2;
k1 = rotl(k1,31);
k1 *= PRIME1;
h ^= k1;
h = rotl(h,27) * PRIME1 + PRIME4;
p += 8;
}
if (p + 4 <= end) {
h ^= (*(uint32_t*)p) * PRIME1;
h = rotl(h,23) * PRIME2 + PRIME3;
p += 4;
}
while (p < end) {
h ^= (*p) * PRIME5;
h = rotl(h,11) * PRIME1;
p++;
}
h ^= h >> 33;
h *= PRIME2;
h ^= h >> 29;
h *= PRIME3;
h ^= h >> 32;
return h;
}
void BloomFilter::setBit(size_t index){
size_t indInArray = index >> 6;
size_t offset = index & 63;
filterArray[indInArray] |= (1ULL << offset);
}
bool BloomFilter::getBit(size_t index) const{
size_t indInArray = index >> 6;
size_t offset = index & 63;
return (filterArray[indInArray] >> offset) & 1;
}
void BloomFilter::AddToFilter(const string& data){
uint64_t h1 = xxhash64(data, 0);
uint64_t h2 = xxhash64(data, 1);
for(size_t i=0;i<hashCount;i++){
size_t idx = (h1 + i*h2) % bitSize;
setBit(idx);
}
}
bool BloomFilter::IsInFilter(const string& data){
uint64_t h1 = xxhash64(data, 0);
uint64_t h2 = xxhash64(data, 1);
for(size_t i=0;i<hashCount;i++){
size_t idx = (h1 + i*h2) % bitSize;
if(!getBit(idx)) return false;
}
return true;
}
size_t BloomFilter::getMemoryCost() const{
return filterArray.capacity()*sizeof(uint64_t);
}