Skip to content

Commit b0e6bfa

Browse files
lemirelemire
authored
Simpler iteration code (simdjson#190)
* Adding convenience method to simplify code. * Simplifying the iteration code.
1 parent b1e8990 commit b0e6bfa

3 files changed

Lines changed: 47 additions & 83 deletions

File tree

‎README.md‎

Lines changed: 10 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -378,46 +378,21 @@ void compute_dump(ParsedJson::iterator &pjh) {
378378
The following function will find all user.id integers:
379379

380380
```C
381-
void simdjson_traverse(std::vector<int64_t> &answer, ParsedJson::iterator &i) {
382-
switch (i.get_type()) {
383-
case '{':
384-
if (i.down()) {
385-
do {
386-
bool founduser = equals(i.get_string(), "user");
387-
i.next(); // move to value
388-
if (i.is_object()) {
389-
if (founduser && i.move_to_key("id")) {
381+
void simdjson_scan(std::vector<int64_t> &answer, ParsedJson::iterator &i) {
382+
while(i.move_forward()) {
383+
if(i.get_scope_type() == '{') {
384+
bool founduser = (i.get_string_length() == 4) && (memcmp(i.get_string(), "user", 4) == 0);
385+
i.move_to_value();
386+
if(founduser) {
387+
if(i.is_object() && i.move_to_key("id",2)) {
390388
if (i.is_integer()) {
391389
answer.push_back(i.get_integer());
392390
}
393391
i.up();
394392
}
395-
simdjson_traverse(answer, i);
396-
} else if (i.is_array()) {
397-
simdjson_traverse(answer, i);
398-
}
399-
} while (i.next());
400-
i.up();
401-
}
402-
break;
403-
case '[':
404-
if (i.down()) {
405-
do {
406-
if (i.is_object_or_array()) {
407-
simdjson_traverse(answer, i);
408-
}
409-
} while (i.next());
410-
i.up();
411-
}
412-
break;
413-
case 'l':
414-
case 'd':
415-
case 'n':
416-
case 't':
417-
case 'f':
418-
default:
419-
break;
420-
}
393+
}
394+
}
395+
}
421396
}
422397
```
423398

‎benchmark/distinctuseridcompetition.cpp‎

Lines changed: 14 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -30,54 +30,28 @@ void print_vec(const std::vector<int64_t> &v) {
3030
std::cout << std::endl;
3131
}
3232

33-
void simdjson_traverse(std::vector<int64_t> &answer, ParsedJson::iterator &i) {
34-
switch (i.get_type()) {
35-
case '{':
36-
if (i.down()) {
37-
do {
38-
bool founduser = (i.get_string_length() == 4) && (memcmp(i.get_string(), "user", 4) == 0);
39-
i.move_to_value(); // move to value
40-
if (i.is_object()) {
41-
if (founduser && i.move_to_key("id")) {
33+
void simdjson_scan(std::vector<int64_t> &answer, ParsedJson::iterator &i) {
34+
while(i.move_forward()) {
35+
if(i.get_scope_type() == '{') {
36+
bool founduser = (i.get_string_length() == 4) && (memcmp(i.get_string(), "user", 4) == 0);
37+
i.move_to_value();
38+
if(founduser) {
39+
if(i.is_object() && i.move_to_key("id",2)) {
4240
if (i.is_integer()) {
4341
answer.push_back(i.get_integer());
44-
}
42+
}
4543
i.up();
46-
}
47-
simdjson_traverse(answer, i);
48-
} else if (i.is_array()) {
49-
simdjson_traverse(answer, i);
50-
}
51-
} while (i.next());
52-
i.up();
53-
}
54-
break;
55-
case '[':
56-
if (i.down()) {
57-
do {
58-
if (i.is_object_or_array()) {
59-
simdjson_traverse(answer, i);
60-
}
61-
} while (i.next());
62-
i.up();
63-
}
64-
break;
65-
case 'l':
66-
case 'd':
67-
case 'n':
68-
case 't':
69-
case 'f':
70-
default:
71-
break;
72-
}
44+
}
45+
}
46+
}
47+
}
7348
}
7449

7550
__attribute__ ((noinline))
7651
std::vector<int64_t> simdjson_justdom(ParsedJson &pj) {
7752
std::vector<int64_t> answer;
7853
ParsedJson::iterator i(pj);
79-
80-
simdjson_traverse(answer, i);
54+
simdjson_scan(answer,i);
8155
remove_duplicates(answer);
8256
return answer;
8357
}
@@ -90,8 +64,7 @@ std::vector<int64_t> simdjson_computestats(const padded_string &p) {
9064
return answer;
9165
}
9266
ParsedJson::iterator i(pj);
93-
94-
simdjson_traverse(answer, i);
67+
simdjson_scan(answer,i);
9568
remove_duplicates(answer);
9669
return answer;
9770
}
@@ -338,7 +311,6 @@ int main(int argc, char *argv[]) {
338311
}
339312
BEST_TIME("simdjson ", simdjson_computestats(p).size(), size, , repeat,
340313
volume, !justdata);
341-
342314
BEST_TIME("rapid ", rapid_computestats(p).size(), size, , repeat, volume,
343315
!justdata);
344316
BEST_TIME("sasjon ", sasjon_computestats(p).size(), size, , repeat, volume,

‎include/simdjson/parsedjson.h‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -207,11 +207,17 @@ struct ParsedJson {
207207
// when at {, go one level deep, looking for a given key
208208
// if successful, we are left pointing at the value,
209209
// if not, we are still pointing at the object ({)
210-
// (in case of repeated keys, this only finds the first one)
210+
// (in case of repeated keys, this only finds the first one).
211211
// We seek the key using C's strcmp so if your JSON strings contain
212212
// NULL chars, this would trigger a false positive: if you expect that
213213
// to be the case, take extra precautions.
214214
inline bool move_to_key(const char * key);
215+
// when at {, go one level deep, looking for a given key
216+
// if successful, we are left pointing at the value,
217+
// if not, we are still pointing at the object ({)
218+
// (in case of repeated keys, this only finds the first one).
219+
// The string we search for can contain NULL values.
220+
inline bool move_to_key(const char * key, uint32_t length);
215221

216222
// when at a key location within an object, this moves to the accompanying value (located next to it).
217223
// this is equivalent but much faster than calling "next()".
@@ -355,10 +361,6 @@ bool ParsedJson::iterator::move_forward() {
355361
} else if ((current_type == ']') || (current_type == '}')) {
356362
// Leaving a scope.
357363
depth--;
358-
if(depth == 0) {
359-
// Should not be necessary
360-
return false;
361-
}
362364
} else if ((current_type == 'd') || (current_type == 'l')) {
363365
// d and l types use 2 locations on the tape, not just one.
364366
location += 1;
@@ -393,6 +395,21 @@ bool ParsedJson::iterator::move_to_key(const char * key) {
393395
return false;
394396
}
395397

398+
bool ParsedJson::iterator::move_to_key(const char * key, uint32_t length) {
399+
if(down()) {
400+
do {
401+
assert(is_string());
402+
bool rightkey = ((get_string_length() == length) && (memcmp(get_string(),key,length)==0));
403+
move_to_value();
404+
if(rightkey) {
405+
return true;
406+
}
407+
} while(next());
408+
assert(up());// not found
409+
}
410+
return false;
411+
}
412+
396413

397414
bool ParsedJson::iterator::prev() {
398415
if(location - 1 < depthindex[depth].start_of_scope) {
@@ -456,7 +473,7 @@ void ParsedJson::iterator::to_start_scope() {
456473
}
457474

458475
bool ParsedJson::iterator::next() {
459-
size_t npos; // next position
476+
size_t npos;
460477
if ((current_type == '[') || (current_type == '{')){
461478
// we need to jump
462479
npos = ( current_val & JSONVALUEMASK);

0 commit comments

Comments
 (0)