Skip to content

Commit 64872bd

Browse files
jkeiserjkeiser
committed
Eliminate stage1_find_marks_flatten.h
1 parent 81f2249 commit 64872bd

8 files changed

Lines changed: 64 additions & 108 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompeti
6464
SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing
6565

6666
# Load headers and sources
67-
LIBHEADERS=src/simdprune_tables.h src/numberparsing.h src/jsoncharutils.h src/arm64/simd_input.h src/arm64/simdutf8check.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h src/generic/stage1_find_marks_flatten.h src/generic/stage1_find_marks.h src/generic/stage2_build_tape.h src/generic/stringparsing.h src/haswell/simd_input.h src/haswell/simdutf8check.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h src/westmere/simd_input.h src/westmere/simdutf8check.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h
67+
LIBHEADERS=src/simdprune_tables.h src/numberparsing.h src/jsoncharutils.h src/arm64/simd_input.h src/arm64/simdutf8check.h src/arm64/stage1_find_marks.h src/arm64/stage2_build_tape.h src/arm64/stringparsing.h src/generic/stage1_find_marks.h src/generic/stage2_build_tape.h src/generic/stringparsing.h src/haswell/simd_input.h src/haswell/simdutf8check.h src/haswell/stage1_find_marks.h src/haswell/stage2_build_tape.h src/haswell/stringparsing.h src/westmere/simd_input.h src/westmere/simdutf8check.h src/westmere/stage1_find_marks.h src/westmere/stage2_build_tape.h src/westmere/stringparsing.h
6868
PUBHEADERS=include/simdjson/common_defs.h include/simdjson/isadetection.h include/simdjson/jsonformatutils.h include/simdjson/jsonioutil.h include/simdjson/jsonminifier.h include/simdjson/jsonparser.h include/simdjson/padded_string.h include/simdjson/parsedjson.h include/simdjson/parsedjsoniterator.h include/simdjson/portability.h include/simdjson/simdjson.h include/simdjson/simdjson_version.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_build_tape.h
6969
HEADERS=$(PUBHEADERS) $(LIBHEADERS)
7070

‎include/simdjson/portability.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ static inline int trailing_zeroes(uint64_t input_num) {
8484
return static_cast<int>(_tzcnt_u64(input_num));
8585
}
8686

87+
static inline uint64_t clear_lowest_bit(uint64_t input_num) {
88+
return _blsr_u64(input_num);
89+
}
90+
8791
static inline int leading_zeroes(uint64_t input_num) {
8892
return static_cast<int>(_lzcnt_u64(input_num));
8993
}
@@ -122,6 +126,15 @@ static inline NO_SANITIZE_UNDEFINED int trailing_zeroes(uint64_t input_num) {
122126
#endif
123127
}
124128

129+
/* result might be undefined when input_num is zero */
130+
static inline uint64_t clear_lowest_bit(uint64_t input_num) {
131+
#ifdef __BMI__ // blsr is BMI1
132+
return _blsr_u64(input_num);
133+
#else
134+
return input_num & (input_num-1);
135+
#endif
136+
}
137+
125138
/* result might be undefined when input_num is zero */
126139
static inline int leading_zeroes(uint64_t input_num) {
127140
#ifdef __BMI2__

‎src/CMakeLists.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ set(SIMDJSON_SRC_HEADERS
4242
arm64/stage1_find_marks.h
4343
arm64/stage2_build_tape.h
4444
arm64/stringparsing.h
45-
generic/stage1_find_marks_flatten.h
4645
generic/stage1_find_marks.h
4746
generic/stage2_build_tape.h
4847
generic/stringparsing.h

‎src/arm64/stage1_find_marks.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ really_inline void find_whitespace_and_operators(
4848
}).to_bitmask();
4949
}
5050

51-
#include "generic/stage1_find_marks_flatten.h"
5251
#include "generic/stage1_find_marks.h"
5352

5453
} // namespace simdjson::arm64

‎src/generic/stage1_find_marks.h‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,56 @@
55

66
static const size_t STEP_SIZE = 128;
77

8+
class bit_indexer {
9+
public:
10+
uint32_t *tail;
11+
12+
bit_indexer(uint32_t *index_buf) : tail(index_buf) {}
13+
14+
// flatten out values in 'bits' assuming that they are are to have values of idx
15+
// plus their position in the bitvector, and store these indexes at
16+
// base_ptr[base] incrementing base as we go
17+
// will potentially store extra values beyond end of valid bits, so base_ptr
18+
// needs to be large enough to handle this
19+
really_inline void write_indexes(uint32_t idx, uint64_t bits) {
20+
// In some instances, the next branch is expensive because it is mispredicted.
21+
// Unfortunately, in other cases,
22+
// it helps tremendously.
23+
if (bits == 0)
24+
return;
25+
uint32_t cnt = hamming(bits);
26+
27+
// Do the first 8 all together
28+
for (int i=0; i<8; i++) {
29+
this->tail[i] = idx + trailing_zeroes(bits);
30+
bits = clear_lowest_bit(bits);
31+
}
32+
33+
// Do the next 8 all together (we hope in most cases it won't happen at all
34+
// and the branch is easily predicted).
35+
if (unlikely(cnt > 8)) {
36+
for (int i=8; i<16; i++) {
37+
this->tail[i] = idx + trailing_zeroes(bits);
38+
bits = clear_lowest_bit(bits);
39+
}
40+
41+
// Most files don't have 16+ structurals per block, so we take several basically guaranteed
42+
// branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :)
43+
// or the start of a value ("abc" true 123) every four characters.
44+
if (unlikely(cnt > 16)) {
45+
uint32_t i = 16;
46+
do {
47+
this->tail[i] = idx + trailing_zeroes(bits);
48+
bits = clear_lowest_bit(bits);
49+
i++;
50+
} while (i < cnt);
51+
}
52+
}
53+
54+
this->tail += cnt;
55+
}
56+
};
57+
858
class json_structural_scanner {
959
public:
1060
// Whether the first character of the next iteration is escaped.

‎src/generic/stage1_find_marks_flatten.h‎

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎src/haswell/stage1_find_marks.h‎

Lines changed: 0 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -84,56 +84,6 @@ really_inline void find_whitespace_and_operators(
8484
#endif // else SIMDJSON_NAIVE_STRUCTURAL
8585
}
8686

87-
class bit_indexer {
88-
public:
89-
uint32_t *tail;
90-
91-
bit_indexer(uint32_t *index_buf) : tail(index_buf) {}
92-
93-
// flatten out values in 'bits' assuming that they are are to have values of idx
94-
// plus their position in the bitvector, and store these indexes at
95-
// base_ptr[base] incrementing base as we go
96-
// will potentially store extra values beyond end of valid bits, so base_ptr
97-
// needs to be large enough to handle this
98-
really_inline void write_indexes(uint32_t idx, uint64_t bits) {
99-
// In some instances, the next branch is expensive because it is mispredicted.
100-
// Unfortunately, in other cases,
101-
// it helps tremendously.
102-
if (bits == 0)
103-
return;
104-
uint32_t cnt = _mm_popcnt_u64(bits);
105-
106-
// Do the first 8 all together
107-
for (int i=0; i<8; i++) {
108-
this->tail[i] = idx + trailing_zeroes(bits);
109-
bits = _blsr_u64(bits);
110-
}
111-
112-
// Do the next 8 all together (we hope in most cases it won't happen at all
113-
// and the branch is easily predicted).
114-
if (unlikely(cnt > 8)) {
115-
for (int i=8; i<16; i++) {
116-
this->tail[i] = idx + trailing_zeroes(bits);
117-
bits = _blsr_u64(bits);
118-
}
119-
120-
// Most files don't have 16+ structurals per block, so we take several basically guaranteed
121-
// branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :)
122-
// or the start of a value ("abc" true 123) every four characters.
123-
if (unlikely(cnt > 16)) {
124-
uint32_t i = 16;
125-
do {
126-
this->tail[i] = idx + trailing_zeroes(bits);
127-
bits = _blsr_u64(bits);
128-
i++;
129-
} while (i < cnt);
130-
}
131-
}
132-
133-
this->tail += cnt;
134-
}
135-
};
136-
13787
#include "generic/stage1_find_marks.h"
13888

13989
} // namespace haswell

‎src/westmere/stage1_find_marks.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ really_inline void find_whitespace_and_operators(
4040
}).to_bitmask();
4141
}
4242

43-
#include "generic/stage1_find_marks_flatten.h"
4443
#include "generic/stage1_find_marks.h"
4544

4645
} // namespace westmere

0 commit comments

Comments
 (0)