Skip to content

Commit ce824f8

Browse files
jkeiserjkeiser
committed
Decrease stage 1 step size to 64 bytes on Westmere/ARM
- Templatize scan_step() with STAGE1_STEP_SIZE - Fix simd8::store() - add NUM_CHUNKS to simd8
1 parent 708f4a0 commit ce824f8

8 files changed

Lines changed: 98 additions & 80 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
/benchmark/statisticalmodel
1313
/json2json
1414
/jsoncheck
15+
/jsoncheck_noavx
16+
/jsonstream_test
1517
/jsonpointer
1618
/jsonstats
1719
/integer_tests

‎src/arm64/simd.h‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -230,17 +230,26 @@ namespace simdjson::arm64::simd {
230230

231231
template<typename T>
232232
struct simd8x64 {
233-
const simd8<T> chunks[4];
233+
static const int NUM_CHUNKS = 64 / sizeof(simd8<T>);
234+
const simd8<T> chunks[NUM_CHUNKS];
234235

235236
really_inline simd8x64() : chunks{simd8<T>(), simd8<T>(), simd8<T>(), simd8<T>()} {}
236237
really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1, const simd8<T> chunk2, const simd8<T> chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {}
237238
really_inline simd8x64(const T ptr[64]) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+16), simd8<T>::load(ptr+32), simd8<T>::load(ptr+48)} {}
238239

239240
really_inline void store(T ptr[64]) {
240-
this->chunks[0].store(ptr);
241-
this->chunks[0].store(ptr+16);
242-
this->chunks[0].store(ptr+32);
243-
this->chunks[0].store(ptr+48);
241+
this->chunks[0].store(ptr+sizeof(simd8<T>)*0);
242+
this->chunks[1].store(ptr+sizeof(simd8<T>)*1);
243+
this->chunks[2].store(ptr+sizeof(simd8<T>)*2);
244+
this->chunks[3].store(ptr+sizeof(simd8<T>)*3);
245+
}
246+
247+
template <typename F>
248+
static really_inline void each_index(F const& each) {
249+
each(0);
250+
each(1);
251+
each(2);
252+
each(3);
244253
}
245254

246255
template <typename F>

‎src/arm64/stage1_find_marks.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ namespace simdjson {
3838

3939
template <>
4040
int find_structural_bits<Architecture::ARM64>(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) {
41-
return arm64::find_structural_bits(buf, len, pj, streaming);
41+
return arm64::stage1::find_structural_bits<64>(buf, len, pj, streaming);
4242
}
4343

4444
} // namespace simdjson

‎src/generic/stage1_find_marks.h‎

Lines changed: 54 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
// We assume the file in which it is included already includes
44
// "simdjson/stage1_find_marks.h" (this simplifies amalgation)
55

6-
static const size_t STEP_SIZE = 128;
6+
namespace stage1 {
77

88
class bit_indexer {
99
public:
@@ -74,34 +74,11 @@ class json_structural_scanner {
7474

7575
json_structural_scanner(uint32_t *_structural_indexes) : structural_indexes{_structural_indexes} {}
7676

77-
// return a bitvector indicating where we have characters that end an odd-length
78-
// sequence of backslashes (and thus change the behavior of the next character
79-
// to follow). A even-length sequence of backslashes, and, for that matter, the
80-
// largest even-length prefix of our odd-length sequence of backslashes, simply
81-
// modify the behavior of the backslashes themselves.
82-
// We also update the prev_iter_ends_odd_backslash reference parameter to
83-
// indicate whether we end an iteration on an odd-length sequence of
84-
// backslashes, which modifies our subsequent search for odd-length
85-
// sequences of backslashes in an obvious way.
86-
really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow);
87-
88-
//
89-
// Check if the current character immediately follows a matching character.
90-
//
91-
// For example, this checks for quotes with backslashes in front of them:
92-
//
93-
// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash);
94-
//
95-
really_inline uint64_t follows(const uint64_t match, uint64_t &overflow);
96-
9777
//
98-
// Check if the current character follows a matching character, with possible "filler" between.
99-
// For example, this checks for empty curly braces, e.g.
78+
// Finish the scan and return any errors.
10079
//
101-
// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { <whitespace>* }
80+
// This may detect errors as well, such as unclosed string and certain UTF-8 errors.
10281
//
103-
really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow);
104-
10582
really_inline ErrorValues detect_errors_on_eof();
10683

10784
//
@@ -114,8 +91,6 @@ class json_structural_scanner {
11491
//
11592
really_inline uint64_t find_strings(const simd::simd8x64<uint8_t> in);
11693

117-
really_inline uint64_t invalid_string_bytes(const uint64_t unescaped, const uint64_t quote_mask);
118-
11994
//
12095
// Determine which characters are *structural*:
12196
// - braces: [] and {}
@@ -135,26 +110,15 @@ class json_structural_scanner {
135110
really_inline uint64_t find_potential_structurals(const simd::simd8x64<uint8_t> in);
136111

137112
//
138-
// Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes.
139-
//
140-
// PERF NOTES:
141-
// We pipe 2 inputs through these stages:
142-
// 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load
143-
// 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available.
144-
// 2. Scan the JSON for critical data: strings, primitives and operators. This is the critical path.
145-
// The output of step 1 depends entirely on this information. These functions don't quite use
146-
// up enough CPU: the second half of the functions is highly serial, only using 1 execution core
147-
// at a time. The second input's scans has some dependency on the first ones finishing it, but
148-
// they can make a lot of progress before they need that information.
149-
// 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that
150-
// to finish: utf-8 checks and generating the output from the last iteration.
151-
//
152-
// The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all
153-
// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough
154-
// workout.
113+
// Find the important bits of JSON in a STEP_SIZE-byte chunk, and add them to structural_indexes.
155114
//
115+
template<size_t STEP_SIZE>
156116
really_inline void scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker);
157117

118+
//
119+
// Parse the entire input in STEP_SIZE-byte chunks.
120+
//
121+
template<size_t STEP_SIZE>
158122
really_inline void scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker);
159123
};
160124

@@ -167,7 +131,7 @@ class json_structural_scanner {
167131
// indicate whether we end an iteration on an odd-length sequence of
168132
// backslashes, which modifies our subsequent search for odd-length
169133
// sequences of backslashes in an obvious way.
170-
really_inline uint64_t json_structural_scanner::follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) {
134+
really_inline uint64_t follows_odd_sequence_of(const uint64_t match, uint64_t &overflow) {
171135
const uint64_t even_bits = 0x5555555555555555ULL;
172136
const uint64_t odd_bits = ~even_bits;
173137
uint64_t start_edges = match & ~(match << 1);
@@ -205,7 +169,7 @@ really_inline uint64_t json_structural_scanner::follows_odd_sequence_of(const ui
205169
//
206170
// const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash);
207171
//
208-
really_inline uint64_t json_structural_scanner::follows(const uint64_t match, uint64_t &overflow) {
172+
really_inline uint64_t follows(const uint64_t match, uint64_t &overflow) {
209173
const uint64_t result = match << 1 | overflow;
210174
overflow = match >> 63;
211175
return result;
@@ -217,7 +181,7 @@ really_inline uint64_t json_structural_scanner::follows(const uint64_t match, ui
217181
//
218182
// in.eq('}') & follows(in.eq('['), in.eq(' '), prev_empty_array) // { <whitespace>* }
219183
//
220-
really_inline uint64_t json_structural_scanner::follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) {
184+
really_inline uint64_t follows(const uint64_t match, const uint64_t filler, uint64_t &overflow) {
221185
uint64_t follows_match = follows(match, overflow);
222186
uint64_t result;
223187
overflow |= add_overflow(follows_match, filler, &result);
@@ -256,15 +220,6 @@ really_inline uint64_t json_structural_scanner::find_strings(const simd::simd8x6
256220
return in_string ^ quote;
257221
}
258222

259-
really_inline uint64_t json_structural_scanner::invalid_string_bytes(const uint64_t unescaped, const uint64_t quote_mask) {
260-
/* All Unicode characters may be placed within the
261-
* quotation marks, except for the characters that MUST be escaped:
262-
* quotation mark, reverse solidus, and the control characters (U+0000
263-
* through U+001F).
264-
* https://tools.ietf.org/html/rfc8259 */
265-
return quote_mask & unescaped;
266-
}
267-
268223
//
269224
// Determine which characters are *structural*:
270225
// - braces: [] and {}
@@ -315,7 +270,8 @@ really_inline uint64_t json_structural_scanner::find_potential_structurals(const
315270
// available capacity with just one input. Running 2 at a time seems to give the CPU a good enough
316271
// workout.
317272
//
318-
really_inline void json_structural_scanner::scan_step(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) {
273+
template<>
274+
really_inline void json_structural_scanner::scan_step<128>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) {
319275
//
320276
// Load up all 128 bytes into SIMD registers
321277
//
@@ -340,23 +296,55 @@ really_inline void json_structural_scanner::scan_step(const uint8_t *buf, const
340296
//
341297
uint64_t unescaped_1 = in_1.lteq(0x1F);
342298
utf8_checker.check_next_input(in_1);
343-
this->structural_indexes.write_indexes(idx-64, prev_structurals); // Output *last* iteration's structurals to ParsedJson
299+
this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson
344300
this->prev_structurals = structurals_1 & ~string_1;
345301
this->unescaped_chars_error |= unescaped_1 & string_1;
346302

347303
uint64_t unescaped_2 = in_2.lteq(0x1F);
348304
utf8_checker.check_next_input(in_2);
349-
this->structural_indexes.write_indexes(idx, prev_structurals); // Output *last* iteration's structurals to ParsedJson
305+
this->structural_indexes.write_indexes(idx, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson
350306
this->prev_structurals = structurals_2 & ~string_2;
351307
this->unescaped_chars_error |= unescaped_2 & string_2;
352308
}
353309

310+
//
311+
// Find the important bits of JSON in a 64-byte chunk, and add them to structural_indexes.
312+
//
313+
template<>
314+
really_inline void json_structural_scanner::scan_step<64>(const uint8_t *buf, const size_t idx, utf8_checker &utf8_checker) {
315+
//
316+
// Load up bytes into SIMD registers
317+
//
318+
simd::simd8x64<uint8_t> in_1(buf);
319+
320+
//
321+
// Find the strings and potential structurals (operators / primitives).
322+
//
323+
// This will include false structurals that are *inside* strings--we'll filter strings out
324+
// before we return.
325+
//
326+
uint64_t string_1 = this->find_strings(in_1);
327+
uint64_t structurals_1 = this->find_potential_structurals(in_1);
328+
329+
//
330+
// Do miscellaneous work while the processor is busy calculating strings and structurals.
331+
//
332+
// After that, weed out structurals that are inside strings and find invalid string characters.
333+
//
334+
uint64_t unescaped_1 = in_1.lteq(0x1F);
335+
utf8_checker.check_next_input(in_1);
336+
this->structural_indexes.write_indexes(idx-64, this->prev_structurals); // Output *last* iteration's structurals to ParsedJson
337+
this->prev_structurals = structurals_1 & ~string_1;
338+
this->unescaped_chars_error |= unescaped_1 & string_1;
339+
}
340+
341+
template<size_t STEP_SIZE>
354342
really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_t len, utf8_checker &utf8_checker) {
355343
size_t lenminusstep = len < STEP_SIZE ? 0 : len - STEP_SIZE;
356344
size_t idx = 0;
357345

358346
for (; idx < lenminusstep; idx += STEP_SIZE) {
359-
this->scan_step(&buf[idx], idx, utf8_checker);
347+
this->scan_step<STEP_SIZE>(&buf[idx], idx, utf8_checker);
360348
}
361349

362350
/* If we have a final chunk of less than 64 bytes, pad it to 64 with
@@ -366,14 +354,15 @@ really_inline void json_structural_scanner::scan(const uint8_t *buf, const size_
366354
uint8_t tmp_buf[STEP_SIZE];
367355
memset(tmp_buf, 0x20, STEP_SIZE);
368356
memcpy(tmp_buf, buf + idx, len - idx);
369-
this->scan_step(&tmp_buf[0], idx, utf8_checker);
357+
this->scan_step<STEP_SIZE>(&tmp_buf[0], idx, utf8_checker);
370358
idx += STEP_SIZE;
371359
}
372360

373361
/* finally, flatten out the remaining structurals from the last iteration */
374362
this->structural_indexes.write_indexes(idx-64, this->prev_structurals);
375363
}
376364

365+
template<size_t STEP_SIZE>
377366
int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) {
378367
if (unlikely(len > pj.byte_capacity)) {
379368
std::cerr << "Your ParsedJson object only supports documents up to "
@@ -383,7 +372,7 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p
383372
}
384373
utf8_checker utf8_checker{};
385374
json_structural_scanner scanner{pj.structural_indexes};
386-
scanner.scan(buf, len, utf8_checker);
375+
scanner.scan<STEP_SIZE>(buf, len, utf8_checker);
387376

388377
simdjson::ErrorValues error = scanner.detect_errors_on_eof();
389378
if (!streaming && unlikely(error != simdjson::SUCCESS)) {
@@ -408,3 +397,5 @@ int find_structural_bits(const uint8_t *buf, size_t len, simdjson::ParsedJson &p
408397
pj.structural_indexes[pj.n_structural_indexes] = 0;
409398
return utf8_checker.errors();
410399
}
400+
401+
} // namespace stage1

‎src/haswell/simd.h‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -198,15 +198,22 @@ namespace simdjson::haswell::simd {
198198

199199
template<typename T>
200200
struct simd8x64 {
201-
const simd8<T> chunks[2];
201+
static const int NUM_CHUNKS = 64 / sizeof(simd8<T>);
202+
const simd8<T> chunks[NUM_CHUNKS];
202203

203204
really_inline simd8x64() : chunks{simd8<T>(), simd8<T>()} {}
204205
really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1) : chunks{chunk0, chunk1} {}
205206
really_inline simd8x64(const T ptr[64]) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+32)} {}
206207

207-
really_inline void store(T *ptr) {
208-
this->chunks[0].store(ptr);
209-
this->chunks[0].store(ptr+sizeof(simd8<T>));
208+
template <typename F>
209+
static really_inline void each_index(F const& each) {
210+
each(0);
211+
each(1);
212+
}
213+
214+
really_inline void store(T ptr[64]) {
215+
this->chunks[0].store(ptr+sizeof(simd8<T>)*0);
216+
this->chunks[1].store(ptr+sizeof(simd8<T>)*1);
210217
}
211218

212219
template <typename F>

‎src/haswell/stage1_find_marks.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ namespace simdjson {
3838

3939
template <>
4040
int find_structural_bits<Architecture::HASWELL>(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) {
41-
return haswell::find_structural_bits(buf, len, pj, streaming);
41+
return haswell::stage1::find_structural_bits<128>(buf, len, pj, streaming);
4242
}
4343

4444
} // namespace simdjson

‎src/westmere/simd.h‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -188,17 +188,26 @@ namespace simdjson::westmere::simd {
188188

189189
template<typename T>
190190
struct simd8x64 {
191-
const simd8<T> chunks[4];
191+
static const int NUM_CHUNKS = 64 / sizeof(simd8<T>);
192+
const simd8<T> chunks[NUM_CHUNKS];
192193

193194
really_inline simd8x64() : chunks{simd8<T>(), simd8<T>(), simd8<T>(), simd8<T>()} {}
194195
really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1, const simd8<T> chunk2, const simd8<T> chunk3) : chunks{chunk0, chunk1, chunk2, chunk3} {}
195196
really_inline simd8x64(const T ptr[64]) : chunks{simd8<T>::load(ptr), simd8<T>::load(ptr+16), simd8<T>::load(ptr+32), simd8<T>::load(ptr+48)} {}
196197

197198
really_inline void store(T ptr[64]) {
198-
this->chunks[0].store(ptr);
199-
this->chunks[0].store(ptr+16);
200-
this->chunks[0].store(ptr+32);
201-
this->chunks[0].store(ptr+48);
199+
this->chunks[0].store(ptr+sizeof(simd8<T>)*0);
200+
this->chunks[1].store(ptr+sizeof(simd8<T>)*1);
201+
this->chunks[2].store(ptr+sizeof(simd8<T>)*2);
202+
this->chunks[3].store(ptr+sizeof(simd8<T>)*3);
203+
}
204+
205+
template <typename F>
206+
static really_inline void each_index(F const& each) {
207+
each(0);
208+
each(1);
209+
each(2);
210+
each(3);
202211
}
203212

204213
template <typename F>

‎src/westmere/stage1_find_marks.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ namespace simdjson {
3838

3939
template <>
4040
int find_structural_bits<Architecture::WESTMERE>(const uint8_t *buf, size_t len, simdjson::ParsedJson &pj, bool streaming) {
41-
return westmere::find_structural_bits(buf, len, pj, streaming);
41+
return westmere::stage1::find_structural_bits<64>(buf, len, pj, streaming);
4242
}
4343

4444
} // namespace simdjson

0 commit comments

Comments
 (0)