forked from simdjson/simdjson
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathjson_string_scanner.h
More file actions
127 lines (116 loc) · 5.87 KB
/
Copy pathjson_string_scanner.h
File metadata and controls
127 lines (116 loc) · 5.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
namespace stage1 {
struct json_string_block {
// Escaped characters (characters following an escape() character)
really_inline uint64_t escaped() const { return _escaped; }
// Escape characters (backslashes that are not escaped--i.e. in \\, includes only the first \)
really_inline uint64_t escape() const { return _backslash & ~_escaped; }
// Real (non-backslashed) quotes
really_inline uint64_t quote() const { return _quote; }
// Start quotes of strings
really_inline uint64_t string_end() const { return _quote & _in_string; }
// End quotes of strings
really_inline uint64_t string_start() const { return _quote & ~_in_string; }
// Only characters inside the string (not including the quotes)
really_inline uint64_t string_content() const { return _in_string & ~_quote; }
// Return a mask of whether the given characters are inside a string (only works on non-quotes)
really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; }
// Return a mask of whether the given characters are inside a string (only works on non-quotes)
really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; }
// Tail of string (everything except the start quote)
really_inline uint64_t string_tail() const { return _in_string ^ _quote; }
// backslash characters
uint64_t _backslash;
// escaped characters (backslashed--does not include the hex characters after \u)
uint64_t _escaped;
// real quotes (non-backslashed ones)
uint64_t _quote;
// string characters (includes start quote but not end quote)
uint64_t _in_string;
};
// Scans blocks for string characters, storing the state necessary to do so
class json_string_scanner {
public:
really_inline json_string_block next(const simd::simd8x64<uint8_t> in);
really_inline error_code finish(bool streaming);
private:
really_inline uint64_t find_escaped(uint64_t escape);
// Whether the last iteration was still inside a string (all 1's = true, all 0's = false).
uint64_t prev_in_string = 0ULL;
// Whether the first character of the next iteration is escaped.
uint64_t prev_escaped = 0ULL;
};
//
// Finds escaped characters (characters following \).
//
// Handles runs of backslashes like \\\" and \\\\" correctly (yielding 0101 and 01010, respectively).
//
// Does this by:
// - Shift the escape mask to get potentially escaped characters (characters after backslashes).
// - Mask escaped sequences that start on *even* bits with 1010101010 (odd bits are escaped, even bits are not)
// - Mask escaped sequences that start on *odd* bits with 0101010101 (even bits are escaped, odd bits are not)
//
// To distinguish between escaped sequences starting on even/odd bits, it finds the start of all
// escape sequences, filters out the ones that start on even bits, and adds that to the mask of
// escape sequences. This causes the addition to clear out the sequences starting on odd bits (since
// the start bit causes a carry), and leaves even-bit sequences alone.
//
// Example:
//
// text | \\\ | \\\"\\\" \\\" \\"\\" |
// escape | xxx | xx xxx xxx xx xx | Removed overflow backslash; will | it into follows_escape
// odd_starts | x | x x x | escape & ~even_bits & ~follows_escape
// even_seq | c| cxxx c xx c | c = carry bit -- will be masked out later
// invert_mask | | cxxx c xx c| even_seq << 1
// follows_escape | xx | x xx xxx xxx xx xx | Includes overflow bit
// escaped | x | x x x x x x x x |
// desired | x | x x x x x x x x |
// text | \\\ | \\\"\\\" \\\" \\"\\" |
//
really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
// If there was overflow, pretend the first character isn't a backslash
backslash &= ~prev_escaped;
uint64_t follows_escape = backslash << 1 | prev_escaped;
// Get sequences starting on even bits by clearing out the odd series using +
const uint64_t even_bits = 0x5555555555555555ULL;
uint64_t odd_sequence_starts = backslash & ~even_bits & ~follows_escape;
uint64_t sequences_starting_on_even_bits;
prev_escaped = add_overflow(odd_sequence_starts, backslash, &sequences_starting_on_even_bits);
uint64_t invert_mask = sequences_starting_on_even_bits << 1; // The mask we want to return is the *escaped* bits, not escapes.
// Mask every other backslashed character as an escaped character
// Flip the mask for sequences that start on even bits, to correct them
return (even_bits ^ invert_mask) & follows_escape;
}
//
// Return a mask of all string characters plus end quotes.
//
// prev_escaped is overflow saying whether the next character is escaped.
// prev_in_string is overflow saying whether we're still in a string.
//
// Backslash sequences outside of quotes will be detected in stage 2.
//
really_inline json_string_block json_string_scanner::next(const simd::simd8x64<uint8_t> in) {
const uint64_t backslash = in.eq('\\');
const uint64_t escaped = find_escaped(backslash);
const uint64_t quote = in.eq('"') & ~escaped;
// prefix_xor flips on bits inside the string (and flips off the end quote).
// Then we xor with prev_in_string: if we were in a string already, its effect is flipped
// (characters inside strings are outside, and characters outside strings are inside).
const uint64_t in_string = prefix_xor(quote) ^ prev_in_string;
// right shift of a signed value expected to be well-defined and standard
// compliant as of C++20, John Regher from Utah U. says this is fine code
prev_in_string = static_cast<uint64_t>(static_cast<int64_t>(in_string) >> 63);
// Use ^ to turn the beginning quote off, and the end quote on.
return {
backslash,
escaped,
quote,
in_string
};
}
really_inline error_code json_string_scanner::finish(bool streaming) {
if (prev_in_string and (not streaming)) {
return UNCLOSED_STRING;
}
return SUCCESS;
}
} // namespace stage1