Skip to content

Commit 0042d9b

Browse files
jkeiserjkeiser
committed
Move UTF8 checking functions into their own file
1 parent 237b886 commit 0042d9b

12 files changed

Lines changed: 3544 additions & 1683 deletions

‎amalgamation.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ $SCRIPTPATH/include/simdjson/jsoncharutils.h
3636
$SCRIPTPATH/include/simdjson/jsonformatutils.h
3737
$SCRIPTPATH/include/simdjson/jsonioutil.h
3838
$SCRIPTPATH/include/simdjson/simdprune_tables.h
39+
$SCRIPTPATH/include/simdjson/simd_input.h
40+
$SCRIPTPATH/include/simdjson/simd_input_haswell.h
41+
$SCRIPTPATH/include/simdjson/simd_input_westmere.h
42+
$SCRIPTPATH/include/simdjson/simd_input_arm64.h
43+
$SCRIPTPATH/include/simdjson/simdutf8check.h
3944
$SCRIPTPATH/include/simdjson/simdutf8check_haswell.h
4045
$SCRIPTPATH/include/simdjson/simdutf8check_westmere.h
4146
$SCRIPTPATH/include/simdjson/simdutf8check_arm64.h

‎include/simdjson/simd_input.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,13 @@ namespace simdjson {
1010

1111
template <Architecture> struct simd_input;
1212

13+
template <Architecture T>
14+
simd_input<T> fill_input(const uint8_t *ptr);
15+
1316
// a straightforward comparison of a mask against input.
1417
template <Architecture T>
1518
uint64_t cmp_mask_against_input(simd_input<T> in, uint8_t m);
1619

17-
template <Architecture T> simd_input<T> fill_input(const uint8_t *ptr);
18-
1920
// find all values less than or equal than the content of maxval (using unsigned
2021
// arithmetic)
2122
template <Architecture T>

‎include/simdjson/simdutf8check.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#ifndef SIMDJSON_SIMDUTF8CHECK_H
2+
#define SIMDJSON_SIMDUTF8CHECK_H
3+
4+
#include "simdjson/simdjson.h"
5+
#include "simdjson/simd_input.h"
6+
7+
namespace simdjson {
8+
9+
// Holds the state required to perform check_utf8().
10+
template <Architecture> struct utf8_checking_state;
11+
12+
template <Architecture T>
13+
void check_utf8(simd_input<T> in, utf8_checking_state<T> &state);
14+
15+
// Checks if the utf8 validation has found any error.
16+
template <Architecture T>
17+
ErrorValues check_utf8_errors(utf8_checking_state<T> &state);
18+
19+
} // namespace simdjson
20+
21+
#endif // SIMDJSON_SIMDUTF8CHECK_H

‎include/simdjson/simdutf8check_arm64.h‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#if defined(_ARM_NEON) || defined(__aarch64__) || \
88
(defined(_MSC_VER) && defined(_M_ARM64))
99

10+
#include "simdjson/simdutf8check.h"
1011
#include <arm_neon.h>
1112
#include <cinttypes>
1213
#include <cstddef>
@@ -175,6 +176,64 @@ check_utf8_bytes(int8x16_t current_bytes, struct processed_utf_bytes *previous,
175176
previous->high_nibbles, has_error);
176177
return pb;
177178
}
179+
180+
template <>
181+
struct utf8_checking_state<Architecture::ARM64> {
182+
int8x16_t has_error{};
183+
processed_utf_bytes previous{};
184+
};
185+
186+
// Checks that all bytes are ascii
187+
really_inline bool check_ascii_neon(simd_input<Architecture::ARM64> in) {
188+
// checking if the most significant bit is always equal to 0.
189+
uint8x16_t high_bit = vdupq_n_u8(0x80);
190+
uint8x16_t t0 = vorrq_u8(in.i0, in.i1);
191+
uint8x16_t t1 = vorrq_u8(in.i2, in.i3);
192+
uint8x16_t t3 = vorrq_u8(t0, t1);
193+
uint8x16_t t4 = vandq_u8(t3, high_bit);
194+
uint64x2_t v64 = vreinterpretq_u64_u8(t4);
195+
uint32x2_t v32 = vqmovn_u64(v64);
196+
uint64x1_t result = vreinterpret_u64_u32(v32);
197+
return vget_lane_u64(result, 0) == 0;
198+
}
199+
200+
template <>
201+
really_inline void check_utf8<Architecture::ARM64>(
202+
simd_input<Architecture::ARM64> in,
203+
utf8_checking_state<Architecture::ARM64> &state) {
204+
if (check_ascii_neon(in)) {
205+
// All bytes are ascii. Therefore the byte that was just before must be
206+
// ascii too. We only check the byte that was just before simd_input. Nines
207+
// are arbitrary values.
208+
const int8x16_t verror =
209+
(int8x16_t){9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1};
210+
state.has_error =
211+
vorrq_s8(vreinterpretq_s8_u8(
212+
vcgtq_s8(state.previous.carried_continuations, verror)),
213+
state.has_error);
214+
} else {
215+
// it is not ascii so we have to do heavy work
216+
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i0),
217+
&(state.previous), &(state.has_error));
218+
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i1),
219+
&(state.previous), &(state.has_error));
220+
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i2),
221+
&(state.previous), &(state.has_error));
222+
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i3),
223+
&(state.previous), &(state.has_error));
224+
}
225+
}
226+
227+
template <>
228+
really_inline ErrorValues check_utf8_errors<Architecture::ARM64>(
229+
utf8_checking_state<Architecture::ARM64> &state) {
230+
uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error);
231+
uint32x2_t v32 = vqmovn_u64(v64);
232+
uint64x1_t result = vreinterpret_u64_u32(v32);
233+
return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR
234+
: simdjson::SUCCESS;
235+
}
236+
178237
} // namespace simdjson
179238
#endif
180239
#endif

‎include/simdjson/simdutf8check_haswell.h‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#define SIMDJSON_SIMDUTF8CHECK_HASWELL_H
33

44
#include "simdjson/portability.h"
5+
#include "simdjson/simdutf8check.h"
56
#include <stddef.h>
67
#include <stdint.h>
78
#include <string.h>
@@ -190,6 +191,48 @@ avx_check_utf8_bytes(__m256i current_bytes,
190191
previous->high_nibbles, has_error);
191192
return pb;
192193
}
194+
195+
template <> struct utf8_checking_state<Architecture::HASWELL> {
196+
__m256i has_error;
197+
avx_processed_utf_bytes previous;
198+
utf8_checking_state() {
199+
has_error = _mm256_setzero_si256();
200+
previous.raw_bytes = _mm256_setzero_si256();
201+
previous.high_nibbles = _mm256_setzero_si256();
202+
previous.carried_continuations = _mm256_setzero_si256();
203+
}
204+
};
205+
206+
template <>
207+
really_inline void check_utf8<Architecture::HASWELL>(
208+
simd_input<Architecture::HASWELL> in,
209+
utf8_checking_state<Architecture::HASWELL> &state) {
210+
__m256i high_bit = _mm256_set1_epi8(0x80u);
211+
if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), high_bit)) == 1) {
212+
// it is ascii, we just check continuation
213+
state.has_error = _mm256_or_si256(
214+
_mm256_cmpgt_epi8(state.previous.carried_continuations,
215+
_mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
216+
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
217+
9, 9, 9, 9, 9, 9, 9, 1)),
218+
state.has_error);
219+
} else {
220+
// it is not ascii so we have to do heavy work
221+
state.previous =
222+
avx_check_utf8_bytes(in.lo, &(state.previous), &(state.has_error));
223+
state.previous =
224+
avx_check_utf8_bytes(in.hi, &(state.previous), &(state.has_error));
225+
}
226+
}
227+
228+
template <>
229+
really_inline ErrorValues check_utf8_errors<Architecture::HASWELL>(
230+
utf8_checking_state<Architecture::HASWELL> &state) {
231+
return _mm256_testz_si256(state.has_error, state.has_error) == 0
232+
? simdjson::UTF8_ERROR
233+
: simdjson::SUCCESS;
234+
}
235+
193236
} // namespace simdjson
194237
UNTARGET_REGION // haswell
195238

‎include/simdjson/simdutf8check_westmere.h‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#define SIMDJSON_SIMDUTF8CHECK_WESTMERE_H
33

44
#include "simdjson/portability.h"
5+
#include "simdjson/simdutf8check.h"
56
#include <stddef.h>
67
#include <stdint.h>
78
#include <string.h>
@@ -161,6 +162,61 @@ check_utf8_bytes(__m128i current_bytes, struct processed_utf_bytes *previous,
161162
previous->high_nibbles, has_error);
162163
return pb;
163164
}
165+
166+
template <>
167+
struct utf8_checking_state<Architecture::WESTMERE> {
168+
__m128i has_error = _mm_setzero_si128();
169+
processed_utf_bytes previous{
170+
_mm_setzero_si128(), // raw_bytes
171+
_mm_setzero_si128(), // high_nibbles
172+
_mm_setzero_si128() // carried_continuations
173+
};
174+
};
175+
176+
template <>
177+
really_inline void check_utf8<Architecture::WESTMERE>(
178+
simd_input<Architecture::WESTMERE> in,
179+
utf8_checking_state<Architecture::WESTMERE> &state) {
180+
__m128i high_bit = _mm_set1_epi8(0x80u);
181+
if ((_mm_testz_si128(_mm_or_si128(in.v0, in.v1), high_bit)) == 1) {
182+
// it is ascii, we just check continuation
183+
state.has_error =
184+
_mm_or_si128(_mm_cmpgt_epi8(state.previous.carried_continuations,
185+
_mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
186+
9, 9, 9, 9, 9, 1)),
187+
state.has_error);
188+
} else {
189+
// it is not ascii so we have to do heavy work
190+
state.previous =
191+
check_utf8_bytes(in.v0, &(state.previous), &(state.has_error));
192+
state.previous =
193+
check_utf8_bytes(in.v1, &(state.previous), &(state.has_error));
194+
}
195+
196+
if ((_mm_testz_si128(_mm_or_si128(in.v2, in.v3), high_bit)) == 1) {
197+
// it is ascii, we just check continuation
198+
state.has_error =
199+
_mm_or_si128(_mm_cmpgt_epi8(state.previous.carried_continuations,
200+
_mm_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
201+
9, 9, 9, 9, 9, 1)),
202+
state.has_error);
203+
} else {
204+
// it is not ascii so we have to do heavy work
205+
state.previous =
206+
check_utf8_bytes(in.v2, &(state.previous), &(state.has_error));
207+
state.previous =
208+
check_utf8_bytes(in.v3, &(state.previous), &(state.has_error));
209+
}
210+
}
211+
212+
template <>
213+
really_inline ErrorValues check_utf8_errors<Architecture::WESTMERE>(
214+
utf8_checking_state<Architecture::WESTMERE> &state) {
215+
return _mm_testz_si128(state.has_error, state.has_error) == 0
216+
? simdjson::UTF8_ERROR
217+
: simdjson::SUCCESS;
218+
}
219+
164220
} // namespace simdjson
165221
UNTARGET_REGION // westmere
166222

‎include/simdjson/stage1_find_marks_arm64.h‎

Lines changed: 0 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -18,63 +18,6 @@ compute_quote_mask<Architecture::ARM64>(uint64_t quote_bits) {
1818
#endif
1919
}
2020

21-
template <>
22-
struct utf8_checking_state<Architecture::ARM64> {
23-
int8x16_t has_error{};
24-
processed_utf_bytes previous{};
25-
};
26-
27-
// Checks that all bytes are ascii
28-
really_inline bool check_ascii_neon(simd_input<Architecture::ARM64> in) {
29-
// checking if the most significant bit is always equal to 0.
30-
uint8x16_t high_bit = vdupq_n_u8(0x80);
31-
uint8x16_t t0 = vorrq_u8(in.i0, in.i1);
32-
uint8x16_t t1 = vorrq_u8(in.i2, in.i3);
33-
uint8x16_t t3 = vorrq_u8(t0, t1);
34-
uint8x16_t t4 = vandq_u8(t3, high_bit);
35-
uint64x2_t v64 = vreinterpretq_u64_u8(t4);
36-
uint32x2_t v32 = vqmovn_u64(v64);
37-
uint64x1_t result = vreinterpret_u64_u32(v32);
38-
return vget_lane_u64(result, 0) == 0;
39-
}
40-
41-
template <>
42-
really_inline void check_utf8<Architecture::ARM64>(
43-
simd_input<Architecture::ARM64> in,
44-
utf8_checking_state<Architecture::ARM64> &state) {
45-
if (check_ascii_neon(in)) {
46-
// All bytes are ascii. Therefore the byte that was just before must be
47-
// ascii too. We only check the byte that was just before simd_input. Nines
48-
// are arbitrary values.
49-
const int8x16_t verror =
50-
(int8x16_t){9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1};
51-
state.has_error =
52-
vorrq_s8(vreinterpretq_s8_u8(
53-
vcgtq_s8(state.previous.carried_continuations, verror)),
54-
state.has_error);
55-
} else {
56-
// it is not ascii so we have to do heavy work
57-
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i0),
58-
&(state.previous), &(state.has_error));
59-
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i1),
60-
&(state.previous), &(state.has_error));
61-
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i2),
62-
&(state.previous), &(state.has_error));
63-
state.previous = check_utf8_bytes(vreinterpretq_s8_u8(in.i3),
64-
&(state.previous), &(state.has_error));
65-
}
66-
}
67-
68-
template <>
69-
really_inline ErrorValues check_utf8_errors<Architecture::ARM64>(
70-
utf8_checking_state<Architecture::ARM64> &state) {
71-
uint64x2_t v64 = vreinterpretq_u64_s8(state.has_error);
72-
uint32x2_t v32 = vqmovn_u64(v64);
73-
uint64x1_t result = vreinterpret_u64_u32(v32);
74-
return vget_lane_u64(result, 0) != 0 ? simdjson::UTF8_ERROR
75-
: simdjson::SUCCESS;
76-
}
77-
7821
template <>
7922
really_inline void find_whitespace_and_structurals<Architecture::ARM64>(
8023
simd_input<Architecture::ARM64> in, uint64_t &whitespace,

‎include/simdjson/stage1_find_marks_haswell.h‎

Lines changed: 0 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -20,47 +20,6 @@ compute_quote_mask<Architecture::HASWELL>(uint64_t quote_bits) {
2020
return quote_mask;
2121
}
2222

23-
template <> struct utf8_checking_state<Architecture::HASWELL> {
24-
__m256i has_error;
25-
avx_processed_utf_bytes previous;
26-
utf8_checking_state() {
27-
has_error = _mm256_setzero_si256();
28-
previous.raw_bytes = _mm256_setzero_si256();
29-
previous.high_nibbles = _mm256_setzero_si256();
30-
previous.carried_continuations = _mm256_setzero_si256();
31-
}
32-
};
33-
34-
template <>
35-
really_inline void check_utf8<Architecture::HASWELL>(
36-
simd_input<Architecture::HASWELL> in,
37-
utf8_checking_state<Architecture::HASWELL> &state) {
38-
__m256i high_bit = _mm256_set1_epi8(0x80u);
39-
if ((_mm256_testz_si256(_mm256_or_si256(in.lo, in.hi), high_bit)) == 1) {
40-
// it is ascii, we just check continuation
41-
state.has_error = _mm256_or_si256(
42-
_mm256_cmpgt_epi8(state.previous.carried_continuations,
43-
_mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
44-
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
45-
9, 9, 9, 9, 9, 9, 9, 1)),
46-
state.has_error);
47-
} else {
48-
// it is not ascii so we have to do heavy work
49-
state.previous =
50-
avx_check_utf8_bytes(in.lo, &(state.previous), &(state.has_error));
51-
state.previous =
52-
avx_check_utf8_bytes(in.hi, &(state.previous), &(state.has_error));
53-
}
54-
}
55-
56-
template <>
57-
really_inline ErrorValues check_utf8_errors<Architecture::HASWELL>(
58-
utf8_checking_state<Architecture::HASWELL> &state) {
59-
return _mm256_testz_si256(state.has_error, state.has_error) == 0
60-
? simdjson::UTF8_ERROR
61-
: simdjson::SUCCESS;
62-
}
63-
6423
template <>
6524
really_inline void find_whitespace_and_structurals<Architecture::HASWELL>(
6625
simd_input<Architecture::HASWELL> in, uint64_t &whitespace,

0 commit comments

Comments
 (0)