Skip to content

Commit 78e75a8

Browse files
lemirelemire
committed
Even faster.
1 parent 7dd590c commit 78e75a8

3 files changed

Lines changed: 19 additions & 41 deletions

File tree

‎Makefile‎

Lines changed: 4 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66

77
.PHONY: clean cleandist
88

9-
CXXFLAGS = -std=c++11 -g2 -O3 -march=native -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux -Idependencies/rapidjson/include -Idependencies/sajson/include
9+
CXXFLAGS = -std=c++11 -march=native -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux -Idependencies/rapidjson/include -Idependencies/sajson/include
1010

1111
ifeq ($(SANITIZE),1)
12-
CXXFLAGS += -g2 -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined
12+
CXXFLAGS += -g3 -O0 -fsanitize=address -fno-omit-frame-pointer -fsanitize=undefined
13+
else
14+
CXXFLAGS += -O3
1315
endif
1416

1517
EXECUTABLES=parse jsoncheck numberparsingcheck stringparsingcheck minifiercompetition parsingcompetition minify allparserscheckfile
@@ -19,7 +21,6 @@ LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/stage1_find_marks.cpp src
1921
MINIFIERHEADERS=include/jsonparser/jsonminifier.h include/jsonparser/simdprune_tables.h
2022
MINIFIERLIBFILES=src/jsonminifier.cpp
2123

22-
EXTRA_EXECUTABLES=parsenocheesy parsenodep8
2324

2425
RAPIDJSON_INCLUDE:=dependencies/rapidjson/include
2526
SAJSON_INCLUDE:=dependencies/sajson/include
@@ -79,30 +80,6 @@ allparserscheckfile: tests/allparserscheckfile.cpp $(HEADERS) $(LIBFILES)
7980
parsehisto: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
8081
$(CXX) $(CXXFLAGS) -o parsehisto benchmark/parse.cpp $(LIBFILES) $(LIBFLAGS) -DBUILDHISTOGRAM
8182

82-
testflatten: parse parsenocheesy parsenodep8 parsenodep10 parsenodep12
83-
for filename in jsonexamples/twitter.json jsonexamples/gsoc-2018.json jsonexamples/citm_catalog.json jsonexamples/canada.json ; do \
84-
echo $$filename ; \
85-
set -x; \
86-
./parsenocheesy $$filename ; \
87-
./parse $$filename ; \
88-
./parsenodep8 $$filename ; \
89-
./parsenodep10 $$filename ; \
90-
./parsenodep12 $$filename ; \
91-
set +x; \
92-
done
93-
94-
parsenocheesy: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
95-
$(CXX) $(CXXFLAGS) -o parsenocheesy benchmark/parse.cpp $(LIBFILES) -DSUPPRESS_CHEESY_FLATTEN
96-
97-
parsenodep8: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
98-
$(CXX) $(CXXFLAGS) -o parsenodep8 benchmark/parse.cpp $(LIBFILES) -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=8
99-
100-
parsenodep10: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
101-
$(CXX) $(CXXFLAGS) -o parsenodep12 benchmark/parse.cpp $(LIBFILES) -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=10
102-
103-
parsenodep12: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
104-
$(CXX) $(CXXFLAGS) -o parsenodep12 benchmark/parse.cpp $(LIBFILES) -DNO_PDEP_PLEASE -DNO_PDEP_WIDTH=12
105-
10683
clean:
10784
rm -f $(EXECUTABLES) $(EXTRA_EXECUTABLES)
10885

‎include/jsonparser/jsoncharutils.h‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ const char digittoval[256] = {
4949
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
5050
-1, -1, -1, -1, -1, -1, -1, -1, -1};
5151

52+
// return true if we have a valid hex between 0000 and FFFF
5253
inline bool hex_to_u32(const u8 *src, u32 *res) {
5354
u8 v1 = src[0];
5455
u8 v2 = src[1];
@@ -59,6 +60,16 @@ inline bool hex_to_u32(const u8 *src, u32 *res) {
5960
return (int32_t)(*res) >= 0;
6061
}
6162

63+
// returns a value with the highest bit set if it is not valud
64+
uint32_t hex_to_u32_nocheck(const u8 *src) {
65+
u8 v1 = src[0];
66+
u8 v2 = src[1];
67+
u8 v3 = src[2];
68+
u8 v4 = src[3];
69+
return digittoval[v1] << 12 | digittoval[v2] << 8 | digittoval[v3] << 4 |
70+
digittoval[v4];
71+
}
72+
6273
// given a code point cp, writes to c
6374
// the utf-8 code, outputting the length in
6475
// bytes, if the length is zero, the code point
@@ -83,7 +94,7 @@ inline size_t codepoint_to_utf8(uint32_t cp, u8 *c) {
8394
c[1] = ((cp >> 6) & 63) + 128;
8495
c[2] = (cp & 63) + 128;
8596
return 3;
86-
} else if (cp <= 0x10FFFF) {
97+
} else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this is not needed
8798
c[0] = (cp >> 18) + 240;
8899
c[1] = ((cp >> 12) & 63) + 128;
89100
c[2] = ((cp >> 6) & 63) + 128;

‎include/jsonparser/stringparsing.h‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -39,32 +39,22 @@ static const u8 escape_map[256] = {
3939
// return true if the unicode codepoint was valid
4040
// We work in little-endian then swap at write time
4141
really_inline bool handle_unicode_codepoint(const u8 **src_ptr, u8 **dst_ptr) {
42-
u32 code_point = 0; // read the hex, potentially reading another \u beyond if it is a surrogate pair
43-
if (!hex_to_u32(*src_ptr + 2, &code_point)) {
44-
return false;
45-
}
42+
u32 code_point = hex_to_u32_nocheck(*src_ptr + 2);
4643
*src_ptr += 6;
4744
// check for low surrogate for characters outside the Basic
4845
// Multilingual Plane.
4946
if (code_point >= 0xd800 && code_point < 0xdc00) {
5047
if (((*src_ptr)[0] != '\\') || (*src_ptr)[1] != 'u') {
5148
return false;
5249
}
53-
u32 code_point_2 = 0;
54-
if (!hex_to_u32(*src_ptr + 2, &code_point_2)) {
55-
return false;
56-
}
57-
if (code_point_2 < 0xdc00 || code_point_2 > 0xdfff) {
58-
return false;
59-
}
50+
u32 code_point_2 = hex_to_u32_nocheck(*src_ptr + 2);
6051
code_point =
6152
(((code_point - 0xd800) << 10) | (code_point_2 - 0xdc00)) + 0x10000;
6253
*src_ptr += 6;
6354
}
6455
size_t offset = codepoint_to_utf8(code_point, *dst_ptr);
65-
// assert(offset > 0);
6656
*dst_ptr += offset;
67-
return true;
57+
return offset > 0;
6858
}
6959

7060
really_inline bool parse_string(const u8 *buf, UNUSED size_t len,

0 commit comments

Comments
 (0)