Skip to content

Commit 2013396

Browse files
lemirelemire
committed
Trying a detailed analysis.
1 parent 7d37dd5 commit 2013396

6 files changed

Lines changed: 73 additions & 17 deletions

File tree

‎Makefile‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ endif
2525
MAINEXECUTABLES=parse minify json2json
2626
TESTEXECUTABLES=jsoncheck numberparsingcheck stringparsingcheck
2727
COMPARISONEXECUTABLES=minifiercompetition parsingcompetition parseandstatcompetition distinctuseridcompetition allparserscheckfile
28+
SUPPLEMENTARYEXECUTABLES=parse_noutf8validation parse_nonumberparsing parse_nostringparsing
2829

2930
HEADERS= include/simdjson/simdutf8check.h include/simdjson/stringparsing.h include/simdjson/numberparsing.h include/simdjson/jsonparser.h include/simdjson/common_defs.h include/simdjson/jsonioutil.h benchmark/benchmark.h benchmark/linux/linux-perf-events.h include/simdjson/parsedjson.h include/simdjson/stage1_find_marks.h include/simdjson/stage2_flatten.h include/simdjson/stage34_unified.h include/simdjson/jsoncharutils.h include/simdjson/jsonformatutils.h
3031
LIBFILES=src/jsonioutil.cpp src/jsonparser.cpp src/stage1_find_marks.cpp src/stage2_flatten.cpp src/stage34_unified.cpp
@@ -85,6 +86,16 @@ $(UJSON4C_INCLUDE):
8586
parse: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
8687
$(CXX) $(CXXFLAGS) -o parse $(LIBFILES) benchmark/parse.cpp $(LIBFLAGS)
8788

89+
parse_noutf8validation: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
90+
$(CXX) $(CXXFLAGS) -o parse_noutf8validation -DSIMDJSON_SKIPUTF8VALIDATION $(LIBFILES) benchmark/parse.cpp $(LIBFLAGS)
91+
92+
parse_nonumberparsing: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
93+
$(CXX) $(CXXFLAGS) -o parse_nonumberparsing -DSIMDJSON_SKIPNUMBERPARSING $(LIBFILES) benchmark/parse.cpp $(LIBFLAGS)
94+
95+
parse_nostringparsing: benchmark/parse.cpp $(HEADERS) $(LIBFILES)
96+
$(CXX) $(CXXFLAGS) -o parse_nostringparsing -DSIMDJSON_SKIPSTRINGPARSING $(LIBFILES) benchmark/parse.cpp $(LIBFLAGS)
97+
98+
8899
jsoncheck:tests/jsoncheck.cpp $(HEADERS) $(LIBFILES)
89100
$(CXX) $(CXXFLAGS) -o jsoncheck $(LIBFILES) tests/jsoncheck.cpp -I. $(LIBFLAGS)
90101

@@ -115,11 +126,9 @@ parseandstatcompetition: benchmark/parseandstatcompetition.cpp $(HEADERS) $(LIBF
115126
distinctuseridcompetition: benchmark/distinctuseridcompetition.cpp $(HEADERS) $(LIBFILES)
116127
$(CXX) $(CXXFLAGS) -o distinctuseridcompetition $(LIBFILES) benchmark/distinctuseridcompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE)
117128

118-
119-
parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES) #$(EXTRAOBJECTS)
129+
parsingcompetition: benchmark/parsingcompetition.cpp $(HEADERS) $(LIBFILES)
120130
$(CXX) $(CXXFLAGS) -o parsingcompetition $(LIBFILES) benchmark/parsingcompetition.cpp -I. $(LIBFLAGS) $(COREDEPSINCLUDE)
121-
#$(EXTRADEPSINCLUDE)
122-
#$(EXTRAOBJECTS)
131+
123132

124133
allparserscheckfile: tests/allparserscheckfile.cpp $(HEADERS) $(LIBFILES) $(EXTRAOBJECTS)
125134
$(CXX) $(CXXFLAGS) -o allparserscheckfile $(LIBFILES) tests/allparserscheckfile.cpp $(EXTRAOBJECTS) -I. $(LIBFLAGS) $(COREDEPSINCLUDE) $(EXTRADEPSINCLUDE)
@@ -132,7 +141,7 @@ cppcheck:
132141

133142

134143
clean:
135-
rm -f $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES)
144+
rm -f $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)
136145

137146
cleandist:
138-
rm -f $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES)
147+
rm -f $(EXTRAOBJECTS) $(MAINEXECUTABLES) $(EXTRA_EXECUTABLES) $(TESTEXECUTABLES) $(COMPARISONEXECUTABLES) $(SUPPLEMENTARYEXECUTABLES)

‎include/simdjson/numberparsing.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,10 @@ static really_inline bool parse_number(const u8 *const buf,
340340
ParsedJson &pj,
341341
const u32 offset,
342342
bool found_minus) {
343+
#ifdef SIMDJSON_SKIPNUMBERPARSING // for performance analysis, it is sometimes useful to skip parsing
344+
pj.write_tape_s64(0); // always write zero
345+
return true; // always succeeds
346+
#else
343347
const char *p = (const char *)(buf + offset);
344348
bool negative = false;
345349
if (found_minus) {
@@ -493,4 +497,5 @@ static really_inline bool parse_number(const u8 *const buf,
493497
#endif
494498
}
495499
return is_structural_or_whitespace(*p);
500+
#endif // SIMDJSON_SKIPNUMBERPARSING
496501
}

‎include/simdjson/stringparsing.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,10 @@ really_inline bool handle_unicode_codepoint(const u8 **src_ptr, u8 **dst_ptr) {
6161
WARN_UNUSED
6262
really_inline bool parse_string(const u8 *buf, UNUSED size_t len,
6363
ParsedJson &pj, UNUSED const u32 depth, u32 offset) {
64-
using namespace std;
64+
#ifdef SIMDJSON_SKIPSTRINGPARSING // for performance analysis, it is sometimes useful to skip parsing
65+
pj.write_tape(0, '"');// don't bother with the string parsing at all
66+
return true; // always succeeds
67+
#else
6568
const u8 *src = &buf[offset + 1]; // we know that buf at offset is a "
6669
u8 *dst = pj.current_string_buf_loc;
6770
#ifdef JSON_TEST_STRINGS // for unit testing
@@ -195,6 +198,7 @@ really_inline bool parse_string(const u8 *buf, UNUSED size_t len,
195198
}
196199
// can't be reached
197200
return true;
201+
#endif // SIMDJSON_SKIPSTRINGPARSING
198202
}
199203

200204

‎scripts/plotparse.sh‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,35 @@ if [ "$os" = "Linux" ]; then
1313
echo "You are using linux."
1414
echo "We are going to just parse using simdjson, and collect perf stats."
1515

16-
make parse
16+
make parse parse_noutf8validation parse_nonumberparsing parse_nostringparsing
1717
myfile=$plotdirectory"/parselinuxtable.txt"
1818
echo $myfile
1919
echo "" > $myfile
20+
21+
myfile_noutf8validation=$plotdirectory"/parselinuxtable_noutf8validation.txt"
22+
echo $myfile_noutf8validation
23+
echo "" > $myfile_noutf8validation
24+
25+
myfile=$plotdirectory"/parselinuxtable_nonumberparsing.txt"
26+
echo $myfile_nonumberparsing
27+
echo "" > $myfile_nonumberparsing
28+
29+
myfile=$plotdirectory"/parselinuxtable_nostringparsing.txt"
30+
echo $myfile_nostringparsing
31+
echo "" > $myfile_nostringparsing
32+
33+
2034
for i in $SCRIPTPATH/../jsonexamples/*.json; do
2135
[ -f "$i" ] || break
2236
echo $i
2337
$SCRIPTPATH/../parse -t "$i" >> "$myfile"
38+
$SCRIPTPATH/../parse_noutf8validation -t "$i" >> "$myfile_noutf8validation"
39+
$SCRIPTPATH/../parse_nonumberparsing -t "$i" >> "$myfile_nonumberparsing"
40+
$SCRIPTPATH/../parse_nostringparsing -t "$i" >> "$myfile_nostringparsing"
2441
done
42+
paste $myfile $myfile_noutf8validation $myfile_nonumberparsing $myfile_nostringparsing > $myfile.tmp
43+
mv $myfile.tmp $myfile
44+
rm $myfile_noutf8validation $myfile_nonumberparsing $myfile_nostringparsing
2545
gnuplot -e "filename='$myfile';name='$plotdirectory/stackedperf.pdf'" $SCRIPTPATH/stackbar.gnuplot
2646
fi
2747

‎scripts/stackbar.gnuplot‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,29 @@ set ytics nomirror
2121

2222
set yrange [0:]
2323

24-
set key right
24+
#set key right
25+
set key outside
2526
set style data histograms
2627
set style histogram rowstacked
2728
set xtic rotate by 300 scale 1
2829

2930
set style line 1 lt rgb "#A00000" lw 1 pt 1 ps 1
3031
set style line 2 lt rgb "#00A000" lw 1 pt 1 ps 1
3132
set style line 3 lt rgb "#5060D0" lw 1 pt 1 ps 1
32-
set style line 4 lt rgb "#FF1493" lw 1 pt 1 ps 1
33+
set style line 4 lt rgb "red" lw 1 pt 1 ps 1
34+
set style line 5 lt rgb "#808000" lw 1 pt 1 ps 1
35+
set style line 6 lt rgb "#00008B" lw 1 pt 1 ps 1
36+
set style line 7 lt rgb "black" lw 1 pt 1 ps 1
37+
set style line 8 lt rgb "blue" lw 1 pt 1 ps 1
38+
set style line 9 lt rgb "violet" lw 1 pt 1 ps 1
3339

34-
plot filename using 3 t "stage 1" ls 2, '' using 4 t "stage 2" ls 3, '' using 5:xtic(1) t "stage 3" ls 1
40+
# plot filename using 3 t "stage 1" ls 2, '' using 4 t "stage 2" ls 3, '' using 5:xtic(1) t "stage 3" ls 1
41+
plot filename using 8 t "stage 1 without utf8 validation" ls 1, '' using ($3-$8) t "utf8 validation (stage 1)" ls 2, '' using 4 t "stage 2" ls 3, '' using ($20 + $15 - $5) t "stage 3 (no number or string)" ls 4, '' using ($5 - $20) t "string parsing (stage 3)" ls 5, '' using ($5 - $15):xtic(1) t "number parsing (stage 3)" ls 6
42+
43+
44+
45+
# 1, 2 mem, 3 st1, 4 st2, 5 st3
46+
# 6, 7 mem, 8 st1, 9 st2, 10 st3 // noutf8
47+
# 11, 12 mem, 13 st1, 14 st2, 15 st3 // nonumber
48+
# 16, 17 mem, 18 st1, 19 st2, 20 st3 // nostring
49+
# string: $5 - $20 , number $5 - $15, no string no number $5 - ($5 - $20) - ($5 - $15) = $20 + $15 - $5

‎src/stage1_find_marks.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,14 @@
1010
#include "simdjson/common_defs.h"
1111
#include "simdjson/parsedjson.h"
1212

13-
#define UTF8VALIDATE
13+
#ifndef SIMDJSON_SKIPUTF8VALIDATION
14+
#define SIMDJSON_UTF8VALIDATE
15+
#endif
16+
1417
// It seems that many parsers do UTF-8 validation.
1518
// RapidJSON does not do it by default, but a flag
1619
// allows it.
17-
#ifdef UTF8VALIDATE
20+
#ifdef SIMDJSON_UTF8VALIDATE
1821
#include "simdjson/simdutf8check.h"
1922
#endif
2023
using namespace std;
@@ -37,7 +40,7 @@ WARN_UNUSED
3740
cerr << "Your ParsedJson object only supports documents up to "<< pj.bytecapacity << " bytes but you are trying to process " << len << " bytes\n";
3841
return false;
3942
}
40-
#ifdef UTF8VALIDATE
43+
#ifdef SIMDJSON_UTF8VALIDATE
4144
__m256i has_error = _mm256_setzero_si256();
4245
struct avx_processed_utf_bytes previous = {
4346
.rawbytes = _mm256_setzero_si256(),
@@ -78,7 +81,7 @@ WARN_UNUSED
7881
#endif
7982
m256 input_lo = _mm256_loadu_si256((const m256 *)(buf + idx + 0));
8083
m256 input_hi = _mm256_loadu_si256((const m256 *)(buf + idx + 32));
81-
#ifdef UTF8VALIDATE
84+
#ifdef SIMDJSON_UTF8VALIDATE
8285
m256 highbit = _mm256_set1_epi8(0x80);
8386
if((_mm256_testz_si256(_mm256_or_si256(input_lo, input_hi),highbit)) == 1) {
8487
// it is ascii, we just check continuation
@@ -261,7 +264,7 @@ WARN_UNUSED
261264
memcpy(tmpbuf,buf+idx,len - idx);
262265
m256 input_lo = _mm256_loadu_si256((const m256 *)(tmpbuf + 0));
263266
m256 input_hi = _mm256_loadu_si256((const m256 *)(tmpbuf + 32));
264-
#ifdef UTF8VALIDATE
267+
#ifdef SIMDJSON_UTF8VALIDATE
265268
m256 highbit = _mm256_set1_epi8(0x80);
266269
if((_mm256_testz_si256(_mm256_or_si256(input_lo, input_hi),highbit)) == 1) {
267270
// it is ascii, we just check continuation
@@ -402,7 +405,7 @@ WARN_UNUSED
402405
structurals &= ~(quote_bits & ~quote_mask);
403406
*(u64 *)(pj.structurals + idx / 8) = structurals;
404407
}
405-
#ifdef UTF8VALIDATE
408+
#ifdef SIMDJSON_UTF8VALIDATE
406409
return _mm256_testz_si256(has_error, has_error);
407410
#else
408411
return true;

0 commit comments

Comments
 (0)