Skip to content

Commit 3b78300

Browse files
lemirelemire
committed
Cleaning
1 parent 5a160b0 commit 3b78300

2 files changed

Lines changed: 118 additions & 88 deletions

File tree

‎benchmark/parse.cpp‎

Lines changed: 112 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1+
#include <assert.h>
2+
#include <ctype.h>
3+
#include <dirent.h>
4+
#include <inttypes.h>
15
#include <stdbool.h>
26
#include <stdio.h>
37
#include <stdlib.h>
48
#include <string.h>
59
#include <unistd.h>
610
#include <x86intrin.h>
7-
#include <ctype.h>
8-
#include <assert.h>
9-
#include <dirent.h>
10-
#include <inttypes.h>
1111

1212
#include <algorithm>
1313
#include <chrono>
@@ -21,15 +21,14 @@
2121
#include <string>
2222
#include <vector>
2323

24-
2524
#include "linux-perf-events.h"
2625
#ifdef __linux__
2726
#include <libgen.h>
2827
#endif
2928
//#define DEBUG
3029
#include "simdjson/common_defs.h"
31-
#include "simdjson/jsonparser.h"
3230
#include "simdjson/jsonioutil.h"
31+
#include "simdjson/jsonparser.h"
3332
#include "simdjson/parsedjson.h"
3433
#include "simdjson/stage1_find_marks.h"
3534
#include "simdjson/stage2_flatten.h"
@@ -45,55 +44,58 @@ int main(int argc, char *argv[]) {
4544

4645
int c;
4746

48-
while ((c = getopt (argc, argv, "1vdt")) != -1)
49-
switch (c)
50-
{
51-
case 't':
52-
justdata = true;
53-
break;
54-
case 'v':
55-
verbose = true;
56-
break;
57-
case 'd':
58-
dump = true;
59-
break;
60-
case 'j':
61-
jsonoutput = true;
62-
break;
63-
case '1':
64-
forceoneiteration = true;
65-
break;
66-
default:
67-
abort ();
68-
}
47+
while ((c = getopt(argc, argv, "1vdt")) != -1)
48+
switch (c) {
49+
case 't':
50+
justdata = true;
51+
break;
52+
case 'v':
53+
verbose = true;
54+
break;
55+
case 'd':
56+
dump = true;
57+
break;
58+
case 'j':
59+
jsonoutput = true;
60+
break;
61+
case '1':
62+
forceoneiteration = true;
63+
break;
64+
default:
65+
abort();
66+
}
6967
if (optind >= argc) {
7068
cerr << "Usage: " << argv[0] << " <jsonfile>" << endl;
7169
exit(1);
7270
}
73-
const char * filename = argv[optind];
74-
if(optind + 1 < argc) {
75-
cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
71+
const char *filename = argv[optind];
72+
if (optind + 1 < argc) {
73+
cerr << "warning: ignoring everything after " << argv[optind + 1] << endl;
7674
}
77-
if(verbose) cout << "[verbose] loading " << filename << endl;
75+
if (verbose)
76+
cout << "[verbose] loading " << filename << endl;
7877
std::string_view p;
7978
try {
8079
p = get_corpus(filename);
81-
} catch (const std::exception& e) { // caught by reference to base
80+
} catch (const std::exception &e) { // caught by reference to base
8281
std::cout << "Could not load the file " << filename << std::endl;
8382
return EXIT_FAILURE;
8483
}
85-
if(verbose) cout << "[verbose] loaded " << filename << " ("<< p.size() << " bytes)" << endl;
84+
if (verbose)
85+
cout << "[verbose] loaded " << filename << " (" << p.size() << " bytes)"
86+
<< endl;
8687
#if defined(DEBUG)
8788
const u32 iterations = 1;
8889
#else
89-
const u32 iterations = forceoneiteration ? 1 : ( p.size() < 1 * 1000 * 1000? 1000 : 10);
90+
const u32 iterations =
91+
forceoneiteration ? 1 : (p.size() < 1 * 1000 * 1000 ? 1000 : 10);
9092
#endif
9193
vector<double> res;
9294
res.resize(iterations);
9395

9496
#if !defined(__linux__)
9597
#define SQUASH_COUNTERS
96-
if(justdata) {
98+
if (justdata) {
9799
printf("justdata (-t) flag only works under linux.\n");
98100
}
99101
#endif
@@ -117,13 +119,14 @@ int main(int argc, char *argv[]) {
117119
bool isok = true;
118120

119121
for (u32 i = 0; i < iterations; i++) {
120-
if(verbose) cout << "[verbose] iteration # " << i << endl;
122+
if (verbose)
123+
cout << "[verbose] iteration # " << i << endl;
121124
#ifndef SQUASH_COUNTERS
122125
unified.start();
123126
#endif
124127
ParsedJson pj;
125128
bool allocok = pj.allocateCapacity(p.size());
126-
if(!allocok) {
129+
if (!allocok) {
127130
std::cerr << "failed to allocate memory" << std::endl;
128131
return EXIT_FAILURE;
129132
}
@@ -135,7 +138,8 @@ int main(int argc, char *argv[]) {
135138
cref0 += results[3];
136139
cmis0 += results[4];
137140
#endif
138-
if(verbose) cout << "[verbose] allocated memory for parsed JSON " << endl;
141+
if (verbose)
142+
cout << "[verbose] allocated memory for parsed JSON " << endl;
139143

140144
auto start = std::chrono::steady_clock::now();
141145
#ifndef SQUASH_COUNTERS
@@ -189,65 +193,92 @@ int main(int argc, char *argv[]) {
189193
res[i] = secs.count();
190194
}
191195
ParsedJson pj = build_parsed_json(p); // do the parsing again to get the stats
192-
if( ! pj.isValid() ) {
196+
if (!pj.isValid()) {
193197
std::cerr << "Could not parse. " << std::endl;
194198
return EXIT_FAILURE;
195199
}
196200
#ifndef SQUASH_COUNTERS
197-
if(justdata) {
198-
float cpb0 = (double)cy0 / (iterations * p.size());
199-
float cpb1 = (double)cy1 / (iterations * p.size());
200-
float cpb2 = (double)cy2 / (iterations * p.size());
201-
float cpb3 = (double)cy3 / (iterations * p.size());
201+
unsigned long total = cy0 + cy1 + cy2 + cy3;
202+
if (justdata) {
203+
float cpb0 = (double)cy0 / (iterations * p.size());
204+
float cpb1 = (double)cy1 / (iterations * p.size());
205+
float cpb2 = (double)cy2 / (iterations * p.size());
206+
float cpb3 = (double)cy3 / (iterations * p.size());
202207
float cpbtotal = (double)total / (iterations * p.size());
203-
printf("\"%s\"\t%f\t%f\t%f\t%f\t%f\n", basename(filename), cpb0,cpb1,cpb2,cpb3,cpbtotal);
208+
char *newfile = (char *)malloc(strlen(filename) + 1);
209+
if (newfile == NULL)
210+
return EXIT_FAILURE;
211+
::strcpy(newfile, filename);
212+
char *snewfile = ::basename(newfile);
213+
size_t nl = strlen(snewfile);
214+
for (size_t j = nl - 1; j > 0; j--) {
215+
if (snewfile[j] == '.') {
216+
snewfile[j] = '\0';
217+
break;
218+
}
219+
}
220+
printf("\"%s\"\t%f\t%f\t%f\t%f\t%f\n", snewfile, cpb0, cpb1, cpb2, cpb3,
221+
cpbtotal);
222+
free(newfile);
204223
} else {
205-
printf("number of bytes %ld number of structural chars %u ratio %.3f\n",
206-
p.size(), pj.n_structural_indexes,
207-
(double)pj.n_structural_indexes / p.size());
208-
unsigned long total = cy0 + cy1 + cy2 + cy3;
209-
printf(
210-
"mem alloc instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: %10lu (failure %10lu)\n",
211-
cl0 / iterations, cy0 / iterations, 100. * cy0 / total, (double)cl0 / cy0, mis0/iterations, (double)cy0/mis0, cref1 / iterations, cmis0 / iterations);
212-
printf(" mem alloc runs at %.2f cycles per input byte.\n",
213-
(double)cy0 / (iterations * p.size()));
214-
printf(
215-
"stage 1 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: %10lu (failure %10lu)\n",
216-
cl1 / iterations, cy1 / iterations, 100. * cy1 / total, (double)cl1 / cy1, mis1/iterations, (double)cy1/mis1, cref1 / iterations, cmis1 / iterations);
217-
printf(" stage 1 runs at %.2f cycles per input byte.\n",
218-
(double)cy1 / (iterations * p.size()));
224+
printf("number of bytes %ld number of structural chars %u ratio %.3f\n",
225+
p.size(), pj.n_structural_indexes,
226+
(double)pj.n_structural_indexes / p.size());
227+
printf("mem alloc instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: "
228+
"%.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: "
229+
"%10lu (failure %10lu)\n",
230+
cl0 / iterations, cy0 / iterations, 100. * cy0 / total,
231+
(double)cl0 / cy0, mis0 / iterations, (double)cy0 / mis0,
232+
cref1 / iterations, cmis0 / iterations);
233+
printf(" mem alloc runs at %.2f cycles per input byte.\n",
234+
(double)cy0 / (iterations * p.size()));
235+
printf("stage 1 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: "
236+
"%.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: "
237+
"%10lu (failure %10lu)\n",
238+
cl1 / iterations, cy1 / iterations, 100. * cy1 / total,
239+
(double)cl1 / cy1, mis1 / iterations, (double)cy1 / mis1,
240+
cref1 / iterations, cmis1 / iterations);
241+
printf(" stage 1 runs at %.2f cycles per input byte.\n",
242+
(double)cy1 / (iterations * p.size()));
219243

220-
printf(
221-
"stage 2 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: %10lu (failure %10lu)\n",
222-
cl2 / iterations, cy2 / iterations, 100. * cy2 / total, (double)cl2 / cy2, mis2/iterations, (double)cy2/mis2, cref2 /iterations, cmis2 / iterations);
223-
printf(" stage 2 runs at %.2f cycles per input byte and ",
224-
(double)cy2 / (iterations * p.size()));
225-
printf("%.2f cycles per structural character.\n",
226-
(double)cy2 / (iterations * pj.n_structural_indexes));
244+
printf("stage 2 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: "
245+
"%.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache "
246+
"accesses: %10lu (failure %10lu)\n",
247+
cl2 / iterations, cy2 / iterations, 100. * cy2 / total,
248+
(double)cl2 / cy2, mis2 / iterations, (double)cy2 / mis2,
249+
cref2 / iterations, cmis2 / iterations);
250+
printf(" stage 2 runs at %.2f cycles per input byte and ",
251+
(double)cy2 / (iterations * p.size()));
252+
printf("%.2f cycles per structural character.\n",
253+
(double)cy2 / (iterations * pj.n_structural_indexes));
227254

228-
printf(
229-
"stage 3 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: %.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache accesses: %10lu (failure %10lu)\n",
230-
cl3 / iterations, cy3 /iterations, 100. * cy3 / total, (double)cl3 / cy3, mis3/iterations, (double)cy3/mis3, cref3 / iterations, cmis3 / iterations);
231-
printf(" stage 3 runs at %.2f cycles per input byte and ",
232-
(double)cy3 / (iterations * p.size()));
233-
printf("%.2f cycles per structural character.\n",
234-
(double)cy3 / (iterations * pj.n_structural_indexes));
255+
printf("stage 3 instructions: %10lu cycles: %10lu (%.2f %%) ins/cycles: "
256+
"%.2f mis. branches: %10lu (cycles/mis.branch %.2f) cache "
257+
"accesses: %10lu (failure %10lu)\n",
258+
cl3 / iterations, cy3 / iterations, 100. * cy3 / total,
259+
(double)cl3 / cy3, mis3 / iterations, (double)cy3 / mis3,
260+
cref3 / iterations, cmis3 / iterations);
261+
printf(" stage 3 runs at %.2f cycles per input byte and ",
262+
(double)cy3 / (iterations * p.size()));
263+
printf("%.2f cycles per structural character.\n",
264+
(double)cy3 / (iterations * pj.n_structural_indexes));
235265

236-
printf(" all stages: %.2f cycles per input byte.\n",
237-
(double)total / (iterations * p.size()));
266+
printf(" all stages: %.2f cycles per input byte.\n",
267+
(double)total / (iterations * p.size()));
238268
}
239269
#endif
240270
double min_result = *min_element(res.begin(), res.end());
241-
if(!justdata) cout << "Min: " << min_result << " bytes read: " << p.size()
242-
<< " Gigabytes/second: " << (p.size()) / (min_result * 1000000000.0)
243-
<< "\n";
244-
if(jsonoutput) {
271+
if (!justdata)
272+
cout << "Min: " << min_result << " bytes read: " << p.size()
273+
<< " Gigabytes/second: " << (p.size()) / (min_result * 1000000000.0)
274+
<< "\n";
275+
if (jsonoutput) {
245276
isok = isok && pj.printjson(std::cout);
246277
}
247-
if(dump) {
278+
if (dump) {
248279
isok = isok && pj.dump_raw_tape(std::cout);
249280
}
250-
free((void*)p.data());
281+
free((void *)p.data());
251282
if (!isok) {
252283
printf(" Parsing failed. \n ");
253284
return EXIT_FAILURE;

‎scripts/plotparse.sh‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,15 @@ if [ "$os" = "Linux" ]; then
1414
echo "We are going to just parse using simdjson, and collect perf stats."
1515

1616
make parse
17+
myfile=$plotdirectory"/parselinuxtable.txt"
18+
echo $myfile
19+
echo "" > $myfile
1720
for i in $SCRIPTPATH/../jsonexamples/*.json; do
1821
[ -f "$i" ] || break
1922
echo $i
20-
shortname=$(basename $SCRIPTPATH/$i"justparse.table")
21-
corename=$(basename ${shortname%.*})".pdf"
22-
$SCRIPTPATH/../parse -t $i > $plotdirectory/$shortname
23-
gnuplot -e "filename='$plotdirectory/$shortname';name='$plotdirectory/$corename'" $SCRIPTPATH/bar.gnuplot
24-
rm $plotdirectory/$shortname
25-
echo
23+
$SCRIPTPATH/../parse -t "$i" >> "$myfile"
2624
done
25+
echo $myfile
2726
fi
2827

2928
make parsingcompetition
@@ -76,4 +75,4 @@ for i in $SCRIPTPATH/../jsonexamples/twitter.json; do
7675
echo
7776
done
7877

79-
echo "see results in "$plotdirectory
78+
echo "see results in "$plotdirectory

0 commit comments

Comments
 (0)