Skip to content

Commit 6807abf

Browse files
lemirelemire
committed
Made the code safer (at the expense of the memory usage).
1 parent 94ea7ce commit 6807abf

6 files changed

Lines changed: 21 additions & 9 deletions

File tree

‎LIMITATIONS.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
To simplify the engineering, we make some assumptions that can be lifted with some effort:
2+
3+
- This library cannot parse JSON document of size 16MB or more.
4+
- We expect the input memory pointer to 256-bit aligned and to be padded (e.g., with spaces) so that it can be read entirely in blocks of 256 bits.

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
.PHONY: clean cleandist
88

9-
CXXFLAGS = -std=c++11 -O2 -march=native -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux -Idependencies/double-conversion -Idependencies/rapidjson/include -Ldependencies/double-conversion/release
9+
CXXFLAGS = -std=c++11 -g2 -O2 -march=native -Wall -Wextra -Wshadow -Iinclude -Ibenchmark/linux -Idependencies/double-conversion -Idependencies/rapidjson/include -Ldependencies/double-conversion/release
1010
LIBFLAGS = -ldouble-conversion
1111

1212
EXECUTABLES=parse jsoncheck minifiercompetition parsingcompetition

‎include/jsonparser/jsonioutil.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,12 @@
99

1010
#include "common_defs.h"
1111

12+
// load a file in memory...
1213
// get a corpus; pad out to cache line so we can always use SIMD
1314
// throws exceptions in case of failure
15+
// first element of the pair is a string (null terminated)
16+
// whereas the second element is the length.
17+
// caller is responsible to free (free std::pair<u8 *, size_t>.first)
1418
std::pair<u8 *, size_t> get_corpus(std::string filename);
1519

1620
#endif

‎include/jsonparser/simdjson_internal.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#endif
1010

1111
#include <iostream>
12+
#define MAX_JSON_BYTES 0xFFFFFF
1213

1314
const u32 MAX_DEPTH = 256;
1415
const u32 DEPTH_SAFETY_MARGIN = 32; // should be power-of-2 as we check this
@@ -28,9 +29,9 @@ struct ParsedJson {
2829
// grossly overprovisioned
2930
u64 tape[MAX_TAPE];
3031
u32 tape_locs[MAX_DEPTH];
31-
u8 string_buf[512 * 1024];
32+
u8 string_buf[MAX_JSON_BYTES];
3233
u8 *current_string_buf_loc;
33-
u8 number_buf[512 * 1024]; // holds either doubles or longs, really
34+
u8 number_buf[MAX_JSON_BYTES * 4]; // holds either doubles or longs, really
3435
u8 *current_number_buf_loc;
3536
};
3637

‎src/jsonioutil.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,16 @@ std::pair<u8 *, size_t> get_corpus(std::string filename) {
55
if (is) {
66
std::stringstream buffer;
77
buffer << is.rdbuf();
8-
size_t length = buffer.str().size();
8+
size_t length = buffer.str().size(); // +1 for null
99
char *aligned_buffer;
10-
if (posix_memalign((void **)&aligned_buffer, 64, ROUNDUP_N(length, 64))) {
10+
size_t paddedlength = ROUNDUP_N(length, 64);
11+
if (posix_memalign((void **)&aligned_buffer, 64, paddedlength + 1)) {
1112
throw std::runtime_error("Could not allocate sufficient memory");
1213
};
13-
memset(aligned_buffer, 0x20, ROUNDUP_N(length, 64));
14+
//memset(aligned_buffer, 0x20, ROUNDUP_N(length + 1, 64));
1415
memcpy(aligned_buffer, buffer.str().c_str(), length);
16+
memset(aligned_buffer + length, 0x20, paddedlength - length);
17+
aligned_buffer[paddedlength] = '\0';
1518
is.close();
1619
return std::make_pair((u8 *)aligned_buffer, length);
1720
}

‎src/jsonparser.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
// This structure is meant to be reused from document to document, as needed.
77
// you can use deallocate_ParsedJson to deallocate the memory.
88
ParsedJson *allocate_ParsedJson(size_t len) {
9-
if (len > 0xffffff) {
10-
std::cerr << "Currently only support JSON files < 16MB, requested length: "
9+
if (len > MAX_JSON_BYTES) {
10+
std::cerr << "Currently only support JSON files having up to "<<MAX_JSON_BYTES<<" bytes, requested length: "
1111
<< len << std::endl;
1212
return NULL;
1313
}
@@ -40,7 +40,7 @@ void deallocate_ParsedJson(ParsedJson *pj_ptr) {
4040
if (pj_ptr == NULL)
4141
return;
4242
delete[] pj_ptr->structural_indexes;
43-
delete[] pj_ptr->structurals;
43+
free(pj_ptr->structurals);
4444
delete pj_ptr;
4545
}
4646

0 commit comments

Comments
 (0)