Getting ready to document the tape format.

lemire · lemire · commit 779ce184fb7c · 2018-12-18T14:21:22.000-05:00
diff --git a/README.md b/README.md
@@ -100,7 +100,7 @@ To simplify the engineering, we make some assumptions.
 ## Features
 
 - The input string is unmodified. (Parsers like sajson and RapidJSON use the input string as a buffer.)
-- We parse integers and floating-point numbers as separate types which allows us to support large 64-bit integers in [-9223372036854775808,9223372036854775808). Among the parsers  that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson stores integers larger than 2147483648 as floating-point numbers.)
+- We parse integers and floating-point numbers as separate types which allows us to support large 64-bit integers in [-9223372036854775808,9223372036854775808), like a Java `long` or a C/C++ `long long`. Among the parsers  that differentiate between integers and floating-point numbers, not all support 64-bit integers. (For example, sajson rejects JSON files with integers larger than or equal to 2147483648. RapidJSON will parse a file containing an overly long integer like 18446744073709551616 as a floating-point number) When we cannot represent exactly an integer as a signed 64-bit value, we reject the JSON document.
 - We do full UTF-8 validation as part of the parsing. (Parsers like fastjson, gason and dropbox json11 do not do UTF-8 validation.)
 - We fully validate the numbers. (Parsers like gason and ultranjson will accept `[0e+]` as valid JSON.)
 - We validate string content for unescaped characters. (Parsers like fastjson and ultrajson accept unescaped line breaks and tags in strings.)
diff --git a/include/simdjson/numberparsing.h b/include/simdjson/numberparsing.h
@@ -483,7 +483,7 @@ static really_inline bool parse_number(const u8 *const buf,
 #endif
     }
   } else {
-    if (unlikely(digitcount >= 19)) { // this is uncommon!!!
+    if (unlikely(digitcount >= 18)) { // this is uncommon!!!
       return parse_large_integer(buf, pj, offset, 
                                  found_minus);
     }
diff --git a/include/simdjson/parsedjson.h b/include/simdjson/parsedjson.h
@@ -217,19 +217,23 @@ struct ParsedJson {
   bool dump_raw_tape(std::ostream &os) {
     if(!isvalid) return false;
     size_t tapeidx = 0;
-    u64 tape_val = tape[tapeidx++];
+    u64 tape_val = tape[tapeidx];
     u8 type = (tape_val >> 56);
+    os << tapeidx << " : " << type; 
+    tapeidx++;
     size_t howmany = 0;
     if (type == 'r') {
       howmany = tape_val & JSONVALUEMASK;
     } else {
       printf("Error: no starting root node?");
       return false;
     }
+    os << "\t// pointing to " << howmany <<" (right after last node)\n";
+    u64 payload;
     for (; tapeidx < howmany; tapeidx++) {
       os << tapeidx << " : "; 
       tape_val = tape[tapeidx];
-      u64 payload = tape_val & JSONVALUEMASK;
+      payload = tape_val & JSONVALUEMASK;
       type = (tape_val >> 56);
       switch (type) {
       case '"': // we have a string
@@ -261,16 +265,16 @@ struct ParsedJson {
         os << "false\n";
         break;
       case '{': // we have an object
-        os << "{\t// pointing to next tape location " << payload << "\n";
+        os << "{\t// pointing to next tape location " << payload << " (first node after the scope) \n";
         break;
       case '}': // we end an object
-        os << "}\t// pointing to previous tape location " << payload << "\n";
+        os << "}\t// pointing to previous tape location " << payload << " (start of the scope) \n";
         break;
       case '[': // we start an array
-        os << "[\t// pointing to next tape location " << payload << "\n";
+        os << "[\t// pointing to next tape location " << payload << " (first node after the scope) \n";
         break;
       case ']': // we end an array
-        os << "]\t// pointing to previous tape location " << payload << "\n";
+        os << "]\t// pointing to previous tape location " << payload << " (start of the scope) \n";
         break;
       case 'r': // we start and end with the root node
         printf("end of root\n");
@@ -279,6 +283,10 @@ struct ParsedJson {
         return false;
       }
     }
+    tape_val = tape[tapeidx];
+    payload = tape_val & JSONVALUEMASK;
+    type = (tape_val >> 56);
+    os << tapeidx << " : "<< type <<"\t// pointing to " << payload <<" (start root)\n";
     return true;
   }
 
diff --git a/jsonchecker/fail40_s64boverflow.json b/jsonchecker/fail40_s64boverflow.json
@@ -0,0 +1 @@
+9223372036854775808
diff --git a/jsonchecker/pass11.json b/jsonchecker/pass11.json
@@ -0,0 +1 @@
+4611686018427387904
diff --git a/src/stage34_unified.cpp b/src/stage34_unified.cpp
@@ -103,12 +103,25 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
   if (depth > pj.depthcapacity) {
     goto fail;
   }
+
   UPDATE_CHAR();
   switch (c) {
   case '{':
+    pj.containing_scope_offset[depth] = pj.get_current_loc();
+    pj.ret_address[depth] = &&start_continue;
+    depth++;
+    if (depth > pj.depthcapacity) {
+      goto fail;
+    }
     pj.write_tape(0, c); // strangely, moving this to object_begin slows things down
     goto object_begin;
   case '[':
+    pj.containing_scope_offset[depth] = pj.get_current_loc();
+    pj.ret_address[depth] = &&start_continue;
+    depth++;
+    if (depth > pj.depthcapacity) {
+      goto fail;
+    }
     pj.write_tape(0, c);
     goto array_begin;
 #define SIMDJSON_ALLOWANYTHINGINROOT
@@ -216,11 +229,6 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
   default:
     goto fail;
   }
-#ifdef SIMDJSON_ALLOWANYTHINGINROOT
-  depth--; // for fall-through cases (e.g., documents containing just a string)
-  pj.annotate_previousloc(pj.containing_scope_offset[depth],
-                          pj.get_current_loc());
-#endif     // ALLOWANYTHINGINROOT
 start_continue:
   DEBUG_PRINTF("in start_object_close\n");
   // the string might not be NULL terminated.
@@ -465,6 +473,7 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
 
 succeed:
   DEBUG_PRINTF("in succeed, depth = %d \n", depth);
+  depth --;
   if(depth != 0) {
     printf("internal bug\n");
     abort();
@@ -473,6 +482,8 @@ bool unified_machine(const u8 *buf, size_t len, ParsedJson &pj) {
     printf("internal bug\n");
     abort();
   }
+  pj.annotate_previousloc(pj.containing_scope_offset[depth],
+                          pj.get_current_loc());
   pj.write_tape(pj.containing_scope_offset[depth], 'r'); // r is root
 
 

Original file line number	Diff line number	Diff line change
`@@ -483,7 +483,7 @@ static really_inline bool parse_number(const u8 *const buf,`
`483`	`483`	`#endif`
`484`	`484`	`}`
`485`	`485`	`} else {`
`486`		`- if (unlikely(digitcount >= 19)) { // this is uncommon!!!`
	`486`	`+ if (unlikely(digitcount >= 18)) { // this is uncommon!!!`
`487`	`487`	`return parse_large_integer(buf, pj, offset,`
`488`	`488`	`found_minus);`
`489`	`489`	`}`