Merge pull request simdjson#729 from simdjson/jkeiser/cmake-amalgamate

jkeiser · web-flow · commit a116e68a47e5 · 2020-04-22T14:16:10.000-07:00
Add amalgamation support to cmake
diff --git a/.dockerignore b/.dockerignore
@@ -1,7 +1,7 @@
 *
 !.git
 !Makefile
-!amalgamation.sh
+!amalgamate.sh
 !benchmark
 !dependencies
 !include
diff --git a/.drone.yml b/.drone.yml
@@ -47,7 +47,7 @@ platform:
 steps:
 - name: build
   image: gcc:8
-  commands: [ make, make amalgamate ]
+  commands: [ make, make amalgamate_test ]
 ---
 kind: pipeline
 name: x64-slowtests
@@ -87,7 +87,7 @@ steps:
   image: gcc:8
   environment:
     EXTRA_FLAGS: -fno-exceptions
-  commands: [ make, make amalgamate ]
+  commands: [ make, make amalgamate_test ]
 ---
 kind: pipeline
 name: x64-noexceptions-slowtests
@@ -139,7 +139,7 @@ platform:
 steps:
 - name: build
   image: gcc:8
-  commands: [ make, make amalgamate ]
+  commands: [ make, make amalgamate_test ]
 ---
 kind: pipeline
 name: arm64-slowtests
diff --git a/.gitignore b/.gitignore
@@ -133,6 +133,11 @@ objs
 /tests/allparserscheckfile
 /tests/basictests
 /tests/checkimplementation
+/tests/compilation_failure_tests/dangling_parser_load_should_compile
+/tests/compilation_failure_tests/dangling_parser_parse_padstring_should_compile
+/tests/compilation_failure_tests/dangling_parser_parse_stdstring_should_compile
+/tests/compilation_failure_tests/dangling_parser_parse_uchar_should_compile
+/tests/compilation_failure_tests/dangling_parser_parse_uint8_should_compile
 /tests/compilation_failure_tests/example_compiletest_should_compile
 /tests/errortests
 /tests/extracting_values_example
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -138,6 +138,15 @@ if (NOT MSVC)
   add_custom_target(simdjson-user-cmakecache ALL DEPENDS ${SIMDJSON_USER_CMAKECACHE})
 endif()
 
+#
+# Set up test data
+#
+enable_testing()
+add_subdirectory(jsonchecker)
+add_subdirectory(jsonexamples)
+add_library(test-data INTERFACE)
+target_link_libraries(test-data INTERFACE jsonchecker-data jsonexamples-data)
+
 #
 # Create the top level simdjson library (must be done at this level to use both src/ and include/
 # directories) and tools
@@ -146,16 +155,11 @@ add_subdirectory(include)
 add_subdirectory(src)
 add_subdirectory(windows)
 add_subdirectory(tools)
+add_subdirectory(singleheader)
 
 #
 # Compile tools / tests / benchmarks
 #
-enable_testing()
-
-add_library(test-data INTERFACE)
-target_compile_definitions(test-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonchecker/")
-target_compile_definitions(test-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/")
-set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/jsonexamples/twitter.json)
 
 add_subdirectory(dependencies)
 add_subdirectory(tests)
diff --git a/HACKING.md b/HACKING.md
@@ -37,7 +37,7 @@ Other important files and directories:
 * **.drone.yml:** Definitions for Drone CI.
 * **.appveyor.yml:** Definitions for Appveyor CI (Windows).
 * **.circleci:** Definitions for Circle CI.
-* **amalgamation.sh:** Generates singleheader/simdjson.h and singleheader/simdjson.cpp for release.
+* **amalgamate.sh:** Generates singleheader/simdjson.h and singleheader/simdjson.cpp for release.
 * **benchmark:** This is where we do benchmarking. Benchmarking is core to every change we make; the
   cardinal rule is don't regress performance without knowing exactly why, and what you're trading
   for it. If you're not sure what else to do to check your performance, this is always a good start:
@@ -73,7 +73,7 @@ you can regenerate them by running this at the top level:
 make amalgamate
 ```
 
-The amalgamator is at `amalgamation.sh` at the top level. It generates singleheader/simdjson.h by
+The amalgamator is at `amalgamate.sh` at the top level. It generates singleheader/simdjson.h by
 reading through include/simdjson.h, copy/pasting each header file into the amalgamated file at the
 point it gets included (but only once per header). singleheader/simdjson.cpp is generated from
 src/simdjson.cpp the same way, except files under generic/ may be included and copy/pasted multiple
diff --git a/Makefile b/Makefile
@@ -178,13 +178,16 @@ quicktests: run_basictests run_quickstart readme_examples readme_examples_noexce
 slowtests: run_testjson2json_sh run_issue150_sh
 
 amalgamate:
-	./amalgamation.sh
+	singleheader/amalgamate.sh
 
-singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamation_demo.cpp: amalgamation.sh src/simdjson.cpp $(SRCHEADERS) $(INCLUDEHEADERS)
-	./amalgamation.sh
+singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp: singleheader/amalgamate.sh src/simdjson.cpp $(SRCHEADERS) $(INCLUDEHEADERS)
+	singleheader/amalgamate.sh
 
-singleheader/demo: singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamation_demo.cpp
-	$(CXX) $(CXXFLAGS) -o singleheader/demo singleheader/amalgamation_demo.cpp -Isingleheader
+singleheader/demo: singleheader/simdjson.h singleheader/simdjson.cpp singleheader/amalgamate_demo.cpp
+	$(CXX) $(CXXFLAGS) -o singleheader/demo singleheader/amalgamate_demo.cpp -Isingleheader
+
+amalgamate_test: singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson
+	singleheader/demo jsonexamples/twitter.json jsonexamples/amazon_cellphones.ndjson
 
 submodules:
 	-git submodule update --init --recursive
diff --git a/jsonchecker/CMakeLists.txt b/jsonchecker/CMakeLists.txt
@@ -0,0 +1,3 @@
+set(SIMDJSON_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
+add_library(jsonchecker-data INTERFACE)
+target_compile_definitions(jsonchecker-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
diff --git a/jsonexamples/CMakeLists.txt b/jsonexamples/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(SIMDJSON_BENCHMARK_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)
+set(EXAMPLE_JSON ${CMAKE_CURRENT_SOURCE_DIR}/twitter.json PARENT_SCOPE)
+set(EXAMPLE_NDJSON ${CMAKE_CURRENT_SOURCE_DIR}/amazon_cellphones.ndjson PARENT_SCOPE)
+add_library(jsonexamples-data INTERFACE)
+target_compile_definitions(jsonexamples-data INTERFACE SIMDJSON_BENCHMARK_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")
diff --git a/singleheader/CMakeLists.txt b/singleheader/CMakeLists.txt
@@ -0,0 +1,21 @@
+#
+# Amalgamation
+#
+if (NOT MSVC)
+  set(SINGLEHEADER_FILES simdjson.h simdjson.cpp amalgamate_demo.cpp README.md)
+  add_custom_command(
+    OUTPUT ${SINGLEHEADER_FILES}
+    COMMAND ${CMAKE_COMMAND} -E env
+      AMALGAMATE_SOURCE_PATH=${PROJECT_SOURCE_DIR}/src
+      AMALGAMATE_INPUT_PATH=${PROJECT_SOURCE_DIR}/include
+      AMALGAMATE_OUTPUT_PATH=${CMAKE_CURRENT_BINARY_DIR}
+      ${CMAKE_CURRENT_SOURCE_DIR}/amalgamate.sh
+    DEPENDS simdjson-source amalgamate.sh
+  )
+
+  add_custom_target(amalgamate DEPENDS ${SINGLEHEADER_FILES})
+
+  add_executable(amalgamate_demo amalgamate_demo.cpp)
+  target_link_libraries(amalgamate_demo simdjson-include-source)
+  add_test(amalgamate_demo amalgamate_demo ${EXAMPLE_JSON} ${EXAMPLE_NDJSON})
+endif()
diff --git a/singleheader/README.md b/singleheader/README.md
@@ -1 +1,2 @@
-c++ -O3 -std=c++17 -pthread -o amalgamation_demo amalgamation_demo.cpp  && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
+Try :
+c++ -O3 -std=c++17 -pthread -o amalgamate_demo amalgamate_demo.cpp  && ./amalgamate_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
diff --git a/singleheader/amalgamate.sh b/singleheader/amalgamate.sh
@@ -3,16 +3,16 @@
 # Generates an "amalgamation build" for roaring. Inspired by similar
 # script used by whefs.
 ########################################################################
+set -e
+
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
 
 echo "We are about to amalgamate all simdjson files into one source file. "
 echo "See https://www.sqlite.org/amalgamation.html and https://en.wikipedia.org/wiki/Single_Compilation_Unit for rationale. "
 
-AMAL_H="simdjson.h"
-AMAL_C="simdjson.cpp"
-
-SRCPATH="$SCRIPTPATH/src"
-INCLUDEPATH="$SCRIPTPATH/include"
+if [ -z "$AMALGAMATE_SOURCE_PATH" ]; then AMALGAMATE_SOURCE_PATH="$SCRIPTPATH/../src"; fi
+if [ -z "$AMALGAMATE_INCLUDE_PATH" ]; then AMALGAMATE_INCLUDE_PATH="$SCRIPTPATH/../include"; fi
+if [ -z "$AMALGAMATE_OUTPUT_PATH" ]; then AMALGAMATE_OUTPUT_PATH="$SCRIPTPATH"; fi
 
 # this list excludes the "src/generic headers"
 ALLCFILES="
@@ -27,33 +27,33 @@ simdjson.h
 found_includes=()
 
 for file in ${ALLCFILES}; do
-    test -e "$SRCPATH/$file" && continue
-    echo "FATAL: source file [$SRCPATH/$file] not found."
+    test -e "$AMALGAMATE_SOURCE_PATH/$file" && continue
+    echo "FATAL: source file [$AMALGAMATE_SOURCE_PATH/$file] not found."
     exit 127
 done
 
 for file in ${ALLCHEADERS}; do
-    test -e "$INCLUDEPATH/$file" && continue
-    echo "FATAL: source file [$INCLUDEPATH/$file] not found."
+    test -e "$AMALGAMATE_INCLUDE_PATH/$file" && continue
+    echo "FATAL: source file [$AMALGAMATE_INCLUDE_PATH/$file] not found."
     exit 127
 done
 
 function doinclude()
 {
     file=$1
     line="${@:2}"
-    if [ -f $INCLUDEPATH/$file ]; then
+    if [ -f $AMALGAMATE_INCLUDE_PATH/$file ]; then
         if [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
             found_includes+=("$file")
-            dofile $INCLUDEPATH/$file
+            dofile $AMALGAMATE_INCLUDE_PATH $file
         fi;
-    elif [ -f $SRCPATH/$file ]; then
+    elif [ -f $AMALGAMATE_SOURCE_PATH/$file ]; then
         # generic includes are included multiple times
         if [[ "${file}" == *'generic/'*'.h' ]]; then
-            dofile $SRCPATH/$file
+            dofile $AMALGAMATE_SOURCE_PATH $file
         elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
             found_includes+=("$file")
-            dofile $SRCPATH/$file
+            dofile $AMALGAMATE_SOURCE_PATH $file
         else
             echo "/* $file already included: $line */"
         fi
@@ -65,9 +65,9 @@ function doinclude()
 
 function dofile()
 {
+    file="$1/$2"
     # Last lines are always ignored. Files should end by an empty lines.
-    RELFILE=${1#"$SCRIPTPATH/"}
-    echo "/* begin file $RELFILE */"
+    echo "/* begin file ${2} */"
     # echo "#line 8 \"$1\"" ## redefining the line/file is not nearly as useful as it sounds for debugging. It breaks IDEs.
     while IFS= read -r line || [ -n "$line" ];
     do
@@ -84,23 +84,31 @@ function dofile()
             # Otherwise we simply copy the line
             echo "$line"
         fi
-    done < "$1"
+    done < "$file"
     echo "/* end file $RELFILE */"
 }
+
 timestamp=$(date)
+mkdir -p $AMALGAMATE_OUTPUT_PATH
+
+AMAL_H="${AMALGAMATE_OUTPUT_PATH}/simdjson.h"
+AMAL_C="${AMALGAMATE_OUTPUT_PATH}/simdjson.cpp"
+DEMOCPP="${AMALGAMATE_OUTPUT_PATH}/amalgamate_demo.cpp"
+README="$AMALGAMATE_OUTPUT_PATH/README.md"
+
 echo "Creating ${AMAL_H}..."
-echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_H}"
+echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_H}
 {
     for h in ${ALLCHEADERS}; do
         doinclude $h "ERROR $h not found"
     done
-} >> "${AMAL_H}"
+} >> ${AMAL_H}
 
 
 echo "Creating ${AMAL_C}..."
-echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
+echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_C}
 {
-    echo "#include \"${AMAL_H}\""
+    echo "#include \"simdjson.h\""
 
     echo ""
     echo "/* used for http://dmalloc.com/ Dmalloc - Debug Malloc Library */"
@@ -110,14 +118,13 @@ echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${AMAL_C}"
     echo ""
 
     for file in ${ALLCFILES}; do
-        dofile "$SRCPATH/$file"
+        dofile $AMALGAMATE_SOURCE_PATH $file
     done
-} >> "${AMAL_C}"
+} >> ${AMAL_C}
 
 
-DEMOCPP="amalgamation_demo.cpp"
 echo "Creating ${DEMOCPP}..."
-echo "/* auto-generated on ${timestamp}. Do not edit! */" > "${DEMOCPP}"
+echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${DEMOCPP}
 cat <<< '
 #include <iostream>
 #include "simdjson.h"
@@ -128,11 +135,14 @@ int main(int argc, char *argv[]) {
   }
   const char * filename = argv[1];
   simdjson::dom::parser parser;
-  auto [doc, error] = parser.load(filename); // do the parsing
+  simdjson::error_code error;
+  UNUSED simdjson::dom::element elem;
+  parser.load(filename).tie(elem, error); // do the parsing
   if (error) {
     std::cout << "parse failed" << std::endl;
     std::cout << "error code: " << error << std::endl;
     std::cout << error << std::endl;
+    return EXIT_FAILURE;
   } else {
     std::cout << "parse valid" << std::endl;
   }
@@ -149,34 +159,31 @@ int main(int argc, char *argv[]) {
     std::cout << "parse_many failed" << std::endl;
     std::cout << "error code: " << error << std::endl;
     std::cout << error << std::endl;
+    return EXIT_FAILURE;
   } else {
     std::cout << "parse_many valid" << std::endl;
   }
   return EXIT_SUCCESS;
 }
-' >>  "${DEMOCPP}"
+' >> ${DEMOCPP}
 
-echo "Done with all files generation. "
+CPPBIN=$(basename ${DEMOCPP} .cpp)
 
-echo "Files have been written to  directory: $PWD "
-ls -la ${AMAL_C} ${AMAL_H}  ${DEMOCPP}
-
-echo "Giving final instructions:"
+echo "Try :" > ${README}
+echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP##*/}  && ./${CPPBIN##*/} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" >> ${README}
 
+echo "Done with all files generation."
 
-CPPBIN=${DEMOCPP%%.*}
+echo "Files have been written to directory: ${AMALGAMATE_OUTPUT_PATH}/"
+ls -la ${AMAL_C} ${AMAL_H} ${DEMOCPP} ${README}
 
-echo "Try :"
-echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson"
-
-SINGLEHDR=$SCRIPTPATH/singleheader
-echo "Copying files to $SCRIPTPATH/singleheader "
-mkdir -p $SINGLEHDR
-echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" > $SINGLEHDR/README.md
-cp ${AMAL_C} ${AMAL_H}  ${DEMOCPP} $SINGLEHDR
-ls $SINGLEHDR
+#
+# Instructions to create demo
+#
+echo ""
+echo "Giving final instructions:"
 
-cd $SINGLEHDR && c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP}  && ./${CPPBIN} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson
+cat ${README}
 
 lowercase(){
     echo "$1" | tr 'A-Z' 'a-z'
diff --git a/singleheader/amalgamate_demo.cpp b/singleheader/amalgamate_demo.cpp
@@ -1,4 +1,4 @@
-/* auto-generated on Sun Apr 19 11:36:20 PDT 2020. Do not edit! */
+/* auto-generated on Mon Apr 20 11:05:12 PDT 2020. Do not edit! */
 
 #include <iostream>
 #include "simdjson.h"
@@ -9,11 +9,14 @@ int main(int argc, char *argv[]) {
   }
   const char * filename = argv[1];
   simdjson::dom::parser parser;
-  auto [doc, error] = parser.load(filename); // do the parsing
+  simdjson::error_code error;
+  UNUSED simdjson::dom::element elem;
+  parser.load(filename).tie(elem, error); // do the parsing
   if (error) {
     std::cout << "parse failed" << std::endl;
     std::cout << "error code: " << error << std::endl;
     std::cout << error << std::endl;
+    return EXIT_FAILURE;
   } else {
     std::cout << "parse valid" << std::endl;
   }
@@ -30,6 +33,7 @@ int main(int argc, char *argv[]) {
     std::cout << "parse_many failed" << std::endl;
     std::cout << "error code: " << error << std::endl;
     std::cout << error << std::endl;
+    return EXIT_FAILURE;
   } else {
     std::cout << "parse_many valid" << std::endl;
   }
diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp
diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
diff --git a/tools/release.py b/tools/release.py

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`*`
`2`	`2`	`!.git`
`3`	`3`	`!Makefile`
`4`		`-!amalgamation.sh`
	`4`	`+!amalgamate.sh`
`5`	`5`	`!benchmark`
`6`	`6`	`!dependencies`
`7`	`7`	`!include`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+set(SIMDJSON_TEST_DATA_DIR ${CMAKE_CURRENT_SOURCE_DIR} PARENT_SCOPE)`
	`2`	`+add_library(jsonchecker-data INTERFACE)`
	`3`	`+target_compile_definitions(jsonchecker-data INTERFACE SIMDJSON_TEST_DATA_DIR="${CMAKE_CURRENT_SOURCE_DIR}/")`
Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
`1`		`-c++ -O3 -std=c++17 -pthread -o amalgamation_demo amalgamation_demo.cpp && ./amalgamation_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson`
	`1`	`+Try :`
	`2`	`+c++ -O3 -std=c++17 -pthread -o amalgamate_demo amalgamate_demo.cpp && ./amalgamate_demo ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson`