forked from simdjson/simdjson
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathamalgamate.sh
More file actions
executable file
·192 lines (163 loc) · 5.55 KB
/
amalgamate.sh
File metadata and controls
executable file
·192 lines (163 loc) · 5.55 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
#!/usr/bin/env bash
########################################################################
# Generates an "amalgamation build" for roaring. Inspired by similar
# script used by whefs.
########################################################################
set -e
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
PROJECTPATH=$SCRIPTPATH"/.."
echo "Project at "$PROJECTPATH
echo "We are about to amalgamate all simdjson files into one source file. "
echo "See https://www.sqlite.org/amalgamation.html and https://en.wikipedia.org/wiki/Single_Compilation_Unit for rationale. "
if [ -z "$AMALGAMATE_SOURCE_PATH" ]; then AMALGAMATE_SOURCE_PATH="$SCRIPTPATH/../src"; fi
if [ -z "$AMALGAMATE_INCLUDE_PATH" ]; then AMALGAMATE_INCLUDE_PATH="$SCRIPTPATH/../include"; fi
if [ -z "$AMALGAMATE_OUTPUT_PATH" ]; then AMALGAMATE_OUTPUT_PATH="$SCRIPTPATH"; fi
# this list excludes the "src/generic headers"
ALLCFILES="
simdjson.cpp
"
# order matters
ALLCHEADERS="
simdjson.h
"
found_includes=()
for file in ${ALLCFILES}; do
test -e "$AMALGAMATE_SOURCE_PATH/$file" && continue
echo "FATAL: source file [$AMALGAMATE_SOURCE_PATH/$file] not found."
exit 127
done
for file in ${ALLCHEADERS}; do
test -e "$AMALGAMATE_INCLUDE_PATH/$file" && continue
echo "FATAL: source file [$AMALGAMATE_INCLUDE_PATH/$file] not found."
exit 127
done
function doinclude()
{
file=$1
line="${@:2}"
if [ -f $AMALGAMATE_INCLUDE_PATH/$file ]; then
if [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
found_includes+=("$file")
dofile $AMALGAMATE_INCLUDE_PATH $file
fi
elif [ -f $AMALGAMATE_SOURCE_PATH/$file ]; then
# generic includes are included multiple times
if [[ "${file}" == *'generic/'*'.h' ]]; then
dofile $AMALGAMATE_SOURCE_PATH $file
elif [[ ! " ${found_includes[@]} " =~ " ${file} " ]]; then
found_includes+=("$file")
dofile $AMALGAMATE_SOURCE_PATH $file
else
echo "/* $file already included: $line */"
fi
else
# If we don't recognize it, just emit the #include
echo "$line"
fi
}
function dofile()
{
file="$1/$2"
RELFILE=${file#"$PROJECTPATH/"}
# Last lines are always ignored. Files should end by an empty lines.
echo "/* begin file $RELFILE */"
# echo "#line 8 \"$1\"" ## redefining the line/file is not nearly as useful as it sounds for debugging. It breaks IDEs.
while IFS= read -r line || [ -n "$line" ];
do
if [[ "${line}" == '#include "'*'"'* ]]; then
file=$(echo $line| cut -d'"' -f 2)
# include all from simdjson.cpp except simdjson.h
if [ "${file}" == "simdjson.h" ] && [ "${2}" == "simdjson.cpp" ]; then
echo "$line"
continue
fi
if [[ "${file}" == '../'* ]]; then
file=$(echo $file| cut -d'/' -f 2-)
fi
# we explicitly include simdjson headers, one time each (unless they are generic, in which case multiple times is fine)
doinclude $file $line
else
# Otherwise we simply copy the line
echo "$line"
fi
done < "$file"
echo "/* end file $RELFILE */"
}
timestamp=$(date)
mkdir -p $AMALGAMATE_OUTPUT_PATH
AMAL_H="${AMALGAMATE_OUTPUT_PATH}/simdjson.h"
AMAL_C="${AMALGAMATE_OUTPUT_PATH}/simdjson.cpp"
DEMOCPP="${AMALGAMATE_OUTPUT_PATH}/amalgamate_demo.cpp"
README="$AMALGAMATE_OUTPUT_PATH/README.md"
echo "Creating ${AMAL_H}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_H}
{
for h in ${ALLCHEADERS}; do
doinclude $h "ERROR $h not found"
done
} >> ${AMAL_H}
echo "Creating ${AMAL_C}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${AMAL_C}
{
for file in ${ALLCFILES}; do
dofile $AMALGAMATE_SOURCE_PATH $file
done
} >> ${AMAL_C}
echo "Creating ${DEMOCPP}..."
echo "/* auto-generated on ${timestamp}. Do not edit! */" > ${DEMOCPP}
cat <<< '
#include <iostream>
#include "simdjson.h"
#include "simdjson.cpp"
int main(int argc, char *argv[]) {
if(argc < 2) {
std::cerr << "Please specify at least one file name. " << std::endl;
}
const char * filename = argv[1];
simdjson::dom::parser parser;
simdjson::error_code error;
UNUSED simdjson::dom::element elem;
parser.load(filename).tie(elem, error); // do the parsing
if (error) {
std::cout << "parse failed" << std::endl;
std::cout << "error code: " << error << std::endl;
std::cout << error << std::endl;
return EXIT_FAILURE;
} else {
std::cout << "parse valid" << std::endl;
}
if(argc == 2) {
return EXIT_SUCCESS;
}
// parse_many
const char * filename2 = argv[2];
for (auto result : parser.load_many(filename2)) {
error = result.error();
}
if (error) {
std::cout << "parse_many failed" << std::endl;
std::cout << "error code: " << error << std::endl;
std::cout << error << std::endl;
return EXIT_FAILURE;
} else {
std::cout << "parse_many valid" << std::endl;
}
return EXIT_SUCCESS;
}
' >> ${DEMOCPP}
CPPBIN=$(basename ${DEMOCPP} .cpp)
echo "Try :" > ${README}
echo "c++ -O3 -std=c++17 -pthread -o ${CPPBIN} ${DEMOCPP##*/} && ./${CPPBIN##*/} ../jsonexamples/twitter.json ../jsonexamples/amazon_cellphones.ndjson" >> ${README}
echo "Done with all files generation."
echo "Files have been written to directory: ${AMALGAMATE_OUTPUT_PATH}/"
ls -la ${AMAL_C} ${AMAL_H} ${DEMOCPP} ${README}
#
# Instructions to create demo
#
echo ""
echo "Giving final instructions:"
cat ${README}
lowercase(){
echo "$1" | tr 'A-Z' 'a-z'
}
OS=`lowercase \`uname\``