Skip to content

Commit 77006ed

Browse files
ctruedenctrueden
committed
Fix the benchmark scripts
* Fail fast if something goes wrong. * Only run the benchmarks if results text file is not present. * Build the conda environment into a local directory, on demand. * Call the environment's python binary directly instead of activating. * Emit feedback about what's happening during the benchmark script. * Do not hardcode 0-SNAPSHOT version string. * Run benchmarks with the same Java that Maven uses ($JAVA_HOME). * Install SLF4J binding to avoid repeated SLF4J warning messages. * Do not redirect human-readable output; instead, save to JSON. * Run each iteration for 5s, not 10s (results appear equivalent). * Make the shell script's output much nicer for humans to read. * Update the Python script to ingest the JSON output, not text. * Use milliseconds, not seconds, for benchmarking unit.
1 parent 7a59134 commit 77006ed

File tree

5 files changed

+82
-40
lines changed

5 files changed

+82
-40
lines changed

‎.gitignore‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,3 @@ target/
1515

1616
# Jupyter #
1717
.ipynb_checkpoints
18-
19-
# Docs #
20-
_build/

‎docs/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
_build/

‎docs/ops/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
/.benchmark-env/
2+
/scijava-ops-benchmarks_results.json

‎docs/ops/bin/benchmark.sh‎

Lines changed: 55 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,68 @@
11
#!/bin/bash
22

3-
conda init
3+
set -e
4+
set -o pipefail
5+
6+
if ! command -v mamba >/dev/null 2>&1
7+
then
8+
echo 'Please install mamba before running this script.'
9+
exit 1
10+
fi
411

5-
# Get the path to the script
612
SCRIPT_PATH=$(dirname "$(realpath -s "$0")")
713
DOCS_OPS_PATH="$SCRIPT_PATH/.."
814
INC_PATH="$DOCS_OPS_PATH/../../"
915
BENCHMARKS_PATH="$INC_PATH/scijava-ops-benchmarks"
16+
BENCH_OUT_FILE=scijava-ops-benchmarks_results.json
17+
BENCH_OUT_PATH=$(realpath -s "$DOCS_OPS_PATH/$BENCH_OUT_FILE")
18+
19+
if [ -f "$BENCH_OUT_PATH" ]
20+
then
21+
echo 'Graphing existing benchmark results from:'
22+
echo " $BENCH_OUT_PATH"
23+
echo 'To rerun benchmarks, delete this file first.'
24+
else
25+
echo
26+
echo '=== BUILDING THE CODE ==='
27+
cd "$INC_PATH"
28+
mvn -Denforcer.skip -Dinvoker.skip -Dmaven.test.skip -P benchmarks clean install -pl scijava-ops-benchmarks -am | grep '\(Building.*[0-9]\]\|ERROR\)'
1029

11-
BENCH_OUT_FILE=scijava-ops-benchmarks_results.txt
30+
echo
31+
echo '=== COPYING DEPENDENCIES ==='
32+
cd "$BENCHMARKS_PATH"
33+
{
34+
mvn dependency:copy-dependencies
35+
mvn dependency:copy -Dartifact=org.slf4j:slf4j-simple:1.7.36 -DoutputDirectory=target/dependency
36+
} | grep Copying | while read line
37+
do
38+
stdbuf -o0 echo -n '.'
39+
done
40+
echo
1241

13-
cd "$INC_PATH"
14-
mvn -P benchmarks clean install -pl scijava-ops-benchmarks -am
42+
# NB: Use the version of Java pointed to by the JAVA_HOME variable, if any.
43+
test -x "$JAVA_HOME/bin/java" && JAVA="$JAVA_HOME/bin/java" || JAVA=java
1544

16-
cd "$BENCHMARKS_PATH"
17-
mvn dependency:copy-dependencies
45+
echo
46+
echo '=== RUNNING BENCHMARKS ==='
47+
cd "$DOCS_OPS_PATH"
48+
"$JAVA" \
49+
-cp "$BENCHMARKS_PATH/target/*:$BENCHMARKS_PATH/target/dependency/*" \
50+
--add-opens=java.base/java.io=ALL-UNNAMED \
51+
org.openjdk.jmh.Main \
52+
-rf json -rff "$BENCH_OUT_FILE" \
53+
-r 5
54+
fi
1855

1956
cd "$DOCS_OPS_PATH"
20-
conda env create -f "environment.yml"
21-
java -cp "$BENCHMARKS_PATH/target/scijava-ops-benchmarks-0-SNAPSHOT.jar:$BENCHMARKS_PATH/target/dependency/*" org.openjdk.jmh.Main -o $BENCH_OUT_FILE
57+
envPath=.benchmark-env
58+
pythonPath="$envPath/bin/python"
59+
if [ ! -x "$pythonPath" ]
60+
then
61+
echo
62+
echo '=== CREATING CONDA ENVIRONMENT ==='
63+
mamba create -y -p "$envPath" python=3.10 plotly=5.19.0
64+
fi
2265

23-
source activate ops-docs
24-
python graph_results.py
25-
source deactivate
66+
echo
67+
echo '=== GRAPHING THE RESULTS ==='
68+
"$pythonPath" graph_results.py

‎docs/ops/graph_results.py‎

Lines changed: 24 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
# This script parses JMH benchmarking results into charts developed using plot.ly (https://plotly.com/)
77
# It currently develops one boxplot PER class, with each JMH benchmark method represented as a separate boxplot.
8-
# It expects JMH benchmark results be dumped to a file "scijava-ops-benchmark_results.txt", within its directory.
8+
# It expects JMH benchmark results be dumped to a file "scijava-ops-benchmark_results.json", within its directory.
99

1010
# If you'd like to add a title to the plotly charts, add an entry to the following dict.
1111
#
@@ -33,44 +33,39 @@
3333
}
3434

3535
# Read in the benchmark results
36-
with open("scijava-ops-benchmarks_results.txt") as f:
37-
lines = f.readlines()
38-
39-
# Keep only the lines containing our desired results
40-
for i in range(len(lines) - 1, 0, -1):
41-
if (lines[i].startswith("Benchmark ")):
42-
lines = lines[i+1:]
43-
break
36+
with open("scijava-ops-benchmarks_results.json") as f:
37+
data = json.load(f)
4438

4539
# Build a map of results by benchmark class
4640
benchmark_classes = {}
47-
for line in lines:
48-
words = line.split()
49-
test = words[0]
50-
last_period = test.rfind('.')
51-
cls = test[:last_period]
52-
test = test[last_period+1:]
41+
for row in data:
42+
fqdn_tokens = row["benchmark"].split(".")
43+
cls, test = fqdn_tokens[-2], fqdn_tokens[-1]
5344

5445
if cls not in benchmark_classes:
5546
benchmark_classes[cls] = {}
56-
57-
benchmark_classes[cls][test] = words[1:]
47+
48+
# NB: Convert seconds to milliseconds.
49+
benchmark_classes[cls][test] = {
50+
"score": 1000 * row["primaryMetric"]["score"],
51+
"error": 1000 * row["primaryMetric"]["scoreError"],
52+
}
5853

5954
# For each class, build a chart and dump it to JSON
60-
for cls, data in benchmark_classes.items():
61-
period_pos = cls.rfind(".")
62-
if period_pos > -1:
63-
cls = cls[period_pos+1:]
55+
for cls, test in benchmark_classes.items():
56+
print(f"Generating figure for {cls}", end="")
6457
x = []
6558
y = []
6659
error_y = []
6760

6861
# Add each benchmark in the class
69-
for method, line in data.items():
62+
for method, stats in test.items():
63+
print(".", end="")
7064
method = benchmark_categories.get(method, method)
7165
x.append(method)
72-
y.append(float(line[2]))
73-
error_y.append(float(line[4]))
66+
y.append(stats["score"])
67+
error_y.append(stats["error"])
68+
7469
# Create a bar chart
7570
fig = go.Figure()
7671
fig.add_bar(
@@ -80,9 +75,13 @@
8075
)
8176
fig.update_layout(
8277
title_text=figure_titles.get(cls, "TODO: Add title"),
83-
yaxis_title="Performance (s/op)"
78+
yaxis_title="Performance (ms/op)"
8479
)
8580

8681
# Convert to JSON and dump
8782
with open(f"images/{cls}.json", "w") as f:
8883
f.write(io.to_json(fig))
84+
85+
print()
86+
87+
print("Done!")

0 commit comments

Comments
 (0)