-
Notifications
You must be signed in to change notification settings - Fork 15
Expand file tree
/
Copy pathupload_openbenchmarking.sh
More file actions
executable file
·138 lines (119 loc) · 5.16 KB
/
Copy pathupload_openbenchmarking.sh
File metadata and controls
executable file
·138 lines (119 loc) · 5.16 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#!/usr/bin/env bash
# Publish a full-mode engine benchmark run to OpenBenchmarking.org via Phoronix Test Suite.
#
# One-time setup per bench host (not handled by this script — see
# docs/modules/ROOT/pages/contributing/benchmarks.adoc):
# 1. ./scripts/install_pts.sh
# 2. phoronix-test-suite openbenchmarking-login
# (interactive; stores the account session locally — this script never sees or
# handles OpenBenchmarking.org credentials)
#
# What this script does:
# 1. Validates a full-mode JSON run (from scripts/run_engine_benchmarks.sh) against the
# BenchmarkRecord schema and refuses to proceed if any record is "unstable": true
# (CoV > benchmarks/manifests/engine-release.yml's stability.cov_limit_percent).
# Re-run on a quiet, dedicated host if this gate fails.
# 2. Syncs + statically validates the local PTS profiles/suite (scripts/validate_pts_profiles.sh).
# 3. Re-runs the suite through PTS itself (`phoronix-test-suite batch-benchmark`) — PTS needs
# to produce its own native result to upload; the JSON from step 1 is a parallel, richer
# record kept for schema validation and historical diffing, not a PTS result file. Batch
# mode is configured here (non-interactively, upload OFF) rather than depending on a
# prior `phoronix-test-suite batch-setup` on the host.
# 4. Uploads the resulting PTS result to OpenBenchmarking.org — a public, essentially
# irreversible action, so it only runs with --confirm (or CONFIRM_PUBLISH=yes); without
# it, this prints what it would do and exits.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
usage() {
sed -n '2,25p' "$0"
echo
echo "Usage: $0 <path to a full-mode out/engine/<TIMESTAMP> dir> [--confirm]"
}
JSON_DIR="${1:-}"
if [ -z "$JSON_DIR" ] || [ "$JSON_DIR" = "-h" ] || [ "$JSON_DIR" = "--help" ]; then
usage
exit "$([ -z "$JSON_DIR" ] && echo 2 || echo 0)"
fi
CONFIRM="${CONFIRM_PUBLISH:-no}"
[ "${2:-}" = "--confirm" ] && CONFIRM="yes"
RESULT_IDENTIFIER="${RESULT_IDENTIFIER:-skainet-engine-$(basename "$JSON_DIR")}"
if ! command -v phoronix-test-suite >/dev/null 2>&1; then
echo "ERROR: phoronix-test-suite not installed. Run ./scripts/install_pts.sh first." >&2
exit 2
fi
echo "== Stage 1: validate the JSON harness run at $JSON_DIR =="
"$REPO_ROOT/scripts/check_engine_json.sh" "$JSON_DIR"
UNSTABLE="$(python3 - "$JSON_DIR" <<'PY'
import glob, json, os, sys
d = sys.argv[1]
bad = [p for p in sorted(glob.glob(os.path.join(d, "*.json"))) if json.load(open(p)).get("unstable")]
print("\n".join(bad))
PY
)"
if [ -n "$UNSTABLE" ]; then
echo "ERROR: unstable records present (CoV over the manifest's stability limit) —" >&2
echo " re-run on a quiet, dedicated bench host before publishing:" >&2
echo "$UNSTABLE" >&2
exit 1
fi
echo "Stability gate: OK — no unstable records."
echo
echo "== Stage 2: sync + validate local PTS profiles and suite =="
"$REPO_ROOT/scripts/validate_pts_profiles.sh"
echo
echo "== Stage 3: configure PTS batch mode (non-interactive, upload OFF for this stage) =="
python3 - <<'PY'
import os
import xml.etree.ElementTree as ET
path = os.path.expanduser("~/.phoronix-test-suite/user-config.xml")
os.makedirs(os.path.dirname(path), exist_ok=True)
if os.path.isfile(path):
tree = ET.parse(path)
root = tree.getroot()
else:
root = ET.Element("PhoronixTestSuite")
tree = ET.ElementTree(root)
def ensure_path(parent, *tags):
for tag in tags:
child = parent.find(tag)
if child is None:
child = ET.SubElement(parent, tag)
parent = child
return parent
batch = ensure_path(root, "Options", "BatchMode")
values = {
"Configured": "TRUE",
"SaveResults": "TRUE",
"OpenBrowser": "FALSE",
"UploadResults": "FALSE", # upload is Stage 4, explicit and gated — never a side effect here
"PromptForTestIdentifier": "FALSE",
"PromptForTestDescription": "FALSE",
"PromptSaveName": "FALSE",
"RunAllTestCombinations": "TRUE", # runs both panama/scalar for the 3 provider-optioned profiles
}
for tag, value in values.items():
ensure_path(batch, tag).text = value
tree.write(path, encoding="unicode", xml_declaration=False)
print(f"BatchMode configured in {path}")
PY
echo
echo "== Stage 4: run the suite through PTS (produces the native result PTS can upload) =="
echo "Result identifier: $RESULT_IDENTIFIER"
if [ "$CONFIRM" != "yes" ]; then
echo
echo "DRY RUN — not executing. This would run:"
echo " TEST_RESULTS_NAME=$RESULT_IDENTIFIER phoronix-test-suite batch-benchmark local/skainet-engine-suite"
echo " phoronix-test-suite upload-result $RESULT_IDENTIFIER"
echo
echo "Re-run with --confirm (or CONFIRM_PUBLISH=yes) to actually publish to OpenBenchmarking.org."
exit 0
fi
TEST_RESULTS_NAME="$RESULT_IDENTIFIER" \
TEST_RESULTS_DESCRIPTION="SKaiNET engine suite — full mode, published from $JSON_DIR" \
phoronix-test-suite batch-benchmark local/skainet-engine-suite < /dev/null
echo
echo "== Stage 5: upload to OpenBenchmarking.org =="
echo "Requires a one-time 'phoronix-test-suite openbenchmarking-login' on this host."
phoronix-test-suite upload-result "$RESULT_IDENTIFIER"
echo
echo "Published: $RESULT_IDENTIFIER"