Skip to content

Refactor benchmark codes and scripts #285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
leventeliu merged 5 commits into from
Mar 18, 2019
Merged

Refactor benchmark codes and scripts #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6a9fd16
Refactor benchmark codes and scripts
leventeliu Mar 18, 2019
653ff8c
Merge remote-tracking branch 'origin/develop' into feature/bench_arg
leventeliu Mar 18, 2019
9eb99bd
Minor fix
leventeliu Mar 18, 2019
641b58a
Add function to make bench name for parsing
leventeliu Mar 18, 2019
14395bc
Add eventual consistency benchmark in GNTE
leventeliu Mar 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion alltest.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ main() {
# some benchmarks
go test -tags "$UNITTESTTAGS" -bench=^BenchmarkPersistentCaller_Call$ -run ^$ ./rpc/
bash cleanupDB.sh || true
go test -tags "$UNITTESTTAGS" -bench=^BenchmarkMinerTwo$ -benchtime=5s -run ^$ ./cmd/cql-minerd/
go test -tags "$UNITTESTTAGS" -bench=^BenchmarkMiner$ -benchtime=5s -run ^$ ./cmd/cql-minerd/ -bench-miner-count=2
bash cleanupDB.sh || true
}

Expand Down
42 changes: 26 additions & 16 deletions cmd/cql-minerd/bench.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,27 @@
#!/bin/bash
#! /usr/bin/env bash
set -euo pipefail

make -C ../../ clean && \
make -C ../../ use_all_cores && \
go test -bench=^BenchmarkSQLite$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerOne$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerOneNoSign$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerTwo$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerTwoNoSign$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerThree$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerThreeNoSign$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerOneWithEventualConsistency$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerOneNoSignWithEventualConsistency$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerTwoWithEventualConsistency$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerTwoNoSignWithEventualConsistency$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerThreeWithEventualConsistency$ -benchtime=10s -run ^$ && \
go test -bench=^BenchmarkMinerThreeNoSignWithEventualConsistency$ -benchtime=10s -run ^$

main() {
local wd="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)"
make -C "$wd" clean
make -C "$wd" use_all_cores

local pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd"
#go test -bench=^BenchmarkSQLite$ -benchtime=10s -run=^$ "$pkg"

local flags=(
"-bench=^BenchmarkMiner$"
"-benchtime=10s"
"-run=^$"
)
local i
for ((i=1; i<=3; i++)); do
go test "${flags[@]}" "$pkg" -bench-miner-count=$i
go test "${flags[@]}" "$pkg" -bench-miner-count=$i -bench-bypass-signature
go test "${flags[@]}" "$pkg" -bench-miner-count=$i -bench-eventual-consistency
go test "${flags[@]}" "$pkg" -bench-miner-count=$i -bench-bypass-signature -bench-eventual-consistency
done
}

main "$@"
41 changes: 29 additions & 12 deletions cmd/cql-minerd/benchCustom.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,29 @@
#!/bin/bash

make -C ../../ clean && \
make -C ../../ use_all_cores
export miner_conf_dir=$PWD/../../test/bench_testnet/node_c
go test -bench=^BenchmarkCustomMiner1$ -benchtime=10s -run ^$ |tee custom_miner.log
go test -bench=^BenchmarkCustomMiner2$ -benchtime=10s -run ^$ |tee -a custom_miner.log
go test -bench=^BenchmarkCustomMiner3$ -benchtime=10s -run ^$ |tee -a custom_miner.log

go test -cpu=1 -bench=^BenchmarkCustomMiner1$ -benchtime=10s -run ^$ |tee -a custom_miner.log
go test -cpu=1 -bench=^BenchmarkCustomMiner2$ -benchtime=10s -run ^$ |tee -a custom_miner.log
go test -cpu=1 -bench=^BenchmarkCustomMiner3$ -benchtime=10s -run ^$ |tee -a custom_miner.log
#! /usr/bin/env bash
set -euo pipefail

main() {
local wd="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)"
make -C "$wd" clean
make -C "$wd" use_all_cores

rm -f custom_miner.log
touch custom_miner.log

local flags=(
"-bench=^BenchmarkCustomMiner$"
"-benchtime=10s"
"-run=^$"
)
local pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd"
local i subflags
for ((i=1; i<=3; i++)); do
subflags=(
"-bench-miner-config-dir=$wd/test/bench_testnet/node_c"
"-bench-miner-count=$i"
)
go test "${flags[@]}" "$pkg" "${subflags[@]}" | tee -a custom_miner.log
go test -cpu=1 "${flags[@]}" "$pkg" "${subflags[@]}" | tee -a custom_miner.log
done
}

main "$@"
65 changes: 40 additions & 25 deletions cmd/cql-minerd/benchGNTE.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,45 @@
#!/bin/bash
#! /usr/bin/env bash
set -euo pipefail

param=$1
declare pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd"
declare flags=(
"-bench=^BenchmarkMinerGNTE$"
"-benchtime=10s"
"-run=^$"
)

#make -C ../../ clean && \
#make -C ../../ use_all_cores && \
fast() {
echo "Fast benchmarking with flags: $@"
go test "${flags[@]}" "$pkg" "$@" | tee -a gnte.log
go test "${flags[@]}" "$pkg" "$@" -bench-miner-count=2 | tee -a gnte.log
go test -cpu=1 "${flags[@]}" "$pkg" "$@" -bench-miner-count=2 | tee -a gnte.log
}

if [ "fast" == "$param" ]; then
go test -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee gnte.log
go test -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=1 -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log
else
go test -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee gnte.log
go test -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -bench=^BenchmarkMinerGNTE3$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -bench=^BenchmarkMinerGNTE4$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -bench=^BenchmarkMinerGNTE8$ -benchtime=10s -run ^$ |tee -a gnte.log
full() {
echo "Full benchmarking with flags: $@"
local cpus=("" 4 1) counts=(1 2 3 4 8)
local cpu count caseflags
for cpu in "${cpus[@]}"; do
if [[ -z $cpu ]]; then
caseflags=("${flags[@]}")
else
caseflags=("-cpu=$cpu" "${flags[@]}")
fi
for count in "${counts[@]}"; do
go test "${caseflags[@]}" "$pkg" "$@" -bench-miner-count=$count | tee -a gnte.log
done
done
}

go test -cpu=4 -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=4 -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=4 -bench=^BenchmarkMinerGNTE3$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=4 -bench=^BenchmarkMinerGNTE4$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=4 -bench=^BenchmarkMinerGNTE8$ -benchtime=10s -run ^$ |tee -a gnte.log
main() {
rm -f gnte.log
touch gnte.log
if [[ $# -gt 0 && $1 = "fast" ]]; then
fast
else
full
full -bench-eventual-consistency
fi
}

go test -cpu=1 -bench=^BenchmarkMinerGNTE1$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=1 -bench=^BenchmarkMinerGNTE2$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=1 -bench=^BenchmarkMinerGNTE3$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=1 -bench=^BenchmarkMinerGNTE4$ -benchtime=10s -run ^$ |tee -a gnte.log
go test -cpu=1 -bench=^BenchmarkMinerGNTE8$ -benchtime=10s -run ^$ |tee -a gnte.log
fi
main "$@"
39 changes: 30 additions & 9 deletions cmd/cql-minerd/benchTestnet.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,32 @@
#!/bin/bash
#! /usr/bin/env bash
set -euo pipefail

make -C ../../ clean && \
make -C ../../ use_all_cores
go test -bench=^BenchmarkTestnetMiner1$ -benchtime=10s -run ^$ |tee testnet.log
go test -bench=^BenchmarkTestnetMiner2$ -benchtime=10s -run ^$ |tee -a testnet.log
go test -bench=^BenchmarkTestnetMiner3$ -benchtime=10s -run ^$ |tee -a testnet.log
main() {
local wd="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."; pwd)"
make -C "$wd" clean
make -C "$wd" use_all_cores

go test -cpu=1 -bench=^BenchmarkTestnetMiner1$ -benchtime=10s -run ^$ |tee -a testnet.log
go test -cpu=1 -bench=^BenchmarkTestnetMiner2$ -benchtime=10s -run ^$ |tee -a testnet.log
go test -cpu=1 -bench=^BenchmarkTestnetMiner3$ -benchtime=10s -run ^$ |tee -a testnet.log
rm -f testnet.log
touch testnet.log

local flags=(
"-bench=^BenchmarkTestnetMiner$"
"-benchtime=10s"
"-run=^$"
)
local pkg="github.com/CovenantSQL/CovenantSQL/cmd/cql-minerd"
local cpus=("" 1) counts=(1 2 3)
local cpu count caseflags
for cpu in "${cpus[@]}"; do
if [[ -z $cpu ]]; then
caseflags=("${flags[@]}")
else
caseflags=("-cpu=$cpu" "${flags[@]}")
fi
for count in "${counts[@]}"; do
go test "${caseflags[@]}" "$pkg" -bench-miner-count=$count | tee -a testnet.log
done
done
}

main "$@"
Loading