|
| 1 | +#!/usr/bin/python3 |
| 2 | +import subprocess |
| 3 | +import re |
| 4 | +import statistics |
| 5 | +import math |
| 6 | +import sys |
| 7 | + |
| 8 | +def run(prog, runs): |
| 9 | + rlist = [] |
| 10 | + exp = re.compile(b"Nodes/second\s*: (\d+)") |
| 11 | + for i in range(runs): |
| 12 | + cap = subprocess.run([prog,"bench"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) |
| 13 | + res = int(exp.search(cap.stderr).group(1)) |
| 14 | + sys.stdout.write("\r%s %d: %d" % (prog,i+1,res)) |
| 15 | + sys.stdout.flush() |
| 16 | + rlist.append(res) |
| 17 | + return rlist |
| 18 | + |
| 19 | +runs = int(sys.argv[3]) |
| 20 | +if (runs < 2): |
| 21 | + exit("Statistics require at least two runs") |
| 22 | +base = run(sys.argv[1], runs) |
| 23 | +test = run(sys.argv[2], runs) |
| 24 | +print ("\nResult of ", runs, "runs") |
| 25 | +b_mean = statistics.mean(base) |
| 26 | +b_dev = statistics.stdev(base, b_mean) |
| 27 | +t_mean = statistics.mean(test) |
| 28 | +t_dev = statistics.stdev(test, t_mean) |
| 29 | +print ("Base nps: ", b_mean, " stdev: ", b_dev) |
| 30 | +print ("Test nps: ", t_mean, " stdev: ", t_dev) |
| 31 | +diff = (max(t_mean, b_mean)/min(t_mean, b_mean)-1)*100 |
| 32 | +print ("Speed-up: ", ("+","-")[t_mean < b_mean]+str(round(diff,4))+"%") |
| 33 | +for i in range(len(base)): |
| 34 | + test[i] = test[i] - base[i] |
| 35 | +t_mean = statistics.mean(test) |
| 36 | +t_dev = statistics.stdev(test,t_mean) |
| 37 | +p_val = .5+.5*math.erf(t_mean/t_dev/math.sqrt(2.0)) |
| 38 | +print ("p-value: ", round(p_val,4)) |
0 commit comments