OpenE2K/qemu-e2k
13
4
Fork 0
Code Issues 4 Pull Requests Projects 2 Releases Activity

scripts/simplebench: use standard deviation for +- error

Browse Source 
Standard deviation is more usual to see after +- than current maximum
of deviations.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Message-Id: <20201021145859.11201-16-vsementsov@virtuozzo.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
This commit is contained in:
Vladimir Sementsov-Ogievskiy 2020-10-21 17:58:53 +03:00 committed by Max Reitz
parent 4a44554a65
commit f52e1af0b0
1 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
scripts/simplebench/simplebench.py
View File

@ -18,6 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
import statistics
def bench_one(test_func, test_env, test_case, count=5, initial_run=True): def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
"""Benchmark one test-case """Benchmark one test-case
@ -40,7 +42,7 @@ def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
'dimension': dimension of results, may be 'seconds' or 'iops' 'dimension': dimension of results, may be 'seconds' or 'iops'
'average': average value (iops or seconds) per run (exists only if at 'average': average value (iops or seconds) per run (exists only if at
least one run succeeded) least one run succeeded)
'delta': maximum delta between test_func result and the average 'stdev': standard deviation of results
(exists only if at least one run succeeded) (exists only if at least one run succeeded)
'n-failed': number of failed runs (exists only if at least one run 'n-failed': number of failed runs (exists only if at least one run
failed) failed)
@ -67,10 +69,9 @@ def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
assert all('seconds' in r for r in succeeded) assert all('seconds' in r for r in succeeded)
assert all('iops' not in r for r in succeeded) assert all('iops' not in r for r in succeeded)
dim = 'seconds' dim = 'seconds'
avg = sum(r[dim] for r in succeeded) / len(succeeded)
result['dimension'] = dim result['dimension'] = dim
result['average'] = avg result['average'] = statistics.mean(r[dim] for r in succeeded)
result['delta'] = max(abs(r[dim] - avg) for r in succeeded) result['stdev'] = statistics.stdev(r[dim] for r in succeeded)
if len(succeeded) < count: if len(succeeded) < count:
result['n-failed'] = count - len(succeeded) result['n-failed'] = count - len(succeeded)
@ -81,7 +82,7 @@ def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
def ascii_one(result): def ascii_one(result):
"""Return ASCII representation of bench_one() returned dict.""" """Return ASCII representation of bench_one() returned dict."""
if 'average' in result: if 'average' in result:
s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta']) s = '{:.2f} +- {:.2f}'.format(result['average'], result['stdev'])
if 'n-failed' in result: if 'n-failed' in result:
s += '\n({} failed)'.format(result['n-failed']) s += '\n({} failed)'.format(result['n-failed'])
return s return s