Maintain runtime of each benchmark at ~10 seconds
The idea to run benchmarks for a constant number of iterations is problematic. While the benchmarks may run for 10 seconds on x86_64, they could run for about 30 seconds on powerpc and worse, over 3 minutes on arm. Besides that, adding a new benchmark is cumbersome since one needs to find out the number of iterations needed for a sufficient runtime. A better idea would be to run each benchmark for a specific amount of time. This patch does just that. The run time defaults to 10 seconds and it is configurable at command line: make BENCH_DURATION=5 bench
This commit is contained in:
parent
a6a242fe7c
commit
d569c6eeb4
11
ChangeLog
11
ChangeLog
|
@ -1,3 +1,14 @@
|
||||||
|
2013-04-30 Siddhesh Poyarekar <siddhesh@redhat.com>
|
||||||
|
|
||||||
|
* benchtests/Makefile: Remove *-ITER. Define BENCH_DURATION
|
||||||
|
in CPPFLAGS.
|
||||||
|
($(objpfx)bench-%.c): Remove *-ITER.
|
||||||
|
* benchtests/bench-modf.c: Remove definition of ITER.
|
||||||
|
* benchtests/bench-skeleton.c (TIMESPEC_AFTER): New macro.
|
||||||
|
(main): Loop for DURATION seconds instead of fixed number of
|
||||||
|
iterations.
|
||||||
|
* scripts/bench.pl: Don't expect iterations in parameters.
|
||||||
|
|
||||||
2013-04-29 Roland McGrath <roland@hack.frob.com>
|
2013-04-29 Roland McGrath <roland@hack.frob.com>
|
||||||
|
|
||||||
* io/fchdir.c (__fchdir): Renamed from fchdir.
|
* io/fchdir.c (__fchdir): Renamed from fchdir.
|
||||||
|
|
|
@ -23,12 +23,6 @@
|
||||||
|
|
||||||
# - Append the function name to the bench variable
|
# - Append the function name to the bench variable
|
||||||
|
|
||||||
# - Define foo-ITER with the number of iterations you want to run. Keep it
|
|
||||||
# high enough that the overhead of clock_gettime is only a small fraction of
|
|
||||||
# the total run time of the test. A good idea would be to keep the run time
|
|
||||||
# of each test at around 10 seconds for x86_64. That is just a guideline,
|
|
||||||
# since some scenarios may require higher run times.
|
|
||||||
|
|
||||||
# - Define foo-ARGLIST as a colon separated list of types of the input
|
# - Define foo-ARGLIST as a colon separated list of types of the input
|
||||||
# arguments. Use `void` if function does not take any inputs. Put in quotes
|
# arguments. Use `void` if function does not take any inputs. Put in quotes
|
||||||
# if the input argument is a pointer, e.g.:
|
# if the input argument is a pointer, e.g.:
|
||||||
|
@ -49,83 +43,70 @@ bench := exp pow rint sin cos tan atan modf \
|
||||||
slowexp slowpow slowsin slowcos slowtan slowatan
|
slowexp slowpow slowsin slowcos slowtan slowatan
|
||||||
|
|
||||||
# exp function fast path: sysdeps/ieee754/dbl-64/e_exp.c
|
# exp function fast path: sysdeps/ieee754/dbl-64/e_exp.c
|
||||||
exp-ITER = 5e8
|
|
||||||
exp-ARGLIST = double
|
exp-ARGLIST = double
|
||||||
exp-RET = double
|
exp-RET = double
|
||||||
LDFLAGS-bench-exp = -lm
|
LDFLAGS-bench-exp = -lm
|
||||||
|
|
||||||
# pow function fast path: sysdeps/ieee754/dbl-64/e_pow.c
|
# pow function fast path: sysdeps/ieee754/dbl-64/e_pow.c
|
||||||
pow-ITER = 2e8
|
|
||||||
pow-ARGLIST = double:double
|
pow-ARGLIST = double:double
|
||||||
pow-RET = double
|
pow-RET = double
|
||||||
LDFLAGS-bench-pow = -lm
|
LDFLAGS-bench-pow = -lm
|
||||||
|
|
||||||
rint-ITER = 250000000
|
|
||||||
rint-ARGLIST = double
|
rint-ARGLIST = double
|
||||||
rint-RET = double
|
rint-RET = double
|
||||||
LDFLAGS-bench-rint = -lm
|
LDFLAGS-bench-rint = -lm
|
||||||
|
|
||||||
# exp function slowest path: sysdeps/ieee754/dbl-64/mpexp.c
|
# exp function slowest path: sysdeps/ieee754/dbl-64/mpexp.c
|
||||||
slowexp-ITER = 3e5
|
|
||||||
slowexp-ARGLIST = double
|
slowexp-ARGLIST = double
|
||||||
slowexp-RET = double
|
slowexp-RET = double
|
||||||
slowexp-INCLUDE = slowexp.c
|
slowexp-INCLUDE = slowexp.c
|
||||||
LDFLAGS-bench-slowexp = -lm
|
LDFLAGS-bench-slowexp = -lm
|
||||||
|
|
||||||
# sin function fast path: sysdeps/ieee754/dbl-64/s_sin.c
|
# sin function fast path: sysdeps/ieee754/dbl-64/s_sin.c
|
||||||
sin-ITER = 3e9
|
|
||||||
sin-ARGLIST = double
|
sin-ARGLIST = double
|
||||||
sin-RET = double
|
sin-RET = double
|
||||||
LDFLAGS-bench-sin = -lm
|
LDFLAGS-bench-sin = -lm
|
||||||
|
|
||||||
# cos function fast path: sysdeps/ieee754/dbl-64/s_sin.c
|
# cos function fast path: sysdeps/ieee754/dbl-64/s_sin.c
|
||||||
cos-ITER = 3e9
|
|
||||||
cos-ARGLIST = double
|
cos-ARGLIST = double
|
||||||
cos-RET = double
|
cos-RET = double
|
||||||
LDFLAGS-bench-cos = -lm
|
LDFLAGS-bench-cos = -lm
|
||||||
|
|
||||||
# tan function fast path: sysdeps/ieee754/dbl-64/s_tan.c
|
# tan function fast path: sysdeps/ieee754/dbl-64/s_tan.c
|
||||||
tan-ITER = 3e9
|
|
||||||
tan-ARGLIST = double
|
tan-ARGLIST = double
|
||||||
tan-RET = double
|
tan-RET = double
|
||||||
LDFLAGS-bench-tan = -lm
|
LDFLAGS-bench-tan = -lm
|
||||||
|
|
||||||
# atan function fast path: sysdeps/ieee754/dbl-64/s_atan.c
|
# atan function fast path: sysdeps/ieee754/dbl-64/s_atan.c
|
||||||
atan-ITER = 6e9
|
|
||||||
atan-ARGLIST = double
|
atan-ARGLIST = double
|
||||||
atan-RET = double
|
atan-RET = double
|
||||||
LDFLAGS-bench-atan = -lm
|
LDFLAGS-bench-atan = -lm
|
||||||
|
|
||||||
# pow function slowest path: sysdeps/ieee754/dbl-64/slowpow.c
|
# pow function slowest path: sysdeps/ieee754/dbl-64/slowpow.c
|
||||||
slowpow-ITER = 1e5
|
|
||||||
slowpow-ARGLIST = double:double
|
slowpow-ARGLIST = double:double
|
||||||
slowpow-RET = double
|
slowpow-RET = double
|
||||||
slowpow-INCLUDE = slowpow.c
|
slowpow-INCLUDE = slowpow.c
|
||||||
LDFLAGS-bench-slowpow = -lm
|
LDFLAGS-bench-slowpow = -lm
|
||||||
|
|
||||||
# sin function slowest path: sysdeps/ieee754/dbl-64/sincos32.c
|
# sin function slowest path: sysdeps/ieee754/dbl-64/sincos32.c
|
||||||
slowsin-ITER = 3e7
|
|
||||||
slowsin-ARGLIST = double
|
slowsin-ARGLIST = double
|
||||||
slowsin-RET = double
|
slowsin-RET = double
|
||||||
slowsin-INCLUDE = slowsin.c
|
slowsin-INCLUDE = slowsin.c
|
||||||
LDFLAGS-bench-slowsin = -lm
|
LDFLAGS-bench-slowsin = -lm
|
||||||
|
|
||||||
# cos function slowest path: sysdeps/ieee754/dbl-64/sincos32.c
|
# cos function slowest path: sysdeps/ieee754/dbl-64/sincos32.c
|
||||||
slowcos-ITER = 3e7
|
|
||||||
slowcos-ARGLIST = double
|
slowcos-ARGLIST = double
|
||||||
slowcos-RET = double
|
slowcos-RET = double
|
||||||
slowcos-INCLUDE = slowcos.c
|
slowcos-INCLUDE = slowcos.c
|
||||||
LDFLAGS-bench-slowcos = -lm
|
LDFLAGS-bench-slowcos = -lm
|
||||||
|
|
||||||
# tan function slowest path: sysdeps/ieee754/dbl-64/mptan.c
|
# tan function slowest path: sysdeps/ieee754/dbl-64/mptan.c
|
||||||
slowtan-ITER = 3e7
|
|
||||||
slowtan-ARGLIST = double
|
slowtan-ARGLIST = double
|
||||||
slowtan-RET = double
|
slowtan-RET = double
|
||||||
slowtan-INCLUDE = slowtan.c
|
slowtan-INCLUDE = slowtan.c
|
||||||
LDFLAGS-bench-slowtan = -lm
|
LDFLAGS-bench-slowtan = -lm
|
||||||
|
|
||||||
# atan function slowest path: sysdeps/ieee754/dbl-64/mpatan.c
|
# atan function slowest path: sysdeps/ieee754/dbl-64/mpatan.c
|
||||||
slowatan-ITER = 3e8
|
|
||||||
slowatan-ARGLIST = double
|
slowatan-ARGLIST = double
|
||||||
slowatan-RET = double
|
slowatan-RET = double
|
||||||
slowatan-INCLUDE = slowatan.c
|
slowatan-INCLUDE = slowatan.c
|
||||||
|
@ -141,6 +122,13 @@ include ../Rules
|
||||||
|
|
||||||
binaries-bench := $(addprefix $(objpfx)bench-,$(bench))
|
binaries-bench := $(addprefix $(objpfx)bench-,$(bench))
|
||||||
|
|
||||||
|
# The default duration: 10 seconds.
|
||||||
|
ifndef BENCH_DURATION
|
||||||
|
BENCH_DURATION := 10
|
||||||
|
endif
|
||||||
|
|
||||||
|
CPPFLAGS-nonlib = -DDURATION=$(BENCH_DURATION)
|
||||||
|
|
||||||
# This makes sure CPPFLAGS-nonlib and CFLAGS-nonlib are passed
|
# This makes sure CPPFLAGS-nonlib and CFLAGS-nonlib are passed
|
||||||
# for all these modules.
|
# for all these modules.
|
||||||
cpp-srcs-left := $(binaries-bench:=.c)
|
cpp-srcs-left := $(binaries-bench:=.c)
|
||||||
|
@ -176,5 +164,5 @@ $(objpfx)bench-%.c: %-inputs $(bench-deps)
|
||||||
cat $($*-INCLUDE); \
|
cat $($*-INCLUDE); \
|
||||||
fi; \
|
fi; \
|
||||||
$(..)scripts/bench.pl $(patsubst %-inputs,%,$<) \
|
$(..)scripts/bench.pl $(patsubst %-inputs,%,$<) \
|
||||||
$($*-ITER) $($*-ARGLIST) $($*-RET); } > $@-tmp
|
$($*-ARGLIST) $($*-RET); } > $@-tmp
|
||||||
mv -f $@-tmp $@
|
mv -f $@-tmp $@
|
||||||
|
|
|
@ -33,7 +33,6 @@ struct args
|
||||||
static volatile double ret = 0.0;
|
static volatile double ret = 0.0;
|
||||||
#define BENCH_FUNC(j) ({double iptr; ret = CALL_BENCH_FUNC (j, iptr);})
|
#define BENCH_FUNC(j) ({double iptr; ret = CALL_BENCH_FUNC (j, iptr);})
|
||||||
|
|
||||||
#define ITER 250000000
|
|
||||||
#define FUNCNAME "modf"
|
#define FUNCNAME "modf"
|
||||||
|
|
||||||
#include "bench-skeleton.c"
|
#include "bench-skeleton.c"
|
||||||
|
|
|
@ -22,13 +22,18 @@
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
#include <inttypes.h>
|
#include <inttypes.h>
|
||||||
|
|
||||||
|
#define TIMESPEC_AFTER(a, b) \
|
||||||
|
(((a).tv_sec == (b).tv_sec) ? \
|
||||||
|
((a).tv_nsec > (b).tv_nsec) : \
|
||||||
|
((a).tv_sec > (b).tv_sec))
|
||||||
int
|
int
|
||||||
main (int argc, char **argv)
|
main (int argc, char **argv)
|
||||||
{
|
{
|
||||||
unsigned long i, j, k;
|
unsigned long i, k;
|
||||||
uint64_t total = 0, max = 0, min = 0x7fffffffffffffff;
|
uint64_t total = 0, max = 0, min = 0x7fffffffffffffff;
|
||||||
struct timespec start, end;
|
struct timespec start, end, runtime;
|
||||||
|
|
||||||
|
memset (&runtime, 0, sizeof (runtime));
|
||||||
memset (&start, 0, sizeof (start));
|
memset (&start, 0, sizeof (start));
|
||||||
memset (&end, 0, sizeof (end));
|
memset (&end, 0, sizeof (end));
|
||||||
|
|
||||||
|
@ -39,11 +44,15 @@ main (int argc, char **argv)
|
||||||
Measurements close to the minimum clock resolution won't make much sense,
|
Measurements close to the minimum clock resolution won't make much sense,
|
||||||
but it's better than having nothing at all. */
|
but it's better than having nothing at all. */
|
||||||
unsigned long iters = 1000 * start.tv_nsec;
|
unsigned long iters = 1000 * start.tv_nsec;
|
||||||
unsigned long total_iters = ITER / iters;
|
|
||||||
|
|
||||||
for (i = 0; i < NUM_SAMPLES; i++)
|
/* Run for approxmately DURATION seconds. */
|
||||||
|
clock_gettime (CLOCK_MONOTONIC_RAW, &runtime);
|
||||||
|
runtime.tv_sec += DURATION;
|
||||||
|
|
||||||
|
double d_total_i = 0;
|
||||||
|
while (1)
|
||||||
{
|
{
|
||||||
for (j = 0; j < total_iters; j ++)
|
for (i = 0; i < NUM_SAMPLES; i++)
|
||||||
{
|
{
|
||||||
clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &start);
|
clock_gettime (CLOCK_PROCESS_CPUTIME_ID, &start);
|
||||||
for (k = 0; k < iters; k++)
|
for (k = 0; k < iters; k++)
|
||||||
|
@ -61,12 +70,25 @@ main (int argc, char **argv)
|
||||||
min = cur;
|
min = cur;
|
||||||
|
|
||||||
total += cur;
|
total += cur;
|
||||||
|
|
||||||
|
d_total_i += iters;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct timespec curtime;
|
||||||
|
|
||||||
|
memset (&curtime, 0, sizeof (curtime));
|
||||||
|
clock_gettime (CLOCK_MONOTONIC_RAW, &curtime);
|
||||||
|
if (TIMESPEC_AFTER (curtime, runtime))
|
||||||
|
goto done;
|
||||||
}
|
}
|
||||||
|
|
||||||
double d_total_s = total * 1e-9;
|
double d_total_s;
|
||||||
double d_iters = iters;
|
double d_iters;
|
||||||
double d_total_i = (double)ITER * NUM_SAMPLES;
|
|
||||||
|
done:
|
||||||
|
d_total_s = total * 1e-9;
|
||||||
|
d_iters = iters;
|
||||||
|
|
||||||
printf (FUNCNAME ": ITERS:%g: TOTAL:%gs, MAX:%gns, MIN:%gns, %g iter/s\n",
|
printf (FUNCNAME ": ITERS:%g: TOTAL:%gs, MAX:%gns, MIN:%gns, %g iter/s\n",
|
||||||
d_total_i, d_total_s, max / d_iters, min / d_iters,
|
d_total_i, d_total_s, max / d_iters, min / d_iters,
|
||||||
d_total_i / d_total_s);
|
d_total_i / d_total_s);
|
||||||
|
|
|
@ -22,23 +22,22 @@ use warnings;
|
||||||
# Generate a benchmark source file for a given input.
|
# Generate a benchmark source file for a given input.
|
||||||
|
|
||||||
if (@ARGV < 2) {
|
if (@ARGV < 2) {
|
||||||
die "Usage: bench.pl <function> <iterations> [parameter types] [return type]"
|
die "Usage: bench.pl <function> [parameter types] [return type]"
|
||||||
}
|
}
|
||||||
|
|
||||||
my $arg;
|
my $arg;
|
||||||
my $func = $ARGV[0];
|
my $func = $ARGV[0];
|
||||||
my $iters = $ARGV[1];
|
|
||||||
my @args;
|
my @args;
|
||||||
my $ret = "void";
|
my $ret = "void";
|
||||||
my $getret = "";
|
my $getret = "";
|
||||||
my $retval = "";
|
my $retval = "";
|
||||||
|
|
||||||
if (@ARGV >= 3) {
|
if (@ARGV >= 2) {
|
||||||
@args = split(':', $ARGV[2]);
|
@args = split(':', $ARGV[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (@ARGV == 4) {
|
if (@ARGV == 3) {
|
||||||
$ret = $ARGV[3];
|
$ret = $ARGV[2];
|
||||||
}
|
}
|
||||||
|
|
||||||
my $decl = "extern $ret $func (";
|
my $decl = "extern $ret $func (";
|
||||||
|
@ -88,6 +87,5 @@ if ($ret ne "void") {
|
||||||
|
|
||||||
print "#define BENCH_FUNC(j) ({$getret CALL_BENCH_FUNC (j);})\n";
|
print "#define BENCH_FUNC(j) ({$getret CALL_BENCH_FUNC (j);})\n";
|
||||||
|
|
||||||
print "#define ITER $iters\n";
|
|
||||||
print "#define FUNCNAME \"$func\"\n";
|
print "#define FUNCNAME \"$func\"\n";
|
||||||
print "#include \"bench-skeleton.c\"\n";
|
print "#include \"bench-skeleton.c\"\n";
|
||||||
|
|
Loading…
Reference in New Issue