test: add support for sharding testsuite by passing --test-shard=a.b

This commit is contained in:
Graydon Hoare 2013-08-23 15:30:23 -07:00
parent 2c0f9bd354
commit 2fb5c49abb
4 changed files with 60 additions and 6 deletions

View File

@ -73,16 +73,24 @@ TEST_RATCHET_NOISE_PERCENT=10.0
# Whether to ratchet or merely save benchmarks
ifdef CFG_RATCHET_BENCH
CRATE_TEST_BENCH_ARGS=\
CRATE_TEST_EXTRA_ARGS=\
--test $(TEST_BENCH) \
--ratchet-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4)) \
--ratchet-noise-percent $(TEST_RATCHET_NOISE_PERCENT)
else
CRATE_TEST_BENCH_ARGS=\
CRATE_TEST_EXTRA_ARGS=\
--test $(TEST_BENCH) \
--save-metrics $(call TEST_RATCHET_FILE,$(1),$(2),$(3),$(4))
endif
# If we're sharding the testsuite between parallel testers,
# pass this argument along to the compiletest and crate test
# invocations.
ifdef TEST_SHARD
CTEST_TESTARGS += --test-shard=$(TEST_SHARD)
CRATE_TEST_EXTRA_ARGS += --test-shard=$(TEST_SHARD)
endif
define DEF_TARGET_COMMANDS
ifdef CFG_UNIXY_$(1)
@ -401,7 +409,7 @@ $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
@$$(call E, run: $$<)
$$(Q)$$(call CFG_RUN_TEST_$(2),$$<,$(2),$(3)) $$(TESTARGS) \
--logfile $$(call TEST_LOG_FILE,$(1),$(2),$(3),$(4)) \
$$(call CRATE_TEST_BENCH_ARGS,$(1),$(2),$(3),$(4)) \
$$(call CRATE_TEST_EXTRA_ARGS,$(1),$(2),$(3),$(4)) \
&& touch $$@
endef
@ -415,7 +423,7 @@ $$(call TEST_OK_FILE,$(1),$(2),$(3),$(4)): \
@$(CFG_ADB) shell '(cd $(CFG_ADB_TEST_DIR); LD_LIBRARY_PATH=. \
./$$(notdir $$<) \
--logfile $(CFG_ADB_TEST_DIR)/check-stage$(1)-T-$(2)-H-$(3)-$(4).log \
$$(call CRATE_TEST_BENCH_ARGS,$(1),$(2),$(3),$(4)))' \
$$(call CRATE_TEST_EXTRA_ARGS,$(1),$(2),$(3),$(4)))' \
> tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp
@cat tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).tmp
@touch tmp/check-stage$(1)-T-$(2)-H-$(3)-$(4).log

View File

@ -68,6 +68,11 @@ pub struct config {
// Percent change in metrics to consider noise
ratchet_noise_percent: Option<f64>,
// "Shard" of the testsuite to run: this has the form of
// two numbers (a,b), and causes only those tests with
// positional order equal to a mod b to run.
test_shard: Option<(uint,uint)>,
// A command line to prefix program execution with,
// for running under valgrind
runtool: Option<~str>,

View File

@ -75,6 +75,7 @@ pub fn parse_config(args: ~[~str]) -> config {
optopt("", "target", "the target to build for", "TARGET"),
optopt("", "adb-path", "path to the android debugger", "PATH"),
optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
optopt("", "test-shard", "run shard A, of B shards, worth of the testsuite", "A.B"),
optflag("h", "help", "show this message"),
];
@ -148,6 +149,7 @@ pub fn parse_config(args: ~[~str]) -> config {
~"") { true }
else { false }
} else { false },
test_shard: test::opt_shard(getopts::opt_maybe_str(matches, "test-shard")),
verbose: getopts::opt_present(matches, "verbose")
}
}
@ -172,6 +174,10 @@ pub fn log_config(config: &config) {
logv(c, fmt!("adb_path: %s", config.adb_path));
logv(c, fmt!("adb_test_dir: %s", config.adb_test_dir));
logv(c, fmt!("adb_device_status: %b", config.adb_device_status));
match config.test_shard {
None => logv(c, ~"test_shard: (all)"),
Some((a,b)) => logv(c, fmt!("test_shard: %u.%u", a, b))
}
logv(c, fmt!("verbose: %b", config.verbose));
logv(c, fmt!("\n"));
}
@ -234,6 +240,7 @@ pub fn test_opts(config: &config) -> test::TestOpts {
ratchet_metrics: config.ratchet_metrics.clone(),
ratchet_noise_percent: config.ratchet_noise_percent.clone(),
save_metrics: config.save_metrics.clone(),
test_shard: config.test_shard.clone()
}
}

View File

@ -38,6 +38,7 @@ use std::task;
use std::to_str::ToStr;
use std::f64;
use std::os;
use std::uint;
// The name of a test. By convention this follows the rules for rust
@ -164,6 +165,7 @@ pub struct TestOpts {
ratchet_metrics: Option<Path>,
ratchet_noise_percent: Option<f64>,
save_metrics: Option<Path>,
test_shard: Option<(uint,uint)>,
logfile: Option<Path>
}
@ -184,7 +186,9 @@ fn optgroups() -> ~[getopts::groups::OptGroup] {
"Tests within N% of the recorded metrics will be \
considered as passing", "PERCENTAGE"),
groups::optopt("", "logfile", "Write logs to the specified file instead \
of stdout", "PATH")]
of stdout", "PATH"),
groups::optopt("", "test-shard", "run shard A, of B shards, worth of the testsuite",
"A.B")]
}
fn usage(binary: &str, helpstr: &str) -> ! {
@ -255,6 +259,9 @@ pub fn parse_opts(args: &[~str]) -> OptRes {
let save_metrics = getopts::opt_maybe_str(&matches, "save-metrics");
let save_metrics = save_metrics.map_move(|s| Path(s));
let test_shard = getopts::opt_maybe_str(&matches, "test-shard");
let test_shard = opt_shard(test_shard);
let test_opts = TestOpts {
filter: filter,
run_ignored: run_ignored,
@ -263,12 +270,29 @@ pub fn parse_opts(args: &[~str]) -> OptRes {
ratchet_metrics: ratchet_metrics,
ratchet_noise_percent: ratchet_noise_percent,
save_metrics: save_metrics,
test_shard: test_shard,
logfile: logfile
};
either::Left(test_opts)
}
pub fn opt_shard(maybestr: Option<~str>) -> Option<(uint,uint)> {
match maybestr {
None => None,
Some(s) => {
match s.split_iter('.').to_owned_vec() {
[a, b] => match (uint::from_str(a), uint::from_str(b)) {
(Some(a), Some(b)) => Some((a,b)),
_ => None
},
_ => None
}
}
}
}
#[deriving(Clone, Eq)]
pub struct BenchSamples {
ns_iter_summ: stats::Summary,
@ -772,7 +796,15 @@ pub fn filter_tests(
}
sort::quick_sort(filtered, lteq);
filtered
// Shard the remaining tests, if sharding requested.
match opts.test_shard {
None => filtered,
Some((a,b)) =>
filtered.move_iter().enumerate()
.filter(|&(i,_)| i % b == a)
.map(|(_,t)| t)
.to_owned_vec()
}
}
struct TestFuture {
@ -1234,6 +1266,7 @@ mod tests {
ratchet_noise_percent: None,
ratchet_metrics: None,
save_metrics: None,
test_shard: None
};
let tests = ~[
@ -1272,6 +1305,7 @@ mod tests {
ratchet_noise_percent: None,
ratchet_metrics: None,
save_metrics: None,
test_shard: None
};
let names =