From 2e07c9f55ecabdf78746b9808fec6ab549a07f8a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 7 Feb 2017 11:49:21 -0500 Subject: [PATCH 1/5] ktest: Have POST_TEST run after the test has totally completed The POST_TEST config is to be executed after a test has fully compeleted, whether the test passed or failed. It currently is executed at the moment that the test has been decided if it failed or not. As the test does other clean ups, it isn't truly finished. Move the POST_TEST execution to after all the test cleanups have been done. Signed-off-by: Steven Rostedt (VMware) --- tools/testing/ktest/ktest.pl | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index d08e214ec6e7..caa90d391a96 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1412,6 +1412,10 @@ sub dodie { system("stty $stty_orig"); } + if (defined($post_test)) { + run_command $post_test; + } + die @_, "\n"; } @@ -1624,10 +1628,6 @@ sub save_logs { sub fail { - if (defined($post_test)) { - run_command $post_test; - } - if ($die_on_failure) { dodie @_; } @@ -1660,6 +1660,10 @@ sub fail { save_logs "fail", $store_failures; } + if (defined($post_test)) { + run_command $post_test; + } + return 1; } @@ -2489,10 +2493,6 @@ sub halt { sub success { my ($i) = @_; - if (defined($post_test)) { - run_command $post_test; - } - $successes++; my $name = ""; @@ -2517,6 +2517,10 @@ sub success { doprint "Reboot and wait $sleep_time seconds\n"; reboot_to_good $sleep_time; } + + if (defined($post_test)) { + run_command $post_test; + } } sub answer_bisect { From 32677207dcc5e594254b7fb4fb2352b1755b1d5b Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 7 Feb 2017 12:05:25 -0500 Subject: [PATCH 2/5] ktest: Fix child exit code processing The child_exit errno needs to be shifted by 8 bits to compare against the return values for the bisect variables. Fixes: c5dacb88f0a64 ("ktest: Allow overriding bisect test results") Cc: stable@vger.kernel.org Signed-off-by: Steven Rostedt (VMware) --- tools/testing/ktest/ktest.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index caa90d391a96..a64da242b824 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -2633,7 +2633,7 @@ sub do_run_test { } waitpid $child_pid, 0; - $child_exit = $?; + $child_exit = $? >> 8; my $end_time = time; $test_time = $end_time - $start_time; From 6e98d1b4415fe681ceb245e1374ed5e1942a332b Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 7 Feb 2017 12:22:03 -0500 Subject: [PATCH 3/5] ktest: Add timeout to ssh command Add a timeout to performing an ssh command. This will let testing if a machine is alive or not, or if something else may be amiss. A timeout can be passed to ssh, where ssh will fail if it does not complete within the given timeout. Signed-off-by: Steven Rostedt (VMware) --- tools/testing/ktest/ktest.pl | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index a64da242b824..d9bdd3d6dba6 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1668,19 +1668,18 @@ sub fail { } sub run_command { - my ($command, $redirect) = @_; + my ($command, $redirect, $timeout) = @_; my $start_time; my $end_time; my $dolog = 0; my $dord = 0; my $pid; - $start_time = time; - $command =~ s/\$SSH_USER/$ssh_user/g; $command =~ s/\$MACHINE/$machine/g; doprint("$command ... "); + $start_time = time; $pid = open(CMD, "$command 2>&1 |") or (fail "unable to exec $command" and return 0); @@ -1697,14 +1696,34 @@ sub run_command { $dord = 1; } - while () { - print LOG if ($dolog); - print RD if ($dord); + my $hit_timeout = 0; + + while (1) { + my $fp = \*CMD; + if (defined($timeout)) { + doprint "timeout = $timeout\n"; + } + my $line = wait_for_input($fp, $timeout); + if (!defined($line)) { + my $now = time; + if (defined($timeout) && (($now - $start_time) >= $timeout)) { + doprint "Hit timeout of $timeout, killing process\n"; + $hit_timeout = 1; + kill 9, $pid; + } + last; + } + print LOG $line if ($dolog); + print RD $line if ($dord); } waitpid($pid, 0); my $failed = $?; + if ($hit_timeout) { + $failed = 1; + } + close(CMD); close(LOG) if ($dolog); close(RD) if ($dord); @@ -1728,11 +1747,11 @@ sub run_command { } sub run_ssh { - my ($cmd) = @_; + my ($cmd, $timeout) = @_; my $cp_exec = $ssh_exec; $cp_exec =~ s/\$SSH_COMMAND/$cmd/g; - return run_command "$cp_exec"; + return run_command "$cp_exec", undef , $timeout; } sub run_scp { From 6474ace999edd714151c04c0b0534b89c2e9b6d8 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 7 Feb 2017 12:13:08 -0500 Subject: [PATCH 4/5] ktest.pl: Powercycle the box on reboot if no connection can be made When performing a reboot of the test box, try to ssh to it. If it can't connect for 5 seconds, then powercycle the box. This is useful because the reboot is done via ssh, and if you can't ssh to the box because it is hung, the reboot fails to reboot. Signed-off-by: Steven Rostedt (VMware) --- tools/testing/ktest/ktest.pl | 40 ++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index d9bdd3d6dba6..6a1484cc4436 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1325,26 +1325,44 @@ sub wait_for_monitor; sub reboot { my ($time) = @_; + my $powercycle = 0; - # Make sure everything has been written to disk - run_ssh("sync"); + # test if the machine can be connected to within 5 seconds + my $stat = run_ssh("echo check machine status", 5); + if (!$stat) { + doprint("power cycle\n"); + $powercycle = 1; + } + + if ($powercycle) { + run_command "$power_cycle"; - if (defined($time)) { start_monitor; # flush out current monitor # May contain the reboot success line wait_for_monitor 1; - } - # try to reboot normally - if (run_command $reboot) { - if (defined($powercycle_after_reboot)) { - sleep $powercycle_after_reboot; + } else { + # Make sure everything has been written to disk + run_ssh("sync"); + + if (defined($time)) { + start_monitor; + # flush out current monitor + # May contain the reboot success line + wait_for_monitor 1; + } + + # try to reboot normally + if (run_command $reboot) { + if (defined($powercycle_after_reboot)) { + sleep $powercycle_after_reboot; + run_command "$power_cycle"; + } + } else { + # nope? power cycle it. run_command "$power_cycle"; } - } else { - # nope? power cycle it. - run_command "$power_cycle"; } if (defined($time)) { From 5739438b725c09938bb6af9e4477c4283b8326cf Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (VMware)" Date: Tue, 7 Feb 2017 14:50:03 -0500 Subject: [PATCH 5/5] ktest: Add variable run_command_status to save status of commands executed Create a variable called run_command_status that saves the status of the executed commands and can be used by other functions later to test for status. Signed-off-by: Steven Rostedt (VMware) --- tools/testing/ktest/ktest.pl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 6a1484cc4436..29470b554711 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -179,6 +179,7 @@ my $localversion; my $iteration = 0; my $successes = 0; my $stty_orig; +my $run_command_status = 0; my $bisect_good; my $bisect_bad; @@ -1736,11 +1737,8 @@ sub run_command { } waitpid($pid, 0); - my $failed = $?; - - if ($hit_timeout) { - $failed = 1; - } + # shift 8 for real exit status + $run_command_status = $? >> 8; close(CMD); close(LOG) if ($dolog); @@ -1755,13 +1753,17 @@ sub run_command { doprint "[$delta seconds] "; } - if ($failed) { + if ($hit_timeout) { + $run_command_status = 1; + } + + if ($run_command_status) { doprint "FAILED!\n"; } else { doprint "SUCCESS\n"; } - return !$failed; + return !$run_command_status; } sub run_ssh { @@ -2578,16 +2580,15 @@ sub answer_bisect { } sub child_run_test { - my $failed = 0; # child should have no power $reboot_on_error = 0; $poweroff_on_error = 0; $die_on_failure = 1; - run_command $run_test, $testlog or $failed = 1; + run_command $run_test, $testlog; - exit $failed; + exit $run_command_status; } my $child_done; @@ -3371,7 +3372,6 @@ sub config_bisect { save_config \%good_configs, $good_config; save_config \%bad_configs, $bad_config; - if (defined($config_bisect_check) && $config_bisect_check ne "0") { if ($config_bisect_check ne "good") { doprint "Testing bad config\n";