From 319ab14f051aeb04d6da20d82b389ad875083041 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 30 Jan 2013 12:25:38 -0500 Subject: ktest: Do not require CONSOLE for build or install bisects If the user is doing a build or install bisect, there's no reason to have them define CONSOLE, as the console does not need to be read. The console only needs to be read for boot tests. CONSOLE is not required for normal build or install tests, let's not require it for bisect tests with BISECT_TYPE of build or install. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 35fc584a4ffe..d6690df5e7f1 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -618,6 +618,18 @@ sub set_value { if ($buildonly && $lvalue =~ /^TEST_TYPE(\[.*\])?$/ && $prvalue ne "build") { # Note if a test is something other than build, then we # will need other manditory options. + if ($prvalue ne "install") { + # for bisect, we need to check BISECT_TYPE + if ($prvalue ne "bisect") { + $buildonly = 0; + } + } else { + # install still limits some manditory options. + $buildonly = 2; + } + } + + if ($buildonly && $lvalue =~ /^BISECT_TYPE(\[.*\])?$/ && $prvalue ne "build") { if ($prvalue ne "install") { $buildonly = 0; } else { @@ -1279,6 +1291,7 @@ sub start_monitor { } sub end_monitor { + return if (!defined $console); if (--$monitor_cnt) { return; } @@ -1585,7 +1598,7 @@ sub wait_for_input $rin = ''; vec($rin, fileno($fp), 1) = 1; - $ready = select($rin, undef, undef, $time); + ($ready, $time) = select($rin, undef, undef, $time); $line = ""; @@ -1891,15 +1904,19 @@ sub get_version { sub start_monitor_and_boot { # Make sure the stable kernel has finished booting - start_monitor; - wait_for_monitor 5; - end_monitor; + + # Install bisects, don't need console + if (defined $console) { + start_monitor; + wait_for_monitor 5; + end_monitor; + } get_grub_index; get_version; install; - start_monitor; + start_monitor if (defined $console); return monitor; } -- cgit v1.2.2 From 35275685bf6123529e67c1dc91b8c05e479124e8 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 30 Jan 2013 12:28:15 -0500 Subject: ktest: Strip off '\n' when reading which files were modified The patchcheck test looks at what files are modified for each patch it checks and makes sure that those files do not produce any warnings. Unfortunately, when it read the diffstat, the newlines were added on the files and this made compares miss warnings, and commits that should not have passed, ktest let pass. Fix this by using the perl command "chomp" that strips off whitespace at the end of lines. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tools') diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index d6690df5e7f1..cc9925a68f2a 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1925,6 +1925,10 @@ sub check_buildlog { my @files = `git show $patch | diffstat -l`; + foreach my $file (@files) { + chomp $file; + } + open(IN, "git show $patch |") or dodie "failed to show $patch"; while () { -- cgit v1.2.2 From 04262be3db53d2b77ec09fa3e4d18313b6b9dcf9 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Thu, 31 Jan 2013 10:12:20 -0500 Subject: ktest: Allow a test option to use its default option Options are allowed to use other options, for example: LOG_FILE = ${OUTPUT_DIR}/${MACHINE}.log where the option LOG_FILE used the options OUTPUT_DIR and MACHINE. But if a test option were to use a default option, it will not get substituted: OUTPUT_DIR = ${THIS_DIR}/${MACHINE} TEST_START OUTPUT_DIR = ${OUTPUT_DIR}/t1 For the above test, OUTPUT_DIR will stay literally "${OUTPUT_DIR}/t1" and not be converted to "${THIS_DIR}/${MACHINE}/t1". When the test runs, it will pass the ${OUTPUT_DIR} to the shell, which would probaly interpret it as "", and the output directory will end up as "/t1". Change the code where if a test option has its own option name in its defined field, and a default option exists, then substitute the default option in its place. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index cc9925a68f2a..3d19ee445249 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1074,7 +1074,7 @@ sub read_config { } sub __eval_option { - my ($option, $i) = @_; + my ($name, $option, $i) = @_; # Add space to evaluate the character before $ $option = " $option"; @@ -1106,7 +1106,11 @@ sub __eval_option { my $o = "$var\[$i\]"; my $parento = "$var\[$parent\]"; - if (defined($opt{$o})) { + # If a variable contains itself, use the default var + if (($var eq $name) && defined($opt{$var})) { + $o = $opt{$var}; + $retval = "$retval$o"; + } elsif (defined($opt{$o})) { $o = $opt{$o}; $retval = "$retval$o"; } elsif ($repeated && defined($opt{$parento})) { @@ -1130,7 +1134,7 @@ sub __eval_option { } sub eval_option { - my ($option, $i) = @_; + my ($name, $option, $i) = @_; my $prev = ""; @@ -1146,7 +1150,7 @@ sub eval_option { "Check for recursive variables\n"; } $prev = $option; - $option = __eval_option($option, $i); + $option = __eval_option($name, $option, $i); } return $option; @@ -3683,7 +3687,7 @@ EOF read_config $ktest_config; if (defined($opt{"LOG_FILE"})) { - $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1); + $opt{"LOG_FILE"} = eval_option("LOG_FILE", $opt{"LOG_FILE"}, -1); } # Append any configs entered in manually to the config file. @@ -3760,7 +3764,7 @@ sub set_test_option { my $option = __set_test_option($name, $i); return $option if (!defined($option)); - return eval_option($option, $i); + return eval_option($name, $option, $i); } # First we need to do is the builds -- cgit v1.2.2 From 4283b169abfb0380850b56287ee644697ecf321a Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Wed, 30 Jan 2013 18:37:47 -0500 Subject: ktest: Add make_warnings_file and process full warnings Although the patchcheck test checks for warnings in the files that were changed, this check does not catch warnings that were caused by header file changes and the warnings appear in C files not touched by the commit. Add a new option called WARNINGS_FILE. If this option is set, then the file it points to is read before bulid, and the file should contain a list of known warnings. If a warning appears in the build, this file is checked, and if the warning does not exist in this file, then it fails the build showing the new warning. If the WARNINGS_FILE points to a file that does not exist, this will cause any warning in the build to fail. A new test is also added called "make_warnings_file". This test will create do a build and record any warnings it finds into the WARNINGS_FILE. This test is something that can be run before other tests to build a warnings file of "known warnings", ie, warnings that were there before your changes. Signed-off-by: Steven Rostedt --- .../testing/ktest/examples/include/patchcheck.conf | 37 +++++++ tools/testing/ktest/ktest.pl | 110 +++++++++++++++++++-- tools/testing/ktest/sample.conf | 44 +++++++++ 3 files changed, 185 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/testing/ktest/examples/include/patchcheck.conf b/tools/testing/ktest/examples/include/patchcheck.conf index 339d3e1700ff..0eb0a5ac77da 100644 --- a/tools/testing/ktest/examples/include/patchcheck.conf +++ b/tools/testing/ktest/examples/include/patchcheck.conf @@ -14,6 +14,16 @@ PATCH_START := HEAD~3 PATCH_END := HEAD +# Use the oldconfig if build_type wasn't defined +DEFAULTS IF NOT DEFINED BUILD_TYPE +DO_BUILD_TYPE := oldconfig + +DEFAULTS ELSE +DO_BUILD_TYPE := ${BUILD_TYPE} + +DEFAULTS + + # Change PATCH_CHECKOUT to be the branch you want to test. The test will # do a git checkout of this branch before starting. Obviously both # PATCH_START and PATCH_END must be in this branch (and PATCH_START must @@ -43,6 +53,31 @@ PATCH_TEST_TYPE := boot # (space delimited) #IGNORE_WARNINGS = 39eaf7ef884dcc44f7ff1bac803ca2a1dcf43544 6edb2a8a385f0cdef51dae37ff23e74d76d8a6ce +# Instead of just checking for warnings to files that are changed +# it can be advantageous to check for any new warnings. If a +# header file is changed, it could cause a warning in a file not +# touched by the commit. To detect these kinds of warnings, you +# can use the WARNINGS_FILE option. +# +# If the variable CREATE_WARNINGS_FILE is set, this config will +# enable the WARNINGS_FILE during the patchcheck test. Also, +# before running the patchcheck test, it will create the +# warnings file. +# +DEFAULTS IF DEFINED CREATE_WARNINGS_FILE +WARNINGS_FILE = ${OUTPUT_DIR}/warnings_file + +TEST_START IF DEFINED CREATE_WARNINGS_FILE +# WARNINGS_FILE is already set by the DEFAULTS above +TEST_TYPE = make_warnings_file +# Checkout the commit before the patches to test, +# and record all the warnings that exist before the patches +# to test are added +CHECKOUT = ${PATCHCHECK_START}~1 +# Force a full build +BUILD_NOCLEAN = 0 +BUILD_TYPE = ${DO_BUILD_TYPE} + # If you are running a multi test, and the test failed on the first # test but on, say the 5th patch. If you want to restart on the # fifth patch, set PATCH_START1. This will make the first test start @@ -61,6 +96,7 @@ PATCHCHECK_TYPE = ${PATCH_TEST_TYPE} PATCHCHECK_START = ${PATCH_START1} PATCHCHECK_END = ${PATCH_END} CHECKOUT = ${PATCH_CHECKOUT} +BUILD_TYPE = ${DO_BUILD_TYPE} TEST_START IF ${TEST} == patchcheck && ${MULTI} TEST_TYPE = patchcheck @@ -72,3 +108,4 @@ PATCHCHECK_END = ${PATCH_END} CHECKOUT = ${PATCH_CHECKOUT} # Use multi to test different compilers? MAKE_CMD = CC=gcc-4.5.1 make +BUILD_TYPE = ${DO_BUILD_TYPE} diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 3d19ee445249..3fd768e65998 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -126,6 +126,7 @@ my $start_minconfig_defined; my $output_minconfig; my $minconfig_type; my $use_output_minconfig; +my $warnings_file; my $ignore_config; my $ignore_errors; my $addconfig; @@ -193,6 +194,9 @@ my $patchcheck_end; # which would require more options. my $buildonly = 1; +# tell build not to worry about warnings, even when WARNINGS_FILE is set +my $warnings_ok = 0; + # set when creating a new config my $newconfig = 0; @@ -235,6 +239,7 @@ my %option_map = ( "START_MIN_CONFIG" => \$start_minconfig, "MIN_CONFIG_TYPE" => \$minconfig_type, "USE_OUTPUT_MIN_CONFIG" => \$use_output_minconfig, + "WARNINGS_FILE" => \$warnings_file, "IGNORE_CONFIG" => \$ignore_config, "TEST" => \$run_test, "ADD_CONFIG" => \$addconfig, @@ -1924,7 +1929,60 @@ sub start_monitor_and_boot { return monitor; } +my $check_build_re = ".*:.*(warning|error|Error):.*"; +my $utf8_quote = "\\x{e2}\\x{80}(\\x{98}|\\x{99})"; + +# Read buildlog and check against warnings file for any +# new warnings. +# +# Returns 1 if OK +# 0 otherwise sub check_buildlog { + return 1 if (!defined $warnings_file); + + my %warnings_list; + + # Failed builds should not reboot the target + my $save_no_reboot = $no_reboot; + $no_reboot = 1; + + if (-f $warnings_file) { + open(IN, $warnings_file) or + dodie "Error opening $warnings_file"; + + while () { + if (/$check_build_re/) { + chomp; + $warnings_list{$_} = 1; + } + } + close(IN); + } + + # If warnings file didn't exist, and WARNINGS_FILE exist, + # then we fail on any warning! + + open(IN, $buildlog) or dodie "Can't open $buildlog"; + while () { + if (/$check_build_re/) { + + # Some compilers use UTF-8 extended for quotes + # for distcc heterogeneous systems, this causes issues + s/$utf8_quote/'/g; + + chomp; + if (!defined $warnings_list{$_}) { + fail "New warning found (not in $warnings_file)\n$_\n"; + $no_reboot = $save_no_reboot; + return 0; + } + } + } + $no_reboot = $save_no_reboot; + close(IN); +} + +sub check_patch_buildlog { my ($patch) = @_; my @files = `git show $patch | diffstat -l`; @@ -3080,11 +3138,13 @@ sub patchcheck { build "oldconfig" or return 0; } - - if (!defined($ignored_warnings{$sha1})) { - check_buildlog $sha1 or return 0; + # No need to do per patch checking if warnings file exists + if (!defined($warnings_file) && !defined($ignored_warnings{$sha1})) { + check_patch_buildlog $sha1 or return 0; } + check_buildlog or return 0; + next if ($type eq "build"); my $failed = 0; @@ -3642,6 +3702,39 @@ sub make_min_config { return 1; } +sub make_warnings_file { + my ($i) = @_; + + if (!defined($warnings_file)) { + dodie "Must define WARNINGS_FILE for make_warnings_file test"; + } + + if ($build_type eq "nobuild") { + dodie "BUILD_TYPE can not be 'nobuild' for make_warnings_file test"; + } + + build $build_type or dodie "Failed to build"; + + open(OUT, ">$warnings_file") or dodie "Can't create $warnings_file"; + + open(IN, $buildlog) or dodie "Can't open $buildlog"; + while () { + + # Some compilers use UTF-8 extended for quotes + # for distcc heterogeneous systems, this causes issues + s/$utf8_quote/'/g; + + if (/$check_build_re/) { + print OUT; + } + } + close(IN); + + close(OUT); + + success $i; +} + $#ARGV < 1 or die "ktest.pl version: $VERSION\n usage: ktest.pl config-file\n"; if ($#ARGV == 0) { @@ -3843,9 +3936,9 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { $run_type = $bisect_type; } elsif ($test_type eq "config_bisect") { $run_type = $config_bisect_type; - } - - if ($test_type eq "make_min_config") { + } elsif ($test_type eq "make_min_config") { + $run_type = ""; + } elsif ($test_type eq "make_warnings_file") { $run_type = ""; } @@ -3902,10 +3995,15 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) { } elsif ($test_type eq "make_min_config") { make_min_config $i; next; + } elsif ($test_type eq "make_warnings_file") { + $no_reboot = 1; + make_warnings_file $i; + next; } if ($build_type ne "nobuild") { build $build_type or next; + check_buildlog or next; } if ($test_type eq "install") { diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf index 4012e9330344..0a290fb4cd5e 100644 --- a/tools/testing/ktest/sample.conf +++ b/tools/testing/ktest/sample.conf @@ -793,6 +793,20 @@ # Example for a virtual guest call "Guest". #POWER_OFF = virsh destroy Guest +# To have the build fail on "new" warnings, create a file that +# contains a list of all known warnings (they must match exactly +# to the line with 'warning:', 'error:' or 'Error:'. If the option +# WARNINGS_FILE is set, then that file will be read, and if the +# build detects a warning, it will examine this file and if the +# warning does not exist in it, it will fail the build. +# +# Note, if this option is defined to a file that does not exist +# then any warning will fail the build. +# (see make_warnings_file below) +# +# (optional, default undefined) +#WARNINGS_FILE = ${OUTPUT_DIR}/warnings_file + # The way to execute a command on the target # (default ssh $SSH_USER@$MACHINE $SSH_COMMAND";) # The variables SSH_USER, MACHINE and SSH_COMMAND are defined @@ -1222,3 +1236,33 @@ # MIN_CONFIG_TYPE = test # TEST = ssh ${USER}@${MACHINE} echo hi # +# +# +# +# For TEST_TYPE = make_warnings_file +# +# If you want the build to fail when a new warning is discovered +# you set the WARNINGS_FILE to point to a file of known warnings. +# +# The test "make_warnings_file" will let you create a new warnings +# file before you run other tests, like patchcheck. +# +# What this test does is to run just a build, you still need to +# specify BUILD_TYPE to tell the test what type of config to use. +# A BUILD_TYPE of nobuild will fail this test. +# +# The test will do the build and scan for all warnings. Any warning +# it discovers will be saved in the WARNINGS_FILE (required) option. +# +# It is recommended (but not necessary) to make sure BUILD_NOCLEAN is +# off, so that a full build is done (make mrproper is performed). +# That way, all warnings will be captured. +# +# Example: +# +# TEST_TYPE = make_warnings_file +# WARNINGS_FILE = ${OUTPUT_DIR} +# BUILD_TYPE = useconfig:oldconfig +# CHECKOUT = v3.8 +# BUILD_NOCLEAN = 0 +# -- cgit v1.2.2 From d6845536236a72382a20483887943a599d7c2b69 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Mon, 4 Feb 2013 23:08:49 -0500 Subject: ktest: Search for linux banner for successful reboot Sometimes when a test kernel passed fine, but on reboot it crashed, ktest could get stuck and not proceed. This would be frustrating if you let a test run overnight to find out the next morning that it was stuck on the first test. To fix this, I made reboot check for the REBOOT_SUCCESS_LINE. If the line was not detected, then it would power cycle the box. What it didn't cover was if the REBOOT_SUCCESS_LINE wasn't defined or if a 'good' kernel did not display the line. Instead have it search for the Linux banner "Linux version". The reboot just needs to get to the start of the next kernel, it does not need to test if the next kernel makes it to a boot prompt. After we find the next kernel has booted, then we just wait for either the REBOOT_SUCCESS_LINE to appear or the timeout. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 3fd768e65998..839989d08dc2 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1212,11 +1212,16 @@ sub reboot { } if (defined($time)) { - if (wait_for_monitor($time, $reboot_success_line)) { + # Look for the good kernel to boot + if (wait_for_monitor($time, "Linux version")) { # reboot got stuck? doprint "Reboot did not finish. Forcing power cycle\n"; run_command "$power_cycle"; } + + # Still need to wait for the reboot to finish + wait_for_monitor($time, $reboot_success_line); + end_monitor; } } -- cgit v1.2.2 From 4c0b67a27d96e01a4b4ede2fda57da9f7c50af21 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Tue, 5 Feb 2013 09:56:00 -0500 Subject: ktest: Ignore warnings during reboot The reboot just wants to get to the next kernel. But if a warning (Call Trace) appears, the monitor will report an error, and the reboot will think something went wrong and power cycle the box, even though we successfully made it to the next kernel. Ignore warnings during the reboot until we get to the next kernel. It will still timeout if we never get to the next kernel and then a power cycle will happen. That's what we want it to do. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'tools') diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 839989d08dc2..f3bb5da3193b 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1212,6 +1212,12 @@ sub reboot { } if (defined($time)) { + + # We only want to get to the new kernel, don't fail + # if we stumble over a call trace. + my $save_ignore_errors = $ignore_errors; + $ignore_errors = 1; + # Look for the good kernel to boot if (wait_for_monitor($time, "Linux version")) { # reboot got stuck? @@ -1219,6 +1225,8 @@ sub reboot { run_command "$power_cycle"; } + $ignore_errors = $save_ignore_errors; + # Still need to wait for the reboot to finish wait_for_monitor($time, $reboot_success_line); -- cgit v1.2.2 From 7328735cbf68b7cd4d7ef16e172013743cdc2bc4 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Red Hat)" Date: Mon, 18 Feb 2013 09:35:49 -0500 Subject: ktest: Remove indexes from warnings check The index of a line where a warning is tested can be returned differently on different versions of gcc (or same version compiled differently). That is, a tab + space can give different results. This causes the warning check to produce a false positive. Removing the index from the check fixes this issue. Signed-off-by: Steven Rostedt --- tools/testing/ktest/ktest.pl | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'tools') diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index f3bb5da3193b..4e67d52eb3a2 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -1945,6 +1945,27 @@ sub start_monitor_and_boot { my $check_build_re = ".*:.*(warning|error|Error):.*"; my $utf8_quote = "\\x{e2}\\x{80}(\\x{98}|\\x{99})"; +sub process_warning_line { + my ($line) = @_; + + chomp $line; + + # for distcc heterogeneous systems, some compilers + # do things differently causing warning lines + # to be slightly different. This makes an attempt + # to fixe those issues. + + # chop off the index into the line + # using distcc, some compilers give different indexes + # depending on white space + $line =~ s/^(\s*\S+:\d+:)\d+/$1/; + + # Some compilers use UTF-8 extended for quotes and some don't. + $line =~ s/$utf8_quote/'/g; + + return $line; +} + # Read buildlog and check against warnings file for any # new warnings. # @@ -1965,8 +1986,9 @@ sub check_buildlog { while () { if (/$check_build_re/) { - chomp; - $warnings_list{$_} = 1; + my $warning = process_warning_line $_; + + $warnings_list{$warning} = 1; } } close(IN); @@ -1978,13 +2000,9 @@ sub check_buildlog { open(IN, $buildlog) or dodie "Can't open $buildlog"; while () { if (/$check_build_re/) { + my $warning = process_warning_line $_; - # Some compilers use UTF-8 extended for quotes - # for distcc heterogeneous systems, this causes issues - s/$utf8_quote/'/g; - - chomp; - if (!defined $warnings_list{$_}) { + if (!defined $warnings_list{$warning}) { fail "New warning found (not in $warnings_file)\n$_\n"; $no_reboot = $save_no_reboot; return 0; -- cgit v1.2.2