aboutsummaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2013-02-25 18:43:21 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2013-02-25 18:43:21 -0500
commitc69d0a15d240a854f6d620ac7d5238a41883a920 (patch)
tree61a1014167d33e9a8e8bd08a8d3f0b736d57aa03 /tools
parent9043a2650cd21f96f831a97f516c2c302e21fb70 (diff)
parent7328735cbf68b7cd4d7ef16e172013743cdc2bc4 (diff)
Merge tag 'ktest-v3.9' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest
Pull ktest update from Steven Rostedt: "Added ability to have all builds test warnings. Fixed failing reboot when the reboot produces a non fatal error. Config reading fixes and other cleanups" * tag 'ktest-v3.9' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest: ktest: Remove indexes from warnings check ktest: Ignore warnings during reboot ktest: Search for linux banner for successful reboot ktest: Add make_warnings_file and process full warnings ktest: Allow a test option to use its default option ktest: Strip off '\n' when reading which files were modified ktest: Do not require CONSOLE for build or install bisects
Diffstat (limited to 'tools')
-rw-r--r--tools/testing/ktest/examples/include/patchcheck.conf37
-rwxr-xr-xtools/testing/ktest/ktest.pl190
-rw-r--r--tools/testing/ktest/sample.conf44
3 files changed, 253 insertions, 18 deletions
diff --git a/tools/testing/ktest/examples/include/patchcheck.conf b/tools/testing/ktest/examples/include/patchcheck.conf
index 339d3e1700ff..0eb0a5ac77da 100644
--- a/tools/testing/ktest/examples/include/patchcheck.conf
+++ b/tools/testing/ktest/examples/include/patchcheck.conf
@@ -14,6 +14,16 @@
14PATCH_START := HEAD~3 14PATCH_START := HEAD~3
15PATCH_END := HEAD 15PATCH_END := HEAD
16 16
17# Use the oldconfig if build_type wasn't defined
18DEFAULTS IF NOT DEFINED BUILD_TYPE
19DO_BUILD_TYPE := oldconfig
20
21DEFAULTS ELSE
22DO_BUILD_TYPE := ${BUILD_TYPE}
23
24DEFAULTS
25
26
17# Change PATCH_CHECKOUT to be the branch you want to test. The test will 27# Change PATCH_CHECKOUT to be the branch you want to test. The test will
18# do a git checkout of this branch before starting. Obviously both 28# do a git checkout of this branch before starting. Obviously both
19# PATCH_START and PATCH_END must be in this branch (and PATCH_START must 29# PATCH_START and PATCH_END must be in this branch (and PATCH_START must
@@ -43,6 +53,31 @@ PATCH_TEST_TYPE := boot
43# (space delimited) 53# (space delimited)
44#IGNORE_WARNINGS = 39eaf7ef884dcc44f7ff1bac803ca2a1dcf43544 6edb2a8a385f0cdef51dae37ff23e74d76d8a6ce 54#IGNORE_WARNINGS = 39eaf7ef884dcc44f7ff1bac803ca2a1dcf43544 6edb2a8a385f0cdef51dae37ff23e74d76d8a6ce
45 55
56# Instead of just checking for warnings to files that are changed
57# it can be advantageous to check for any new warnings. If a
58# header file is changed, it could cause a warning in a file not
59# touched by the commit. To detect these kinds of warnings, you
60# can use the WARNINGS_FILE option.
61#
62# If the variable CREATE_WARNINGS_FILE is set, this config will
63# enable the WARNINGS_FILE during the patchcheck test. Also,
64# before running the patchcheck test, it will create the
65# warnings file.
66#
67DEFAULTS IF DEFINED CREATE_WARNINGS_FILE
68WARNINGS_FILE = ${OUTPUT_DIR}/warnings_file
69
70TEST_START IF DEFINED CREATE_WARNINGS_FILE
71# WARNINGS_FILE is already set by the DEFAULTS above
72TEST_TYPE = make_warnings_file
73# Checkout the commit before the patches to test,
74# and record all the warnings that exist before the patches
75# to test are added
76CHECKOUT = ${PATCHCHECK_START}~1
77# Force a full build
78BUILD_NOCLEAN = 0
79BUILD_TYPE = ${DO_BUILD_TYPE}
80
46# If you are running a multi test, and the test failed on the first 81# If you are running a multi test, and the test failed on the first
47# test but on, say the 5th patch. If you want to restart on the 82# test but on, say the 5th patch. If you want to restart on the
48# fifth patch, set PATCH_START1. This will make the first test start 83# fifth patch, set PATCH_START1. This will make the first test start
@@ -61,6 +96,7 @@ PATCHCHECK_TYPE = ${PATCH_TEST_TYPE}
61PATCHCHECK_START = ${PATCH_START1} 96PATCHCHECK_START = ${PATCH_START1}
62PATCHCHECK_END = ${PATCH_END} 97PATCHCHECK_END = ${PATCH_END}
63CHECKOUT = ${PATCH_CHECKOUT} 98CHECKOUT = ${PATCH_CHECKOUT}
99BUILD_TYPE = ${DO_BUILD_TYPE}
64 100
65TEST_START IF ${TEST} == patchcheck && ${MULTI} 101TEST_START IF ${TEST} == patchcheck && ${MULTI}
66TEST_TYPE = patchcheck 102TEST_TYPE = patchcheck
@@ -72,3 +108,4 @@ PATCHCHECK_END = ${PATCH_END}
72CHECKOUT = ${PATCH_CHECKOUT} 108CHECKOUT = ${PATCH_CHECKOUT}
73# Use multi to test different compilers? 109# Use multi to test different compilers?
74MAKE_CMD = CC=gcc-4.5.1 make 110MAKE_CMD = CC=gcc-4.5.1 make
111BUILD_TYPE = ${DO_BUILD_TYPE}
diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl
index 35fc584a4ffe..4e67d52eb3a2 100755
--- a/tools/testing/ktest/ktest.pl
+++ b/tools/testing/ktest/ktest.pl
@@ -126,6 +126,7 @@ my $start_minconfig_defined;
126my $output_minconfig; 126my $output_minconfig;
127my $minconfig_type; 127my $minconfig_type;
128my $use_output_minconfig; 128my $use_output_minconfig;
129my $warnings_file;
129my $ignore_config; 130my $ignore_config;
130my $ignore_errors; 131my $ignore_errors;
131my $addconfig; 132my $addconfig;
@@ -193,6 +194,9 @@ my $patchcheck_end;
193# which would require more options. 194# which would require more options.
194my $buildonly = 1; 195my $buildonly = 1;
195 196
197# tell build not to worry about warnings, even when WARNINGS_FILE is set
198my $warnings_ok = 0;
199
196# set when creating a new config 200# set when creating a new config
197my $newconfig = 0; 201my $newconfig = 0;
198 202
@@ -235,6 +239,7 @@ my %option_map = (
235 "START_MIN_CONFIG" => \$start_minconfig, 239 "START_MIN_CONFIG" => \$start_minconfig,
236 "MIN_CONFIG_TYPE" => \$minconfig_type, 240 "MIN_CONFIG_TYPE" => \$minconfig_type,
237 "USE_OUTPUT_MIN_CONFIG" => \$use_output_minconfig, 241 "USE_OUTPUT_MIN_CONFIG" => \$use_output_minconfig,
242 "WARNINGS_FILE" => \$warnings_file,
238 "IGNORE_CONFIG" => \$ignore_config, 243 "IGNORE_CONFIG" => \$ignore_config,
239 "TEST" => \$run_test, 244 "TEST" => \$run_test,
240 "ADD_CONFIG" => \$addconfig, 245 "ADD_CONFIG" => \$addconfig,
@@ -619,6 +624,18 @@ sub set_value {
619 # Note if a test is something other than build, then we 624 # Note if a test is something other than build, then we
620 # will need other manditory options. 625 # will need other manditory options.
621 if ($prvalue ne "install") { 626 if ($prvalue ne "install") {
627 # for bisect, we need to check BISECT_TYPE
628 if ($prvalue ne "bisect") {
629 $buildonly = 0;
630 }
631 } else {
632 # install still limits some manditory options.
633 $buildonly = 2;
634 }
635 }
636
637 if ($buildonly && $lvalue =~ /^BISECT_TYPE(\[.*\])?$/ && $prvalue ne "build") {
638 if ($prvalue ne "install") {
622 $buildonly = 0; 639 $buildonly = 0;
623 } else { 640 } else {
624 # install still limits some manditory options. 641 # install still limits some manditory options.
@@ -1062,7 +1079,7 @@ sub read_config {
1062} 1079}
1063 1080
1064sub __eval_option { 1081sub __eval_option {
1065 my ($option, $i) = @_; 1082 my ($name, $option, $i) = @_;
1066 1083
1067 # Add space to evaluate the character before $ 1084 # Add space to evaluate the character before $
1068 $option = " $option"; 1085 $option = " $option";
@@ -1094,7 +1111,11 @@ sub __eval_option {
1094 my $o = "$var\[$i\]"; 1111 my $o = "$var\[$i\]";
1095 my $parento = "$var\[$parent\]"; 1112 my $parento = "$var\[$parent\]";
1096 1113
1097 if (defined($opt{$o})) { 1114 # If a variable contains itself, use the default var
1115 if (($var eq $name) && defined($opt{$var})) {
1116 $o = $opt{$var};
1117 $retval = "$retval$o";
1118 } elsif (defined($opt{$o})) {
1098 $o = $opt{$o}; 1119 $o = $opt{$o};
1099 $retval = "$retval$o"; 1120 $retval = "$retval$o";
1100 } elsif ($repeated && defined($opt{$parento})) { 1121 } elsif ($repeated && defined($opt{$parento})) {
@@ -1118,7 +1139,7 @@ sub __eval_option {
1118} 1139}
1119 1140
1120sub eval_option { 1141sub eval_option {
1121 my ($option, $i) = @_; 1142 my ($name, $option, $i) = @_;
1122 1143
1123 my $prev = ""; 1144 my $prev = "";
1124 1145
@@ -1134,7 +1155,7 @@ sub eval_option {
1134 "Check for recursive variables\n"; 1155 "Check for recursive variables\n";
1135 } 1156 }
1136 $prev = $option; 1157 $prev = $option;
1137 $option = __eval_option($option, $i); 1158 $option = __eval_option($name, $option, $i);
1138 } 1159 }
1139 1160
1140 return $option; 1161 return $option;
@@ -1191,11 +1212,24 @@ sub reboot {
1191 } 1212 }
1192 1213
1193 if (defined($time)) { 1214 if (defined($time)) {
1194 if (wait_for_monitor($time, $reboot_success_line)) { 1215
1216 # We only want to get to the new kernel, don't fail
1217 # if we stumble over a call trace.
1218 my $save_ignore_errors = $ignore_errors;
1219 $ignore_errors = 1;
1220
1221 # Look for the good kernel to boot
1222 if (wait_for_monitor($time, "Linux version")) {
1195 # reboot got stuck? 1223 # reboot got stuck?
1196 doprint "Reboot did not finish. Forcing power cycle\n"; 1224 doprint "Reboot did not finish. Forcing power cycle\n";
1197 run_command "$power_cycle"; 1225 run_command "$power_cycle";
1198 } 1226 }
1227
1228 $ignore_errors = $save_ignore_errors;
1229
1230 # Still need to wait for the reboot to finish
1231 wait_for_monitor($time, $reboot_success_line);
1232
1199 end_monitor; 1233 end_monitor;
1200 } 1234 }
1201} 1235}
@@ -1279,6 +1313,7 @@ sub start_monitor {
1279} 1313}
1280 1314
1281sub end_monitor { 1315sub end_monitor {
1316 return if (!defined $console);
1282 if (--$monitor_cnt) { 1317 if (--$monitor_cnt) {
1283 return; 1318 return;
1284 } 1319 }
@@ -1585,7 +1620,7 @@ sub wait_for_input
1585 1620
1586 $rin = ''; 1621 $rin = '';
1587 vec($rin, fileno($fp), 1) = 1; 1622 vec($rin, fileno($fp), 1) = 1;
1588 $ready = select($rin, undef, undef, $time); 1623 ($ready, $time) = select($rin, undef, undef, $time);
1589 1624
1590 $line = ""; 1625 $line = "";
1591 1626
@@ -1891,23 +1926,102 @@ sub get_version {
1891 1926
1892sub start_monitor_and_boot { 1927sub start_monitor_and_boot {
1893 # Make sure the stable kernel has finished booting 1928 # Make sure the stable kernel has finished booting
1894 start_monitor; 1929
1895 wait_for_monitor 5; 1930 # Install bisects, don't need console
1896 end_monitor; 1931 if (defined $console) {
1932 start_monitor;
1933 wait_for_monitor 5;
1934 end_monitor;
1935 }
1897 1936
1898 get_grub_index; 1937 get_grub_index;
1899 get_version; 1938 get_version;
1900 install; 1939 install;
1901 1940
1902 start_monitor; 1941 start_monitor if (defined $console);
1903 return monitor; 1942 return monitor;
1904} 1943}
1905 1944
1945my $check_build_re = ".*:.*(warning|error|Error):.*";
1946my $utf8_quote = "\\x{e2}\\x{80}(\\x{98}|\\x{99})";
1947
1948sub process_warning_line {
1949 my ($line) = @_;
1950
1951 chomp $line;
1952
1953 # for distcc heterogeneous systems, some compilers
1954 # do things differently causing warning lines
1955 # to be slightly different. This makes an attempt
1956 # to fixe those issues.
1957
1958 # chop off the index into the line
1959 # using distcc, some compilers give different indexes
1960 # depending on white space
1961 $line =~ s/^(\s*\S+:\d+:)\d+/$1/;
1962
1963 # Some compilers use UTF-8 extended for quotes and some don't.
1964 $line =~ s/$utf8_quote/'/g;
1965
1966 return $line;
1967}
1968
1969# Read buildlog and check against warnings file for any
1970# new warnings.
1971#
1972# Returns 1 if OK
1973# 0 otherwise
1906sub check_buildlog { 1974sub check_buildlog {
1975 return 1 if (!defined $warnings_file);
1976
1977 my %warnings_list;
1978
1979 # Failed builds should not reboot the target
1980 my $save_no_reboot = $no_reboot;
1981 $no_reboot = 1;
1982
1983 if (-f $warnings_file) {
1984 open(IN, $warnings_file) or
1985 dodie "Error opening $warnings_file";
1986
1987 while (<IN>) {
1988 if (/$check_build_re/) {
1989 my $warning = process_warning_line $_;
1990
1991 $warnings_list{$warning} = 1;
1992 }
1993 }
1994 close(IN);
1995 }
1996
1997 # If warnings file didn't exist, and WARNINGS_FILE exist,
1998 # then we fail on any warning!
1999
2000 open(IN, $buildlog) or dodie "Can't open $buildlog";
2001 while (<IN>) {
2002 if (/$check_build_re/) {
2003 my $warning = process_warning_line $_;
2004
2005 if (!defined $warnings_list{$warning}) {
2006 fail "New warning found (not in $warnings_file)\n$_\n";
2007 $no_reboot = $save_no_reboot;
2008 return 0;
2009 }
2010 }
2011 }
2012 $no_reboot = $save_no_reboot;
2013 close(IN);
2014}
2015
2016sub check_patch_buildlog {
1907 my ($patch) = @_; 2017 my ($patch) = @_;
1908 2018
1909 my @files = `git show $patch | diffstat -l`; 2019 my @files = `git show $patch | diffstat -l`;
1910 2020
2021 foreach my $file (@files) {
2022 chomp $file;
2023 }
2024
1911 open(IN, "git show $patch |") or 2025 open(IN, "git show $patch |") or
1912 dodie "failed to show $patch"; 2026 dodie "failed to show $patch";
1913 while (<IN>) { 2027 while (<IN>) {
@@ -3055,11 +3169,13 @@ sub patchcheck {
3055 build "oldconfig" or return 0; 3169 build "oldconfig" or return 0;
3056 } 3170 }
3057 3171
3058 3172 # No need to do per patch checking if warnings file exists
3059 if (!defined($ignored_warnings{$sha1})) { 3173 if (!defined($warnings_file) && !defined($ignored_warnings{$sha1})) {
3060 check_buildlog $sha1 or return 0; 3174 check_patch_buildlog $sha1 or return 0;
3061 } 3175 }
3062 3176
3177 check_buildlog or return 0;
3178
3063 next if ($type eq "build"); 3179 next if ($type eq "build");
3064 3180
3065 my $failed = 0; 3181 my $failed = 0;
@@ -3617,6 +3733,39 @@ sub make_min_config {
3617 return 1; 3733 return 1;
3618} 3734}
3619 3735
3736sub make_warnings_file {
3737 my ($i) = @_;
3738
3739 if (!defined($warnings_file)) {
3740 dodie "Must define WARNINGS_FILE for make_warnings_file test";
3741 }
3742
3743 if ($build_type eq "nobuild") {
3744 dodie "BUILD_TYPE can not be 'nobuild' for make_warnings_file test";
3745 }
3746
3747 build $build_type or dodie "Failed to build";
3748
3749 open(OUT, ">$warnings_file") or dodie "Can't create $warnings_file";
3750
3751 open(IN, $buildlog) or dodie "Can't open $buildlog";
3752 while (<IN>) {
3753
3754 # Some compilers use UTF-8 extended for quotes
3755 # for distcc heterogeneous systems, this causes issues
3756 s/$utf8_quote/'/g;
3757
3758 if (/$check_build_re/) {
3759 print OUT;
3760 }
3761 }
3762 close(IN);
3763
3764 close(OUT);
3765
3766 success $i;
3767}
3768
3620$#ARGV < 1 or die "ktest.pl version: $VERSION\n usage: ktest.pl config-file\n"; 3769$#ARGV < 1 or die "ktest.pl version: $VERSION\n usage: ktest.pl config-file\n";
3621 3770
3622if ($#ARGV == 0) { 3771if ($#ARGV == 0) {
@@ -3662,7 +3811,7 @@ EOF
3662read_config $ktest_config; 3811read_config $ktest_config;
3663 3812
3664if (defined($opt{"LOG_FILE"})) { 3813if (defined($opt{"LOG_FILE"})) {
3665 $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1); 3814 $opt{"LOG_FILE"} = eval_option("LOG_FILE", $opt{"LOG_FILE"}, -1);
3666} 3815}
3667 3816
3668# Append any configs entered in manually to the config file. 3817# Append any configs entered in manually to the config file.
@@ -3739,7 +3888,7 @@ sub set_test_option {
3739 my $option = __set_test_option($name, $i); 3888 my $option = __set_test_option($name, $i);
3740 return $option if (!defined($option)); 3889 return $option if (!defined($option));
3741 3890
3742 return eval_option($option, $i); 3891 return eval_option($name, $option, $i);
3743} 3892}
3744 3893
3745# First we need to do is the builds 3894# First we need to do is the builds
@@ -3818,9 +3967,9 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
3818 $run_type = $bisect_type; 3967 $run_type = $bisect_type;
3819 } elsif ($test_type eq "config_bisect") { 3968 } elsif ($test_type eq "config_bisect") {
3820 $run_type = $config_bisect_type; 3969 $run_type = $config_bisect_type;
3821 } 3970 } elsif ($test_type eq "make_min_config") {
3822 3971 $run_type = "";
3823 if ($test_type eq "make_min_config") { 3972 } elsif ($test_type eq "make_warnings_file") {
3824 $run_type = ""; 3973 $run_type = "";
3825 } 3974 }
3826 3975
@@ -3877,10 +4026,15 @@ for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
3877 } elsif ($test_type eq "make_min_config") { 4026 } elsif ($test_type eq "make_min_config") {
3878 make_min_config $i; 4027 make_min_config $i;
3879 next; 4028 next;
4029 } elsif ($test_type eq "make_warnings_file") {
4030 $no_reboot = 1;
4031 make_warnings_file $i;
4032 next;
3880 } 4033 }
3881 4034
3882 if ($build_type ne "nobuild") { 4035 if ($build_type ne "nobuild") {
3883 build $build_type or next; 4036 build $build_type or next;
4037 check_buildlog or next;
3884 } 4038 }
3885 4039
3886 if ($test_type eq "install") { 4040 if ($test_type eq "install") {
diff --git a/tools/testing/ktest/sample.conf b/tools/testing/ktest/sample.conf
index 4012e9330344..0a290fb4cd5e 100644
--- a/tools/testing/ktest/sample.conf
+++ b/tools/testing/ktest/sample.conf
@@ -793,6 +793,20 @@
793# Example for a virtual guest call "Guest". 793# Example for a virtual guest call "Guest".
794#POWER_OFF = virsh destroy Guest 794#POWER_OFF = virsh destroy Guest
795 795
796# To have the build fail on "new" warnings, create a file that
797# contains a list of all known warnings (they must match exactly
798# to the line with 'warning:', 'error:' or 'Error:'. If the option
799# WARNINGS_FILE is set, then that file will be read, and if the
800# build detects a warning, it will examine this file and if the
801# warning does not exist in it, it will fail the build.
802#
803# Note, if this option is defined to a file that does not exist
804# then any warning will fail the build.
805# (see make_warnings_file below)
806#
807# (optional, default undefined)
808#WARNINGS_FILE = ${OUTPUT_DIR}/warnings_file
809
796# The way to execute a command on the target 810# The way to execute a command on the target
797# (default ssh $SSH_USER@$MACHINE $SSH_COMMAND";) 811# (default ssh $SSH_USER@$MACHINE $SSH_COMMAND";)
798# The variables SSH_USER, MACHINE and SSH_COMMAND are defined 812# The variables SSH_USER, MACHINE and SSH_COMMAND are defined
@@ -1222,3 +1236,33 @@
1222# MIN_CONFIG_TYPE = test 1236# MIN_CONFIG_TYPE = test
1223# TEST = ssh ${USER}@${MACHINE} echo hi 1237# TEST = ssh ${USER}@${MACHINE} echo hi
1224# 1238#
1239#
1240#
1241#
1242# For TEST_TYPE = make_warnings_file
1243#
1244# If you want the build to fail when a new warning is discovered
1245# you set the WARNINGS_FILE to point to a file of known warnings.
1246#
1247# The test "make_warnings_file" will let you create a new warnings
1248# file before you run other tests, like patchcheck.
1249#
1250# What this test does is to run just a build, you still need to
1251# specify BUILD_TYPE to tell the test what type of config to use.
1252# A BUILD_TYPE of nobuild will fail this test.
1253#
1254# The test will do the build and scan for all warnings. Any warning
1255# it discovers will be saved in the WARNINGS_FILE (required) option.
1256#
1257# It is recommended (but not necessary) to make sure BUILD_NOCLEAN is
1258# off, so that a full build is done (make mrproper is performed).
1259# That way, all warnings will be captured.
1260#
1261# Example:
1262#
1263# TEST_TYPE = make_warnings_file
1264# WARNINGS_FILE = ${OUTPUT_DIR}
1265# BUILD_TYPE = useconfig:oldconfig
1266# CHECKOUT = v3.8
1267# BUILD_NOCLEAN = 0
1268#