[tools] testbot/TestWTBS: Allow mission-specific directives, e.g. for test=module.
A different set of tests are run depending on whether a VM's mission is test=test (the default) or test=module. Being unable to distinguish these two cases prevents checking that the right set of test units is being run in many cases. So add support for the wine:build, wine:test, wine:module categories to allow providing specific checks for the test=build/test/module cases respectively. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/tests/TestWTBS | 184 +++++++++++++++++++++++++++++------------ 1 file changed, 131 insertions(+), 53 deletions(-) diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index bc4e0d9ba..f6ca54e84 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -48,6 +48,7 @@ use WineTestBot::Config; # For $PatchesMailingList use WineTestBot::Jobs; use WineTestBot::Log; use WineTestBot::LogUtils; +use WineTestBot::Missions; use WineTestBot::Patches; use WineTestBot::VMs; @@ -190,13 +191,15 @@ Property names are of the form 'category.name' where the category is one of: - build for checks to perform on the build task. - win32, win64 for checks to perform on the Windows 32- or 64-bit test results respectively. -- win for the checks to perform on both the 32- and 64-bit test results; that - is equivalent to duplicating the directive for the win32 and win64 - categories. -- wine for the checks to perform on the Wine test results. +- win for the checks to perform on both the 32- and 64-bit test results. This + is equivalent to duplicating the directive for win32 and win64. +- wine:build, wine:test, wine:module for checks to perform on test results of + missions with test=build, test=test and test=module respectively. +- wine for the checks to perform on the Wine test results. This is equivalent + to duplicating the directive for the wine:build, wine:test and wine:module. - tests for checks to perform on all test results, that is equivalent to - duplicating the directive for the win32, win64 and wine categories (but not - build since it does not run the tests). + duplicating the directive for the win and wine categories (but not build + since it does not run the tests). A corollary is that any property which is documented as being valid for a category can also be specified for any of its subcategories. So for instance if @@ -208,8 +211,9 @@ sub DumpTestInfo($) { my ($TestInfo) = @_; - foreach my $Category ("webpatch", "patch", "job", "tasks", "tests", "win", - "build", "win32", "win64", "wine") + foreach my $Category ("webpatch", "patch", "job", "tasks", "tests", "build", + "win", "win32", "win64", + "wine", "wine:build", "wine:test", "wine:module") { WineTestBot::LogUtils::_WriteLogErrorsToFh(*STDERR, $TestInfo->{$Category}); WineTestBot::LogUtils::_DumpErrors($Category, $TestInfo->{$Category}); @@ -293,14 +297,15 @@ sub LoadTestInfo($) # Split up the information to jobs, tasks, etc. my $TestInfo = { webpatch => {}, patch => {}, job => {}, - tasks => {}, tests => {}, win => {}, - build => {}, win32 => {}, win64 => {}, wine => {}, + tasks => {}, tests => {}, build => {}, + win => {}, win32 => {}, win64 => {}, + wine => {}, "wine:build" => {}, "wine:test" => {}, "wine:module" => {}, }; my $HasTestInfo; foreach my $Entry (keys %{$RawInfo}) { my $Field = lcfirst($Entry); - if ($Field =~ s/^(webpatch|patch|job|tasks|tests|win|build|win32|win64|wine)\.//) + if ($Field =~ s/^(webpatch|patch|job|tasks|tests|build|win|win32|win64|wine|wine:build|wine:test|wine:module)\.//) { my $TaskType = $1; $TestInfo->{$TaskType}->{$Field} = $RawInfo->{$Entry}; @@ -314,7 +319,7 @@ sub LoadTestInfo($) foreach my $RawGroupName (@{$RawInfo->{ErrGroupNames}}) { my $GroupName = lcfirst($RawGroupName); - if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine)\.(report|log|testbot)\.//) + if ($GroupName =~ s/^(tasks|tests|build|win|win32|win64|wine|wine:build|wine:test|wine:module)\.(report|log|testbot)\.//) { my $ErrInfo = ($TestInfo->{$1}->{"$2.errors"} ||= {}); push @{$ErrInfo->{ErrGroupNames}}, $GroupName; @@ -338,11 +343,16 @@ sub LoadTestInfo($) SetDefault($TestInfo, "tasks", "Status", "completed"); SetDefault($TestInfo, "tasks", "HasTask", 1); } + if (defined $TestInfo->{"wine:build"}->{TestUnits}) + { + fail("wine:build.TestUnits should not be set"); + } # Then propagate the defaults foreach my $Pair (["tasks", ["build", "tests"]], ["tests", ["win", "wine"]], - ["win", ["win32", "win64"]]) + ["win", ["win32", "win64"]], + ["wine", ["wine:test", "wine:module"]]) { my ($Src, $TaskTypes) = @$Pair; foreach my $Field (keys %{$TestInfo->{$Src}}) @@ -368,41 +378,51 @@ sub LoadTestInfo($) } } } + # Reset the inherited wine:build.TestUnits since it makes no sense + delete $TestInfo->{"wine:build"}->{TestUnits}; # Automatically check the Task: lines in simple cases - # Make sure no test is run for build tasks - my $GrepV = ($TestInfo->{build}->{"report.GrepV"} ||= []); - push @$GrepV, '.'; - $GrepV = ($TestInfo->{build}->{"log.GrepV"} ||= []); - push @$GrepV, '^Task: tests$'; - if (($TestInfo->{build}->{Status} || "") =~ /^bad(?:build|patch)$/ or - $TestInfo->{build}->{HasTimeout}) - { - push @$GrepV, '^Task: ok$'; - } - else + foreach my $Build ("build", "wine:build") { - my $Grep = ($TestInfo->{build}->{"log.Grep"} ||= []); - push @$Grep, '^Task: ok$'; + # Make sure no test is run for build tasks + my $GrepV = ($TestInfo->{$Build}->{"report.GrepV"} ||= []); + push @$GrepV, '.'; + $GrepV = ($TestInfo->{$Build}->{"log.GrepV"} ||= []); + push @$GrepV, '^Task: tests$'; + + if (($TestInfo->{$Build}->{Status} || "") =~ /^bad(?:build|patch)$/ or + $TestInfo->{$Build}->{HasTimeout}) + { + push @$GrepV, '^Task: ok$'; + } + else + { + my $Grep = ($TestInfo->{$Build}->{"log.Grep"} ||= []); + push @$Grep, '^Task: ok$'; + } } - # Note: Depending on where the timeout occurs, the wine task log may or - # may not have a 'Task: ok' line. - if (($TestInfo->{wine}->{Status} || "") eq "completed" and - !$TestInfo->{wine}->{HasTimeout}) + foreach my $Test ("test", "module") { - my $Grep = ($TestInfo->{wine}->{"log.Grep"} ||= []); - if (CheckValue($TestInfo->{wine}->{TestUnits})) + # Note: Depending on where the timeout occurs, the wine task log may or + # may not have a 'Task: ok' line. + if (($TestInfo->{"wine:$Test"}->{Status} || "") eq "completed" and + !$TestInfo->{"wine:$Test"}->{HasTimeout}) { - push @$Grep, '^Task: tests$'; + my $Grep = ($TestInfo->{"wine:$Test"}->{"log.Grep"} ||= []); + if (CheckValue($TestInfo->{"wine:$Test"}->{TestUnits})) + { + push @$Grep, '^Task: tests$'; + } + push @$Grep, '^Task: ok$'; } - push @$Grep, '^Task: ok$'; } # Validate and fix the Grep* fields foreach my $GrepType ("Grep", "GrepV") { - foreach my $TaskType ("tasks", "tests", "build", "win", "win32", "win64", "wine") + foreach my $TaskType ("tasks", "tests", "build", "win", "win32", "win64", + "wine", "wine:build", "wine:test", "wine:module") { foreach my $LogType ("report", "log", "testbot") { @@ -426,15 +446,19 @@ sub LoadTestInfo($) CheckValue($TestInfo->{job}->{Status}) and !SkipCheck($TestInfo->{win32}->{TestFailures}) and !SkipCheck($TestInfo->{win64}->{TestFailures}) and - !SkipCheck($TestInfo->{wine}->{TestFailures})) + !SkipCheck($TestInfo->{"wine:test"}->{TestFailures}) and + !SkipCheck($TestInfo->{"wine:module"}->{TestFailures})) { my $Status = ($TestInfo->{job}->{Status} ne 'completed' or $TestInfo->{win32}->{TestFailures} or $TestInfo->{win64}->{TestFailures} or - $TestInfo->{wine}->{TestFailures} or + $TestInfo->{"wine:test"}->{TestFailures} or + $TestInfo->{"wine:module"}->{TestFailures} or $TestInfo->{win32}->{HasTimeout} or $TestInfo->{win64}->{HasTimeout} or - $TestInfo->{wine}->{HasTimeout}) ? "Failed" : "OK"; + $TestInfo->{"wine:build"}->{HasTimeout} or + $TestInfo->{"wine:test"}->{HasTimeout} or + $TestInfo->{"wine:module"}->{HasTimeout}) ? "Failed" : "OK"; SetDefault($TestInfo, "webpatch", "Status", $Status); } @@ -828,9 +852,30 @@ sub CheckTask($$$$) $TestUnits->{$TaskType}->{"*skipped*"} = 1; } + # Assume the VM's Missions field has not changed since the tests were run + my ($ErrMessage, $Missions) = ParseMissionStatement($Task->Missions); + if (@$Missions != 1) + { + fail(TaskKeyStr($Task) ." has an invalid missions field: ". $Task->Missions); + return; + } + + my %ReportTypes; + if ($TaskType =~ /^win/) + { + foreach my $Mission (@{$Missions->[0]->{Missions}}) + { + my $ReportName = GetMissionBaseName($Mission) .".report"; + my $MissionType = $TaskType; + $MissionType .= ":". ($Mission->{test} || "test") if ($TaskType eq "wine"); + $ReportTypes{$ReportName} = $MissionType; + } + } + my $CheckTimeouts = ($Task->Status eq "completed" and CheckValue($TaskInfo->{HasTimeout})); + my $ExpectedFailures; my ($ReportCount, $TimeoutCount, $NewFailures) = (0, 0, 0); foreach my $LogName (@{GetLogFileNames($Task->GetDir())}) { @@ -838,15 +883,33 @@ sub CheckTask($$$$) my $LogInfo = LoadLogErrors($LogPath); $NewFailures += $LogInfo->{NewCount} || 0; + # Get the mission-specific "wine:xxx" report directives + my $MissionType = $ReportTypes{$LogName} || $TaskType; + my $MissionInfo = $TestInfo->{$MissionType}; my $LogType = "log"; if ($LogName =~ /\.report$/) { - $ReportCount++; $LogType = "report"; + $ReportCount++; + + ok($ReportTypes{$LogName}, "Check that $LogName is expected"); + if ($TaskType eq "wine") { my $ReportTestUnits = GetReportTestUnits($LogPath); - map { $TestUnits->{wine}->{$_} = 1 } (keys %$ReportTestUnits); + map { $TestUnits->{$MissionType}->{$_} = 1 } (keys %$ReportTestUnits); + } + + if (CheckValue($MissionInfo->{TestFailures})) + { + ok(($LogInfo->{ErrCount} || 0) <= $MissionInfo->{TestFailures}, + "Check Failures of $LogName in task ". TaskKeyStr($Task)) + or diag("report error count = ", ($LogInfo->{ErrCount} || 0), ", expected at most $MissionInfo->{TestFailures}"); + $ExpectedFailures += $MissionInfo->{TestFailures}; + } + else + { + $ExpectedFailures = undef; } } elsif ($LogName =~ /^testbot\./) @@ -854,24 +917,35 @@ sub CheckTask($$$$) $LogType = "testbot"; } - if ($TaskInfo->{"$LogType.errors"} or $CheckTimeouts) + if ($MissionInfo->{"$LogType.errors"} or $CheckTimeouts) { - my $HasTimeout = CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"}, + my $HasTimeout = CheckLogErrors($LogInfo, $MissionInfo->{"$LogType.errors"}, TaskKeyStr($Task) ."/$LogName", - $TaskInfo->{HasTimeout}); + $MissionInfo->{HasTimeout}); $TimeoutCount++ if ($HasTimeout); } - GrepTaskLog($Task, $LogName, $TaskInfo, "$LogType."); + GrepTaskLog($Task, $LogName, $MissionInfo, "$LogType."); } if ($CheckTimeouts) { ok($TimeoutCount >= $ReportCount, "Expecting 1+ timeout per report: $TimeoutCount timeouts, $ReportCount reports"); } - if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures})) + if ($Task->Status eq "completed") { - # Scale the expected TestFailures count with the number of times the test - # was run, i.e. $ReportCount, or take it as is if no report is available. - is($Task->TestFailures, $TaskInfo->{TestFailures} * ($ReportCount || 1), "Check Failures of task ". TaskKeyStr($Task)); + if (defined $ExpectedFailures) + { + is($Task->TestFailures, $ExpectedFailures, "Check Failures of task ". TaskKeyStr($Task)); + } + elsif (CheckValue($TaskInfo->{TestFailures}) and !$ReportCount) + { + # Scale the expected TestFailures count with the number of times the test + # was run, i.e. $ReportCount, or take it as is if no report is available. + is($Task->TestFailures, $TaskInfo->{TestFailures}, "Check Failures of task ". TaskKeyStr($Task)); + } + # else there are reports for which we cannot check the TestFailures count. + # In particular this can happen if the test failure count can be checked + # for test=test but not for test=module. Then whether TestFailures can be + # checked or not depends on the missions mix. } return $NewFailures; } @@ -973,7 +1047,7 @@ sub CheckJobTree($;$) $CheckedJobs{$JobId} = 1; my ($HasTask, $HasNewFailures); - my $TestUnits = { wine => {} }; + my $TestUnits = {}; my $Email = $Emails{$JobId}; my %FailedVMs; @@ -1059,12 +1133,16 @@ sub CheckJobTree($;$) } next if ($TestUnits->{$Type}->{"*skipped*"}); - if (CheckValue($TypeInfo->{TestUnits})) + my @MissionTypes = $Type eq "wine" ? ("wine:test", "wine:module") : ($Type); + foreach my $MissionType (@MissionTypes) { - foreach my $TestUnit (split / +/, $TypeInfo->{TestUnits}) + my $MissionInfo = $TestInfo->{$MissionType}; + next if (!CheckValue($MissionInfo->{TestUnits})); + + foreach my $TestUnit (split / +/, $MissionInfo->{TestUnits}) { - ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job $JobId") - or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$Type}})); + ok($TestUnits->{$MissionType}->{$TestUnit}, "Check that $TestUnit was tested by $MissionType VMs for job $JobId") + or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$MissionType}})); } } } -- 2.20.1
participants (1)
-
Francois Gouget