[tools 1/3] testbot/TestWTBS: Allow restricting the range of jobs to check.
This allows ignoring a previous test run that produced buggy results. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- This is useful to only get the errors of the latest test suite run. testbot/tests/TestWTBS | 69 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index 786d3692..6e1e7ede 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -87,10 +87,33 @@ p tests.TestFailures 1 =cut my $Usage; +sub check_opt_val($$) +{ + my ($option, $val) = @_; + + if (defined $val) + { + error("$option can only be specified once\n"); + $Usage = 2; # but continue processing this option + } + if (!@ARGV) + { + error("missing value for $option\n"); + $Usage = 2; + return undef; + } + return shift @ARGV; +} + +my $OptJobs; while (@ARGV) { my $Arg = shift @ARGV; - if ($Arg eq "--help") + if ($Arg eq "--jobs") + { + $OptJobs = check_opt_val($Arg, $OptJobs); + } + elsif ($Arg eq "--help") { $Usage = 0; } @@ -101,6 +124,30 @@ while (@ARGV) } } +my @JobRanges; +if (!defined $Usage) +{ + if (defined $OptJobs) + { + foreach my $Range (split /,+/, $OptJobs) + { + if ($Range =~ /^([0-9]+)$/) + { + push @JobRanges, [$1, $1]; + } + elsif ($Range =~ /^([0-9]*)\.\.([0-9]*)/) + { + push @JobRanges, [$1 || 0, $2 || 0]; + } + else + { + error("'$Range' is not a valid job range\n"); + $Usage = 2; + } + } + } +} + if (defined $Usage) { if ($Usage) @@ -108,11 +155,15 @@ if (defined $Usage) error("try '$name0 --help' for more information\n"); exit $Usage; } - print "Usage: $name0 [--help]\n"; + print "Usage: $name0 [--jobs RANGES] [--help]\n"; print "\n"; print "Tests the Patches subject parser.\n"; print "\n"; print "Where:\n"; + print " --jobs RANGES Only check the jobs in one of the specified comma-separated\n"; + print " job id ranges. Each range is either a single job id, or of the\n"; + print " form 'FIRST..LAST' where FIRST and LAST are either the empty\n"; + print " string or a job id.\n"; print " --help Shows this usage message.\n"; exit 0; } @@ -369,6 +420,19 @@ sub CheckJob($$) } } +sub IsJobInRange($) +{ + my ($Job) = @_; + + return 1 if (!@JobRanges); + foreach my $Range (@JobRanges) + { + return 1 if (($Range->[0] == 0 or $Range->[0] <= $Job->Id) and + ($Range->[1] == 0 or $Job->Id <= $Range->[1])); + } + return 0; +} + =pod =item <tasks.HasTask> @@ -388,6 +452,7 @@ p win.HasTask 0 sub CheckJobTree($) { my ($Job) = @_; + return if (!IsJobInRange($Job)); my ($TestInfo, $HasTask); -- 2.20.1
This allows verifying that the TestBot detected the errors issued by the test suite; or reported the expected consistency issues. For instance: ----- TestWTBS ----- g 0 tests.report.Report validation errors n 0 kernel32:comm has unaccounted for failure messages ----- TestWTBS ----- Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/tests/TestWTBS | 114 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index 6e1e7ede..af430601 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -297,6 +297,24 @@ sub LoadTestInfo($) fail("$FileName: $Entry is not a valid property name"); } } + foreach my $RawGroupName (@{$RawInfo->{ErrGroupNames}}) + { + my $GroupName = lcfirst($RawGroupName); + if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine)\.(log|report)\.//) + { + my $ErrInfo = ($TestInfo->{$1}->{"$2.errors"} ||= {}); + push @{$ErrInfo->{ErrGroupNames}}, $GroupName; + my $ErrGroup = $RawInfo->{ErrGroups}->{$RawGroupName}; + $ErrInfo->{ErrGroups}->{$GroupName} = $ErrGroup; + $ErrInfo->{ErrCount} += @{$ErrGroup->{Errors}}; + $ErrInfo->{NewCount} += $ErrGroup->{NewCount} || 0; + $HasTestInfo = 1; + } + else + { + fail("$FileName: $RawGroupName is not a valid group name"); + } + } return undef if (!$HasTestInfo); # Fill in some useful defaults @@ -345,6 +363,95 @@ sub IsMailingListJob($) return $Job->Remarks =~ /^\[\Q$PatchesMailingList\E\] /; } +=pod +=item <tasks.(log|report).groupname> + +Verifies the presence of new errors in the specified error log or report +of the tasks in the specified category. For this, list the group and errors +that are expected to appear; where the group name is prefixed by the task +category and either 'log' or 'report'. + +For instance: +g 0 tests.report.kernel32 +n 0 comm.c:2210: Test failed: WTBS Simulate an unreported test failure +g 0 tests.report.Report validation errors +n 0 kernel32:comm has unaccounted for failure messages + +This checks that the 'comm.c' failure appears in the 'kernel32' group +of the reports of all test tasks; that the TestBot issues a report consistency +failure as well; and that no other new error is present. If no preexisting +errors are expected, that is if test unit produces no failure when run outside +the test suite, then one could also issue the following check: + +p tests.TestFailures 2 + +=cut + +sub CheckLogErrors($$$) +{ + my ($LogInfo, $RefInfo, $LogKey) = @_; + + foreach my $GroupName (@{$LogInfo->{ErrGroupNames}}) + { + my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName}; + + my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName}; + if ($RefGroup) + { + my $Diff = Algorithm::Diff->new($LogGroup->{Errors}, $RefGroup->{Errors}, + { keyGen => \&WineTestBot::LogUtils::_GetLineKey }); + my ($LogIndex, $RefIndex) = (0, 0); + while ($Diff->Next()) + { + my $SameCount = $Diff->Same(); + if ($SameCount) + { + $LogIndex += $SameCount; + $RefIndex += $SameCount; + } + else + { + # Check for extra new errors in the log + my $ExtraCount = $Diff->Items(1); + foreach (1..$ExtraCount) + { + next if (!$LogGroup->{IsNew}->[$LogIndex]); + fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$LogIndex]"); + $LogIndex++; + } + # And for missing expected errors + my $MissingCount = $Diff->Items(2); + foreach (1..$MissingCount) + { + fail("$LogKey is missing an error in $GroupName: $RefGroup->{Errors}->[$RefIndex]"); + $RefIndex++; + } + } + } + } + else + { + # Check that the log's extra errors are old errors + foreach my $ErrIndex (0..@{$LogGroup->{Errors}} - 1) + { + next if (!$LogGroup->{IsNew}->[$ErrIndex]); + fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$ErrIndex]"); + } + } + } + foreach my $GroupName (@{$RefInfo->{ErrGroupNames}}) + { + my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName}; + next if ($LogGroup); + + my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName}; + foreach my $Error (@{$RefGroup->{Errors}}) + { + fail("$LogKey is missing an error in $GroupName: $Error"); + } + } +} + =pod =item <job.Status> @@ -378,9 +485,16 @@ sub CheckTask($$$) my $LogPath = $Task->GetDir() ."/$LogName"; my $LogInfo = LoadLogErrors($LogPath); + my $LogType = "log"; if ($LogName =~ /\.report$/) { $ReportCount++; + $LogType = "report"; + } + if ($TaskInfo->{"$LogType.errors"}) + { + CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"}, + TaskKeyStr($Task) ."/$LogName"); } } if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures})) -- 2.20.1
This is useful to verify the TestBot runs extra tests when patching a resource file or other non-test file. For instance: ----- TestWTBS ----- p win.TestUnits cabinet:extract cabinet:fdi ----- TestWTBS ----- Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/tests/TestWTBS | 54 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index af430601..b4460b81 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -344,6 +344,25 @@ sub LoadTestInfo($) return $TestInfo; } +sub GetReportTestUnits($) +{ + my ($LogPath) = @_; + + my $TestUnits = {}; + if (open(my $LogFile, "<", $LogPath)) + { + foreach my $Line (<$LogFile>) + { + if ($Line =~ m%^([_.a-z0-9-]+:[_a-z0-9]*) start (?:-|[/_.a-z0-9-]+) (?:-|[.0-9a-f]+)%) + { + $TestUnits->{$1} = 1; + } + } + close($LogFile); + } + return $TestUnits; +} + # # Verify the Jobs and Tasks @@ -469,9 +488,9 @@ By default the TestFailures field is not checked. =cut -sub CheckTask($$$) +sub CheckTask($$$$) { - my ($Task, $TaskType, $TestInfo) = @_; + my ($Task, $TaskType, $TestInfo, $TestUnits) = @_; my $TaskInfo = $TestInfo->{$TaskType}; if (CheckValue($TaskInfo->{Status})) @@ -490,6 +509,11 @@ sub CheckTask($$$) { $ReportCount++; $LogType = "report"; + if ($TaskType eq "wine") + { + my $ReportTestUnits = GetReportTestUnits($LogPath); + map { $TestUnits->{wine}->{$_} = 1 } (keys %$ReportTestUnits); + } } if ($TaskInfo->{"$LogType.errors"}) { @@ -561,6 +585,14 @@ lines below to make sure the TestBot did not try to run the tests on Windows. p build.HasTask 0 p win.HasTask 0 +=item <tests.TestUnits> + +Checks that the TestBot ran at least the test units mentioned in this +space-separated list on the VMs matching the specified category. + +For instance: +p win.TestUnits cabinet:extract cabinet:fdi + =cut sub CheckJobTree($) @@ -569,6 +601,7 @@ sub CheckJobTree($) return if (!IsJobInRange($Job)); my ($TestInfo, $HasTask); + my $TestUnits = { wine => {} }; my $Steps = $Job->Steps; foreach my $Step (sort { $a->No <=> $b->No } @{$Job->Steps->GetItems()}) @@ -589,7 +622,14 @@ sub CheckJobTree($) foreach my $Task (sort { $a->No <=> $b->No } @{$Step->Tasks->GetItems()}) { $HasTask->{$TaskType} = 1; - CheckTask($Task, $TaskType, $TestInfo); + CheckTask($Task, $TaskType, $TestInfo, $TestUnits); + if ($TaskType =~ /^win(?:32|64)$/) + { + my $TestUnit = $Step->FileName; + $TestUnit =~ s/_test(?:64)?\.exe$//; + $TestUnit .= ":". $Task->CmdLineArg; + $TestUnits->{$TaskType}->{$TestUnit} = 1; + } } } CheckJob($Job, $TestInfo); @@ -608,6 +648,14 @@ sub CheckJobTree($) $HasTask->{$Type} ||= 0; is($HasTask->{$Type}, $TypeInfo->{HasTask}, "Check the presence of $Type tasks for job ". $Job->Id); } + if (CheckValue($TypeInfo->{TestUnits})) + { + foreach my $TestUnit (split / +/, $TypeInfo->{TestUnits}) + { + ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job ". $Job->Id) or + or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$Type}})); + } + } } } } -- 2.20.1
On Wed, 20 Jan 2021, Francois Gouget wrote: [...]
+ ok($TestUnits->{$Type}->{$TestUnit}, "Check that $TestUnit was tested by $Type VMs for job ". $Job->Id) or + or diag("TestUnits=", join(" ", sort keys %{$TestUnits->{$Type}}));
There's a clear typo there (it was fixed in a later patch). I also found an issue in 2/3 for checking the .errors files so I'll resend this patch series. -- Francois Gouget <fgouget(a)codeweavers.com>
participants (1)
-
Francois Gouget