This allows verifying that the TestBot detected the errors issued by the test suite; or reported the expected consistency issues.
For instance: ----- TestWTBS ----- g 0 tests.report.Report validation errors n 0 kernel32:comm has unaccounted for failure messages ----- TestWTBS -----
Signed-off-by: Francois Gouget fgouget@codeweavers.com --- testbot/tests/TestWTBS | 114 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS index 6e1e7ede..af430601 100755 --- a/testbot/tests/TestWTBS +++ b/testbot/tests/TestWTBS @@ -297,6 +297,24 @@ sub LoadTestInfo($) fail("$FileName: $Entry is not a valid property name"); } } + foreach my $RawGroupName (@{$RawInfo->{ErrGroupNames}}) + { + my $GroupName = lcfirst($RawGroupName); + if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine).(log|report).//) + { + my $ErrInfo = ($TestInfo->{$1}->{"$2.errors"} ||= {}); + push @{$ErrInfo->{ErrGroupNames}}, $GroupName; + my $ErrGroup = $RawInfo->{ErrGroups}->{$RawGroupName}; + $ErrInfo->{ErrGroups}->{$GroupName} = $ErrGroup; + $ErrInfo->{ErrCount} += @{$ErrGroup->{Errors}}; + $ErrInfo->{NewCount} += $ErrGroup->{NewCount} || 0; + $HasTestInfo = 1; + } + else + { + fail("$FileName: $RawGroupName is not a valid group name"); + } + } return undef if (!$HasTestInfo);
# Fill in some useful defaults @@ -345,6 +363,95 @@ sub IsMailingListJob($) return $Job->Remarks =~ /^[\Q$PatchesMailingList\E] /; }
+=pod +=item <tasks.(log|report).groupname> + +Verifies the presence of new errors in the specified error log or report +of the tasks in the specified category. For this, list the group and errors +that are expected to appear; where the group name is prefixed by the task +category and either 'log' or 'report'. + +For instance: +g 0 tests.report.kernel32 +n 0 comm.c:2210: Test failed: WTBS Simulate an unreported test failure +g 0 tests.report.Report validation errors +n 0 kernel32:comm has unaccounted for failure messages + +This checks that the 'comm.c' failure appears in the 'kernel32' group +of the reports of all test tasks; that the TestBot issues a report consistency +failure as well; and that no other new error is present. If no preexisting +errors are expected, that is if test unit produces no failure when run outside +the test suite, then one could also issue the following check: + +p tests.TestFailures 2 + +=cut + +sub CheckLogErrors($$$) +{ + my ($LogInfo, $RefInfo, $LogKey) = @_; + + foreach my $GroupName (@{$LogInfo->{ErrGroupNames}}) + { + my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName}; + + my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName}; + if ($RefGroup) + { + my $Diff = Algorithm::Diff->new($LogGroup->{Errors}, $RefGroup->{Errors}, + { keyGen => &WineTestBot::LogUtils::_GetLineKey }); + my ($LogIndex, $RefIndex) = (0, 0); + while ($Diff->Next()) + { + my $SameCount = $Diff->Same(); + if ($SameCount) + { + $LogIndex += $SameCount; + $RefIndex += $SameCount; + } + else + { + # Check for extra new errors in the log + my $ExtraCount = $Diff->Items(1); + foreach (1..$ExtraCount) + { + next if (!$LogGroup->{IsNew}->[$LogIndex]); + fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$LogIndex]"); + $LogIndex++; + } + # And for missing expected errors + my $MissingCount = $Diff->Items(2); + foreach (1..$MissingCount) + { + fail("$LogKey is missing an error in $GroupName: $RefGroup->{Errors}->[$RefIndex]"); + $RefIndex++; + } + } + } + } + else + { + # Check that the log's extra errors are old errors + foreach my $ErrIndex (0..@{$LogGroup->{Errors}} - 1) + { + next if (!$LogGroup->{IsNew}->[$ErrIndex]); + fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$ErrIndex]"); + } + } + } + foreach my $GroupName (@{$RefInfo->{ErrGroupNames}}) + { + my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName}; + next if ($LogGroup); + + my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName}; + foreach my $Error (@{$RefGroup->{Errors}}) + { + fail("$LogKey is missing an error in $GroupName: $Error"); + } + } +} + =pod
=item <job.Status> @@ -378,9 +485,16 @@ sub CheckTask($$$) my $LogPath = $Task->GetDir() ."/$LogName"; my $LogInfo = LoadLogErrors($LogPath);
+ my $LogType = "log"; if ($LogName =~ /.report$/) { $ReportCount++; + $LogType = "report"; + } + if ($TaskInfo->{"$LogType.errors"}) + { + CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"}, + TaskKeyStr($Task) ."/$LogName"); } } if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures}))