Module: tools
Branch: master
Commit: 1b3cd386fb5b51525336b721c4027c9042394da8
URL: https://source.winehq.org/git/tools.git/?a=commit;h=1b3cd386fb5b51525336b72…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Wed Jan 20 13:57:00 2021 +0100
testbot/TestWTBS: Allow verifying the presence of specific errors.
This allows verifying that the TestBot detected the errors issued by the
test suite; or reported the expected consistency issues.
For instance:
----- TestWTBS -----
g 0 tests.report.Report validation errors
n 0 kernel32:comm has unaccounted for failure messages
----- TestWTBS -----
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 114 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 114 insertions(+)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 6e1e7ed..af43060 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -297,6 +297,24 @@ sub LoadTestInfo($)
fail("$FileName: $Entry is not a valid property name");
}
}
+ foreach my $RawGroupName (@{$RawInfo->{ErrGroupNames}})
+ {
+ my $GroupName = lcfirst($RawGroupName);
+ if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine)\.(log|report)\.//)
+ {
+ my $ErrInfo = ($TestInfo->{$1}->{"$2.errors"} ||= {});
+ push @{$ErrInfo->{ErrGroupNames}}, $GroupName;
+ my $ErrGroup = $RawInfo->{ErrGroups}->{$RawGroupName};
+ $ErrInfo->{ErrGroups}->{$GroupName} = $ErrGroup;
+ $ErrInfo->{ErrCount} += @{$ErrGroup->{Errors}};
+ $ErrInfo->{NewCount} += $ErrGroup->{NewCount} || 0;
+ $HasTestInfo = 1;
+ }
+ else
+ {
+ fail("$FileName: $RawGroupName is not a valid group name");
+ }
+ }
return undef if (!$HasTestInfo);
# Fill in some useful defaults
@@ -345,6 +363,95 @@ sub IsMailingListJob($)
return $Job->Remarks =~ /^\[\Q$PatchesMailingList\E\] /;
}
+=pod
+=item <tasks.(log|report).groupname>
+
+Verifies the presence of new errors in the specified error log or report
+of the tasks in the specified category. For this, list the group and errors
+that are expected to appear; where the group name is prefixed by the task
+category and either 'log' or 'report'.
+
+For instance:
+g 0 tests.report.kernel32
+n 0 comm.c:2210: Test failed: WTBS Simulate an unreported test failure
+g 0 tests.report.Report validation errors
+n 0 kernel32:comm has unaccounted for failure messages
+
+This checks that the 'comm.c' failure appears in the 'kernel32' group
+of the reports of all test tasks; that the TestBot issues a report consistency
+failure as well; and that no other new error is present. If no preexisting
+errors are expected, that is if test unit produces no failure when run outside
+the test suite, then one could also issue the following check:
+
+p tests.TestFailures 2
+
+=cut
+
+sub CheckLogErrors($$$)
+{
+ my ($LogInfo, $RefInfo, $LogKey) = @_;
+
+ foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+ {
+ my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
+
+ my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+ if ($RefGroup)
+ {
+ my $Diff = Algorithm::Diff->new($LogGroup->{Errors}, $RefGroup->{Errors},
+ { keyGen => \&WineTestBot::LogUtils::_GetLineKey });
+ my ($LogIndex, $RefIndex) = (0, 0);
+ while ($Diff->Next())
+ {
+ my $SameCount = $Diff->Same();
+ if ($SameCount)
+ {
+ $LogIndex += $SameCount;
+ $RefIndex += $SameCount;
+ }
+ else
+ {
+ # Check for extra new errors in the log
+ my $ExtraCount = $Diff->Items(1);
+ foreach (1..$ExtraCount)
+ {
+ next if (!$LogGroup->{IsNew}->[$LogIndex]);
+ fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$LogIndex]");
+ $LogIndex++;
+ }
+ # And for missing expected errors
+ my $MissingCount = $Diff->Items(2);
+ foreach (1..$MissingCount)
+ {
+ fail("$LogKey is missing an error in $GroupName: $RefGroup->{Errors}->[$RefIndex]");
+ $RefIndex++;
+ }
+ }
+ }
+ }
+ else
+ {
+ # Check that the log's extra errors are old errors
+ foreach my $ErrIndex (0..@{$LogGroup->{Errors}} - 1)
+ {
+ next if (!$LogGroup->{IsNew}->[$ErrIndex]);
+ fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$ErrIndex]");
+ }
+ }
+ }
+ foreach my $GroupName (@{$RefInfo->{ErrGroupNames}})
+ {
+ my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
+ next if ($LogGroup);
+
+ my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+ foreach my $Error (@{$RefGroup->{Errors}})
+ {
+ fail("$LogKey is missing an error in $GroupName: $Error");
+ }
+ }
+}
+
=pod
=item <job.Status>
@@ -378,9 +485,16 @@ sub CheckTask($$$)
my $LogPath = $Task->GetDir() ."/$LogName";
my $LogInfo = LoadLogErrors($LogPath);
+ my $LogType = "log";
if ($LogName =~ /\.report$/)
{
$ReportCount++;
+ $LogType = "report";
+ }
+ if ($TaskInfo->{"$LogType.errors"})
+ {
+ CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"},
+ TaskKeyStr($Task) ."/$LogName");
}
}
if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures}))