Module: tools
Branch: master
Commit: cc359d12a2e19187aba7a30e3ffd536c86f1596b
URL: https://source.winehq.org/git/tools.git/?a=commit;h=cc359d12a2e19187aba7a30…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Thu Jan 28 04:18:19 2021 +0100
testbot/TestWTBS: Add support for checking timeouts.
Timeouts can get reported in many different ways depending on whether
they are detected by TestLauncher, the WineRun* script or the Engine.
That makes a simple error matching or grep impossible. So provide
specific support for checking that a task timed out.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 94 +++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 74 insertions(+), 20 deletions(-)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 67b1c8f..6f78406 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -402,7 +402,10 @@ sub LoadTestInfo($)
my $Status = ($TestInfo->{job}->{Status} ne 'completed' or
$TestInfo->{win32}->{TestFailures} or
$TestInfo->{win64}->{TestFailures} or
- $TestInfo->{wine}->{TestFailures}) ? "Failed" : "OK";
+ $TestInfo->{wine}->{TestFailures} or
+ $TestInfo->{win32}->{HasTimeout} or
+ $TestInfo->{win64}->{HasTimeout} or
+ $TestInfo->{wine}->{HasTimeout}) ? "Failed" : "OK";
SetDefault($TestInfo, "webpatch", "Status", $Status);
}
@@ -555,6 +558,16 @@ sub IsMailingListJob($)
return $Job->Remarks =~ /^\[\Q$PatchesMailingList\E\] /;
}
+sub IsTimeoutError($$)
+{
+ my ($GroupName, $Error) = @_;
+
+ return (($GroupName ne "" and # Test dll or Report validation errors
+ $Error =~ /^[_a-z0-9]+: Timeout$/) or # Test unit
+ ($GroupName eq "" and # testbot.log case
+ $Error eq "The task timed out"));
+}
+
=pod
=item <tasks.(report|log|testbot).groupname>
@@ -584,17 +597,19 @@ directives, win32 will not inherit anything from the win category.
=cut
-sub CheckLogErrors($$$)
+sub CheckLogErrors($$$$)
{
- my ($LogInfo, $RefInfo, $LogKey) = @_;
+ my ($LogInfo, $RefInfo, $LogKey, $IgnoreTimeouts) = @_;
+ my $HasTimeout;
foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
{
my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
-
- my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+ # Prevent $RefInfo from springing into existence
+ my $RefGroup = $RefInfo ? $RefInfo->{ErrGroups}->{$GroupName} : undef;
if ($RefGroup)
{
+ # Match the log errors to the expected errors for this group
my $Diff = Algorithm::Diff->new($LogGroup->{Errors}, $RefGroup->{Errors},
{ keyGen => \&WineTestBot::LogUtils::_GetLineKey });
my ($LogIndex, $RefIndex) = (0, 0);
@@ -613,6 +628,12 @@ sub CheckLogErrors($$$)
foreach (1..$ExtraCount)
{
next if (!$LogGroup->{IsNew}->[$LogIndex]);
+ if ($IgnoreTimeouts and
+ IsTimeoutError($GroupName, $LogGroup->{Errors}->[$LogIndex]))
+ {
+ next;
+ }
+
fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$LogIndex]");
$LogIndex++;
}
@@ -626,27 +647,39 @@ sub CheckLogErrors($$$)
}
}
}
- else
+
+ # Check for timeouts and new errors if no error was expected for that group
+ foreach my $ErrIndex (0..@{$LogGroup->{Errors}} - 1)
{
- # Check that the log's extra errors are old errors
- foreach my $ErrIndex (0..@{$LogGroup->{Errors}} - 1)
+ if (IsTimeoutError($GroupName, $LogGroup->{Errors}->[$ErrIndex]))
+ {
+ $HasTimeout = 1;
+ next if ($IgnoreTimeouts); # ignore expected timeouts
+ }
+ if ($RefInfo and !$RefGroup and $LogGroup->{IsNew}->[$ErrIndex])
{
- next if (!$LogGroup->{IsNew}->[$ErrIndex]);
fail("$LogKey has an unexpected new error in $GroupName: $LogGroup->{Errors}->[$ErrIndex]");
}
}
}
- foreach my $GroupName (@{$RefInfo->{ErrGroupNames}})
- {
- my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
- next if ($LogGroup);
- my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
- foreach my $Error (@{$RefGroup->{Errors}})
+ if ($RefInfo)
+ {
+ # Check for groups missing from the log file
+ foreach my $GroupName (@{$RefInfo->{ErrGroupNames}})
{
- fail("$LogKey is missing an error in $GroupName: $Error");
+ my $LogGroup = $LogInfo->{ErrGroups}->{$GroupName};
+ next if ($LogGroup);
+
+ my $RefGroup = $RefInfo->{ErrGroups}->{$GroupName};
+ foreach my $Error (@{$RefGroup->{Errors}})
+ {
+ fail("$LogKey is missing an error in $GroupName: $Error");
+ }
}
}
+
+ return $HasTimeout;
}
sub GrepFile($$$$;$)
@@ -716,6 +749,18 @@ By default the Status field should be 'completed'.
Checks the value of the task's TestFailures field.
By default the TestFailures field is not checked.
+=item <tasks.HasTimeout>
+
+When true verifies that the task timed out.
+This also disables reporting timeout errors as unexpected errors.
+
+Use this instead of looking for a specific error in the report because
+timeouts can get reported in many different ways depending whether they are
+detected by TestLauncher, the WineRun* script or the Engine. It can also
+depend on whether the test was the last one run by the task (in the Wine case).
+By default this check is skipped and timeout errors are reported like any
+other unexpected error.
+
=item <tasks.(report|log|testbot).Grep>
=item <tasks.(report|log|testbot).GrepV>
@@ -754,7 +799,10 @@ sub CheckTask($$$$)
$TestUnits->{$TaskType}->{"*skipped*"} = 1;
}
- my ($ReportCount, $NewFailures) = (0, 0);
+ my $CheckTimeouts = ($Task->Status eq "completed" and
+ CheckValue($TaskInfo->{HasTimeout}));
+
+ my ($ReportCount, $TimeoutCount, $NewFailures) = (0, 0, 0);
foreach my $LogName (@{GetLogFileNames($Task->GetDir())})
{
my $LogPath = $Task->GetDir() ."/$LogName";
@@ -777,13 +825,19 @@ sub CheckTask($$$$)
$LogType = "testbot";
}
- if ($TaskInfo->{"$LogType.errors"})
+ if ($TaskInfo->{"$LogType.errors"} or $CheckTimeouts)
{
- CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"},
- TaskKeyStr($Task) ."/$LogName");
+ my $HasTimeout = CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"},
+ TaskKeyStr($Task) ."/$LogName",
+ $TaskInfo->{HasTimeout});
+ $TimeoutCount++ if ($HasTimeout);
}
GrepTaskLog($Task, $LogName, $TaskInfo, "$LogType.");
}
+ if ($CheckTimeouts)
+ {
+ ok($TimeoutCount >= $ReportCount, "Expecting 1+ timeout per report: $TimeoutCount timeouts, $ReportCount reports");
+ }
if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures}))
{
# Scale the expected TestFailures count with the number of times the test
Module: tools
Branch: master
Commit: c6f00ecf6767781726a5d0d79ca1765e38105ad3
URL: https://source.winehq.org/git/tools.git/?a=commit;h=c6f00ecf6767781726a5d0d…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Wed Jan 27 03:57:27 2021 +0100
testbot/TestWTBS: Allow checking emails that don't contain a patch.
When an email contains no patch it is not saved in the patches folder,
which also makes putting TestWTBS directives in the email body pointless.
This patch allows telling TestWTBS this is expected by adding 'WTBS' and
'NoPatch' anywhere in the subject line. Conversely, if the subject
contains 'WTBS' but not 'NoPatch', TestWTBS expects to find a file in
the patches folder.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index d0e7cab..67b1c8f 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -1152,9 +1152,19 @@ sub CheckPatches()
foreach my $Patch (sort { $b->Id <=> $a->Id } @{$Patches->GetItems()})
{
my $PatchFileName = "$DataDir/patches/". $Patch->Id;
- next if (!-f $PatchFileName);
- my $TestInfo = LoadTestInfo($PatchFileName);
- CheckPatch($Jobs, $Patch, $TestInfo) if ($TestInfo);
+ if (-f $PatchFileName)
+ {
+ ok($Patch->Subject !~ /NoPatch/, "Found a file for patch ". $Patch->Id .": ". $Patch->Subject)
+ or diag("Disposition=". $Patch->Disposition);
+ my $TestInfo = LoadTestInfo($PatchFileName);
+ CheckPatch($Jobs, $Patch, $TestInfo) if ($TestInfo);
+ }
+ elsif ($Patch->Subject =~ /\bWTBS\b/)
+ {
+ ok(($Patch->Subject =~ /NoPatch/ or $Patch->Disposition eq "Reply to a patch series"),
+ "Found no file for patch ". $Patch->Id .": ". $Patch->Subject)
+ or diag("Disposition=". $Patch->Disposition);
+ }
}
}