Module: tools Branch: master Commit: c6b58b0f635a5066d89ebf6eede5d9fae76dcfce URL: https://gitlab.winehq.org/winehq/tools/-/commit/c6b58b0f635a5066d89ebf6eede5...
Author: Francois Gouget fgouget@codeweavers.com Date: Mon Nov 7 19:45:32 2022 +0100
testbot: Track and count the bad test units.
Build a list of test units that have failures in a given report and save that information in the .errors file. Count the deduplicated bad test units and set the task's FailedTestUnits field accordingly.
---
testbot/bin/WineRunTask.pl | 10 ++++++---- testbot/bin/WineRunWineTest.pl | 10 ++++++---- testbot/lib/WineTestBot/LogUtils.pm | 34 +++++++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 11 deletions(-)
diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl index 73908305..bce67cf2 100755 --- a/testbot/bin/WineRunTask.pl +++ b/testbot/bin/WineRunTask.pl @@ -212,9 +212,9 @@ sub LogTaskError($)
my $ReportNames;
-sub WrapUpAndExit($;$$$$$$$) +sub WrapUpAndExit($;$$$$$$$$) { - my ($Status, $TestFailures, $Retry, $TimedOut, $Reason, $NewTestFailures, $NewWarnings, $Warnings) = @_; + my ($Status, $TestFailures, $Retry, $TimedOut, $Reason, $NewTestFailures, $NewWarnings, $Warnings, $FailedTestUnits) = @_; my $NewVMStatus = $Status eq 'queued' ? 'offline' : 'dirty'; my $VMResult = $Reason ? $Reason : $Status eq "boterror" ? "boterror" : @@ -258,6 +258,7 @@ sub WrapUpAndExit($;$$$$$$$) $Task->Warnings($Warnings); $Task->NewTestFailures($NewTestFailures); $Task->TestFailures($TestFailures); + $Task->FailedTestUnits($FailedTestUnits); if ($Status eq 'queued') { $Task->Started(undef); @@ -515,7 +516,7 @@ if (!$Pid)
my $NewStatus = 'completed'; my ($TaskFailures, $TaskTimedOut, $TAError, $PossibleCrash); -my ($NewTestFailures, $NewWarnings, $Warnings); +my ($NewTestFailures, $NewWarnings, $Warnings, $FailedTestUnits); Debug(Elapsed($Start), " Waiting for the script (", $Timeout, "s timeout)\n"); if (!defined $TA->Wait($Pid, $Timeout, $Keepalive)) { @@ -580,6 +581,7 @@ if ($TA->GetFile($RptFileName, "$TaskDir/$RptFileName")) { $NewWarnings = ($LogInfo->{wNew} || 0) - ($LogInfo->{wKnownNew} || 0); } + map { $FailedTestUnits->{$_} = 1 } @{$LogInfo->{FailedTestUnits}}; } } elsif (!defined $TAError) @@ -597,4 +599,4 @@ $TA->Disconnect();
FatalTAError(undef, $TAError, $PossibleCrash) if (defined $TAError);
-WrapUpAndExit($NewStatus, $TaskFailures, undef, $TaskTimedOut, undef, $NewTestFailures, $NewWarnings, $Warnings); +WrapUpAndExit($NewStatus, $TaskFailures, undef, $TaskTimedOut, undef, $NewTestFailures, $NewWarnings, $Warnings, $FailedTestUnits ? scalar(%$FailedTestUnits) : undef); diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl index c2dccf29..dd578e47 100755 --- a/testbot/bin/WineRunWineTest.pl +++ b/testbot/bin/WineRunWineTest.pl @@ -209,9 +209,9 @@ sub LogTaskError($)
my $ReportNames;
-sub WrapUpAndExit($;$$$$$$$) +sub WrapUpAndExit($;$$$$$$$$) { - my ($Status, $TestFailures, $Retry, $TimedOut, $Reason, $NewTestFailures, $NewWarnings, $Warnings) = @_; + my ($Status, $TestFailures, $Retry, $TimedOut, $Reason, $NewTestFailures, $NewWarnings, $Warnings, $FailedTestUnits) = @_; my $NewVMStatus = $Status eq 'queued' ? 'offline' : 'dirty'; my $VMResult = defined $Reason ? $Reason : $Status eq "boterror" ? "boterror" : @@ -255,6 +255,7 @@ sub WrapUpAndExit($;$$$$$$$) $Task->Warnings($Warnings); $Task->NewTestFailures($NewTestFailures); $Task->TestFailures($TestFailures); + $Task->FailedTestUnits($FailedTestUnits); if ($Status eq 'queued') { $Task->Started(undef); @@ -486,7 +487,7 @@ if (!$Pid)
my $NewStatus = 'completed'; my ($ErrMessage, $TaskFailures, $TaskTimedOut, $TAError, $PossibleCrash); -my ($NewTestFailures, $NewWarnings, $Warnings); +my ($NewTestFailures, $NewWarnings, $Warnings, $FailedTestUnits); Debug(Elapsed($Start), " Waiting for the script (", $Task->Timeout, "s timeout)\n"); if (!defined $TA->Wait($Pid, $Task->Timeout, 60)) { @@ -613,6 +614,7 @@ foreach my $RptFileName (@$ReportNames) $NewWarnings += ($LogInfo->{wNew} || 0) - ($LogInfo->{wKnownNew} || 0); } } + map { $FailedTestUnits->{$_} = 1 } @{$LogInfo->{FailedTestUnits}}; } elsif (!defined $TAError and $TA->GetLastError() !~ /: No such file or directory/) @@ -635,4 +637,4 @@ $TA->Disconnect(); LogTaskError($ErrMessage) if (defined $ErrMessage); FatalTAError(undef, $TAError, $PossibleCrash) if (defined $TAError);
-WrapUpAndExit($NewStatus, $TaskFailures, undef, $TaskTimedOut, undef, $NewTestFailures, $NewWarnings, $Warnings); +WrapUpAndExit($NewStatus, $TaskFailures, undef, $TaskTimedOut, undef, $NewTestFailures, $NewWarnings, $Warnings, $FailedTestUnits ? scalar(%$FailedTestUnits) : undef); diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm index e4145628..fc840f97 100644 --- a/testbot/lib/WineTestBot/LogUtils.pm +++ b/testbot/lib/WineTestBot/LogUtils.pm @@ -422,7 +422,11 @@ sub _AddExtra($$;$) { my ($LogInfo, $Error, $Cur) = @_;
- $Error = "$Cur->{Dll}:$Cur->{Unit} $Error" if (defined $Cur); + if (defined $Cur) + { + $Error = "$Cur->{Dll}:$Cur->{Unit} $Error"; + $Cur->{Extra}++; + } push @{$LogInfo->{Extra}}, $Error; }
@@ -513,6 +517,12 @@ sub _CloseTestUnit($$$) } } } + if ($Cur->{Dll} and + ($Cur->{Rc} or $Cur->{Extra} or $Cur->{LineFailures} or + $Cur->{SummaryFailures} or $Cur->{LineFlaky} or $Cur->{SummaryFlaky})) + { + $LogInfo->{_FailedTestUnits}->{"$Cur->{Dll}:$Cur->{Unit}"} = 1; + } }
sub _AddReportMsg($$$$$) @@ -555,6 +565,9 @@ The number of test units. =item TimeoutCount The number of test units that timed out.
+=item FailedTestUnits +A list of test units that have failures (flaky or not). + =item eCount, eNew, MsgGroupNames, MsgGroups, BadLog See ParseTaskLog() for details.
@@ -575,6 +588,7 @@ sub ParseWineTestReport($$$)
TestUnitCount => 0, TimeoutCount => 0, + FailedTestUnits => [],
MsgGroupNames => [], MsgGroups => {}, @@ -795,12 +809,12 @@ sub ParseWineTestReport($$$) # The main summary line is missing if ($Rc & 0xc0000000) { - _AddExtra($LogInfo, sprintf("%s:%s crashed (%08x)", $Dll, $Unit, $Rc & 0xffffffff)); + _AddExtra($LogInfo, sprintf("crashed (%08x)", $Rc & 0xffffffff), $Cur); $Cur->{IsBroken} = 1; } elsif ($LogInfo->{IsWineTest} and !$Cur->{IsBroken}) { - _AddExtra($LogInfo, "$Dll:$Unit has no test summary line (early exit of the main process?)"); + _AddExtra($LogInfo, "has no test summary line (early exit of the main process?)", $Cur); } } elsif ($Rc & 0xc0000000) @@ -817,6 +831,7 @@ sub ParseWineTestReport($$$) $Cur->{IsBroken} = 1 if ($LogInfo->{TaskTimedOut}); _CloseTestUnit($LogInfo, $Cur, 1); close($LogFile); + $LogInfo->{FailedTestUnits} = [ sort keys %{$LogInfo->{_FailedTestUnits}} ];
# Move the extra errors into their own error group my $ExtraCount = @{$LogInfo->{Extra}}; @@ -1124,11 +1139,24 @@ sub _WriteLogErrorsToFh($$) { my ($Fh, $LogInfo) = @_;
+ # Save some properties... foreach my $Name ("BadRef", "NoRef") { next if (!defined $LogInfo->{$Name}); print $Fh "p $Name $LogInfo->{$Name}\n"; } + + # some arrays... + foreach my $Name ("FailedTestUnits") + { + next if (!defined $LogInfo->{$Name}); + foreach my $Value (@{$LogInfo->{$Name}}) + { + print $Fh "a $Name $Value\n"; + } + } + + # and the message groups + known failures foreach my $GroupName (@{$LogInfo->{MsgGroupNames}}) { my $Group = $LogInfo->{MsgGroups}->{$GroupName};