Module: tools
Branch: master
Commit: 2ea0ffd802cf42f073296baf5bd76fea778c74c2
URL: https://source.winehq.org/git/tools.git/?a=commit;h=2ea0ffd802cf42f073296ba…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Wed Jun 15 18:21:51 2022 +0200
testbot/WineSendLog: Take into account the known failures.
Don't count known failures as new failures, in particular when updating
the patches site.
Add links to the WineHQ bugs for known failures found in the logs.
And identify the bug each known failure line corresponds to.
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=48912
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/bin/WineSendLog.pl | 78 ++++++++++++++++++++++++++++++++--------------
1 file changed, 54 insertions(+), 24 deletions(-)
diff --git a/testbot/bin/WineSendLog.pl b/testbot/bin/WineSendLog.pl
index 3897ac03..a2f2369b 100755
--- a/testbot/bin/WineSendLog.pl
+++ b/testbot/bin/WineSendLog.pl
@@ -44,6 +44,7 @@ $Name0 =~ s+^.*/++;
use Algorithm::Diff;
use WineTestBot::Config;
+use WineTestBot::Failures;
use WineTestBot::Jobs;
use WineTestBot::Log;
use WineTestBot::LogUtils;
@@ -177,10 +178,11 @@ sub SendLog($)
# Collect and count the new and old failures, etc.
#
- my (@New, $JobInfo);
- my ($NewCount, $OldCount, $BotCount, $NotRun) = (0, 0, 0, 0);
+ my ($JobInfo, @New, %BugDescriptions);
+ my ($NewCount, $FailCount, $OldCount, $BotCount, $NotRun) = (0, 0, 0, 0, 0);
my $FileType = "test executable";
+ my $Failures = CreateFailures($Job);
my $StepsTasks = CreateStepsTasks(undef, $Job);
my $SortedStepsTasks = $StepsTasks->GetSortedItems();
foreach my $StepTask (@$SortedStepsTasks)
@@ -193,12 +195,13 @@ sub SendLog($)
my $TaskDir = $StepTask->GetTaskDir();
my $LogNames = GetLogFileNames($TaskDir);
- $JobInfo->{$StepTask->Id}->{LogNames} = $LogNames;
- $JobInfo->{$StepTask->Id}->{NewCount} = 0;
- $JobInfo->{$StepTask->Id}->{OldCount} = 0;
+ my $TaskInfo = $JobInfo->{$StepTask->Id} = {};
+ $TaskInfo->{LogNames} = $LogNames;
+ $TaskInfo->{NewCount} = 0;
+ $TaskInfo->{OldCount} = 0;
if ($StepTask->Status eq "boterror")
{
- $JobInfo->{$StepTask->Id}->{BotCount} = 1;
+ $TaskInfo->{BotCount} = 1;
$BotCount++;
}
$FileType = "patch" if ($StepTask->FileType eq "patch");
@@ -207,7 +210,7 @@ sub SendLog($)
{
my $LogInfo = LoadLogErrors("$TaskDir/$LogName");
next if (!defined $LogInfo->{BadLog} and !$LogInfo->{ErrCount});
- $JobInfo->{$StepTask->Id}->{$LogName} = $LogInfo;
+ $TaskInfo->{$LogName} = $LogInfo;
my $HasLogHeader;
foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
@@ -216,7 +219,21 @@ sub SendLog($)
my $Group = $LogInfo->{ErrGroups}->{$GroupName};
foreach my $ErrIndex (0..$#{$Group->{Errors}})
{
- if ($Group->{IsNew}->[$ErrIndex])
+ my $LineNo = $Group->{LineNos}->[$ErrIndex];
+ my $LineFailureIds = $Group->{Failures}->{$ErrIndex};
+ if ($LineFailureIds)
+ {
+ foreach my $FailureId (@$LineFailureIds)
+ {
+ my $Failure = $Failures->GetItem($FailureId);
+ next if (!$Failure);
+ $BugDescriptions{$Failure->BugId} = $Failure->BugDescription;
+ # Add bug information to $LogInfo
+ push @{$LogInfo->{Bugs}->{$LineNo}}, $Failure->BugId;
+ }
+ $TaskInfo->{OldCount}++;
+ }
+ elsif ($Group->{IsNew}->[$ErrIndex])
{
if (!$HasLogHeader)
{
@@ -229,16 +246,16 @@ sub SendLog($)
$HasGroupHeader = 1;
}
push @New, "$Group->{Errors}->[$ErrIndex]\n";
- $JobInfo->{$StepTask->Id}->{NewCount}++;
+ $TaskInfo->{NewCount}++;
}
else
{
- $JobInfo->{$StepTask->Id}->{OldCount}++;
+ $TaskInfo->{OldCount}++;
}
}
}
- $NewCount += $JobInfo->{$StepTask->Id}->{NewCount};
- $OldCount += $JobInfo->{$StepTask->Id}->{OldCount};
+ $NewCount += $TaskInfo->{NewCount};
+ $OldCount += $TaskInfo->{OldCount};
}
}
@@ -281,10 +298,9 @@ sub SendLog($)
{
print $Sendmail <<"EOF";
It looks like your $FileType introduces some new failures. Please
-investigate and fix them if they are indeed new. Note that rare
-failures and failures with always changing text (e.g. because of memory
-addresses) can cause false positives. If this is what happened, then
-fixing those would really help.
+investigate and fix them if they are indeed new. If they are not new,
+fixing them anyway would help a lot. Otherwise please ask for the known
+failures list to be updated.
EOF
}
@@ -309,10 +325,21 @@ EOF
if ($OldCount)
{
print $Sendmail <<"EOF";
-Some preexisting failures (not caused by your $FileType) happened.
+There are some preexisting failures (not caused by your $FileType).
If you know how to fix them that would be helpful.
-
EOF
+ if (%BugDescriptions)
+ {
+ print $Sendmail <<"EOF";
+In particular some failures are tracked in the bug(s) below:
+EOF
+ foreach my $BugId (sort { $a <=> $b } keys %BugDescriptions)
+ {
+ print $Sendmail "* $BugId - $BugDescriptions{$BugId}\n";
+ print $Sendmail " $WineBugUrl$BugId\n";
+ }
+ print $Sendmail "\n";
+ }
}
print $Sendmail " Failures\n";
@@ -360,7 +387,7 @@ EOF
my $TaskDir = $StepTask->GetTaskDir();
foreach my $LogName (@{$TaskInfo->{LogNames}})
{
- my $LogInfo = LoadLogErrors("$TaskDir/$LogName");
+ my $LogInfo = $TaskInfo->{$LogName};
next if (!defined $LogInfo->{BadLog} and !$LogInfo->{ErrCount});
print $Sendmail "\n=== ", GetTitle($StepTask, $LogName), " ===\n";
@@ -372,7 +399,11 @@ EOF
my $Group = $LogInfo->{ErrGroups}->{$GroupName};
foreach my $ErrIndex (0..$#{$Group->{Errors}})
{
- my $Prefix = $Group->{IsNew}->[$ErrIndex] ? "new" : "old";
+ my $LineNo = $Group->{LineNos}->[$ErrIndex];
+ my $Bugs = $LogInfo->{Bugs}->{$LineNo};
+ my $Prefix = $Bugs ? join(" ", sort { $a <=> $b } @$Bugs) :
+ $Group->{IsNew}->[$ErrIndex] ? "new" :
+ "old";
print $Sendmail "[$Prefix] $Group->{Errors}->[$ErrIndex]\n";
}
}
@@ -433,10 +464,9 @@ EOF
{
print $Sendmail <<"EOF";
It looks like your $FileType introduced the new failures shown below.
-Please investigate and fix them if they are indeed new. Note that rare
-failures and failures with always changing text (e.g. because of memory
-addresses) can cause false positives. If this is what happened, then
-fixing those would really help.
+Please investigate and fix them before resubmitting your $FileType.
+If they are not new, fixing them anyway would help a lot. Otherwise
+please ask for the known failures list to be updated.
EOF
}
Module: tools
Branch: master
Commit: 1b238d27711cf9b62fb5f304d19610c0472c7b05
URL: https://source.winehq.org/git/tools.git/?a=commit;h=1b238d27711cf9b62fb5f30…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Wed Jun 15 18:21:35 2022 +0200
testbot: Match the failures with the test logs.
This augments the .errors files with information mapping errors to known
failures.
Wine-Bug: https://bugs.winehq.org/show_bug.cgi?id=48912
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/lib/WineTestBot/LogUtils.pm | 180 +++++++++++++++++++++++++++++++++++-
1 file changed, 178 insertions(+), 2 deletions(-)
diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm
index 2e3de83b..b8aaaa43 100644
--- a/testbot/lib/WineTestBot/LogUtils.pm
+++ b/testbot/lib/WineTestBot/LogUtils.pm
@@ -36,7 +36,10 @@ our @EXPORT = qw(GetLogFileNames GetLogLabel
use Algorithm::Diff;
use File::Basename;
+use ObjectModel::Collection; # For CombineKey()
use WineTestBot::Config; # For $MaxUnitSize
+use WineTestBot::Failures;
+use WineTestBot::Tasks; # FIXME Hack for SaveLogFailures()
use WineTestBot::Utils; # For LocaleName()
@@ -187,6 +190,12 @@ sub _AddLogError($$$;$$)
Returns a hashtable containing a summary of the task log:
=over
+=item LogName
+The log file basename.
+
+=item LogPath
+The full log file path.
+
=item Type
'tests' if the task ran Wine tests and 'build' otherwise.
@@ -913,6 +922,10 @@ copy of that line (with CR/LF converted to a simple LF).
The format for new error lines is identical to that for old errors but with a
different type.
+=item f <errindex> <failureid1>...
+Failure lines contain the index of the error in the error group and a list of
+the known failures they match (normally at most one).
+
=back
=back
=cut
@@ -981,6 +994,10 @@ sub LoadLogErrorsFromFh($$)
{
_AddLogError($LogInfo, $LogInfo->{CurGroup}, $Value, $Property, "new");
}
+ elsif ($Type eq "f")
+ {
+ $LogInfo->{CurGroup}->{Failures}->{$Property} = [ split / /, $Value ];
+ }
else
{
$LogInfo->{BadLog} = "$LogInfo->{LineNo}: Found an unknown line type ($Type)";
@@ -1058,8 +1075,12 @@ sub _WriteLogErrorsToFh($$)
print $Fh "g $Group->{LineNo} $GroupName\n";
foreach my $Index (0..$#{$Group->{Errors}})
{
+ my $LineNo = $Group->{LineNos}->[$Index];
my $IsNew = $Group->{IsNew}->[$Index] ? "n" : "o";
- print $Fh "$IsNew $Group->{LineNos}->[$Index] $Group->{Errors}->[$Index]\n";
+ print $Fh "$IsNew $LineNo $Group->{Errors}->[$Index]\n";
+
+ my $Failures = $Group->{Failures}->{$Index};
+ print $Fh "f $Index @$Failures\n" if ($Failures);
}
}
}
@@ -1299,6 +1320,158 @@ sub TagNewErrors($$)
}
}
+=pod
+=over 12
+
+=item C<MatchLogFailures()>
+
+Checks the errors against known failures.
+
+The $LogInfo structure is augmented with the following fields:
+=over
+
+=item ErrGroups
+=over
+
+=item Failures
+A hashtable mapping error indices to the list of matching known
+failure ids.
+
+=back
+=back
+
+Returns a hashtable containing a summary of the log failures:
+=over
+
+=item LogName
+The log file basename.
+
+=item Collection
+A collection containing the relevant Failure objects.
+
+=item Failures
+A hashtable indexed by the failure ids. Each entry contains:
+
+=over
+
+=item Failure
+The failure object.
+
+=item NewCount
+A count errors matched by this known failure that were tagged as new.
+
+=item OldCount
+A count errors matched by this known failure that were tagged as old.
+
+=back
+
+=back
+=back
+=cut
+
+sub MatchLogFailures($$)
+{
+ my ($LogInfo, $Task) = @_;
+
+ my $LogFailures = {
+ Task => $Task,
+ LogName => $LogInfo->{LogName}
+ };
+ return $LogFailures if (!$LogInfo->{ErrCount});
+
+ my %FailureTree;
+ my $ConfigName = $Task->VMName .":$LogInfo->{LogName}";
+
+ $LogFailures->{Collection} = CreateFailures();
+ foreach my $Failure (@{$LogFailures->{Collection}->GetItems()})
+ {
+ # Ignore failures that don't apply to this configuration
+ my $ConfigRegExp = $Failure->ConfigRegExp;
+ my $Match = eval { $ConfigRegExp and $ConfigName =~ /$ConfigRegExp/ };
+ next if (!$Match);
+
+ my $UnitFailures = $FailureTree{$Failure->ErrorGroup}->{$Failure->TestUnit} ||= [];
+ push @$UnitFailures, $Failure;
+ }
+
+ foreach my $GroupName (@{$LogInfo->{ErrGroupNames}})
+ {
+ next if (!$FailureTree{$GroupName});
+
+ my $Group = $LogInfo->{ErrGroups}->{$GroupName};
+ foreach my $ErrIndex (0..$#{$Group->{Errors}})
+ {
+ my $Line = $Group->{Errors}->[$ErrIndex];
+ my $TestUnit = $Line =~ /^([_a-z0-9]+)\.c:\d+:/ ? $1 : "";
+ my $UnitFailures = $FailureTree{$GroupName}->{$TestUnit};
+ next if (!$UnitFailures);
+
+ foreach my $UnitFailure (@$UnitFailures)
+ {
+ my $RegExp = $UnitFailure->FailureRegExp;
+ my $Match = eval { $RegExp and $Line =~ /$RegExp/ };
+ next if (!$Match);
+
+ my $LineFailures = $Group->{Failures}->{$ErrIndex} ||= [];
+ push @$LineFailures, $UnitFailure->Id;
+
+ my $LogFailure = $LogFailures->{Failures}->{$UnitFailure->Id};
+ if (!$LogFailure)
+ {
+ $LogFailure = $LogFailures->{Failures}->{$UnitFailure->Id} =
+ { Failure => $UnitFailure };
+ }
+ my $Count = $Group->{IsNew}->[$ErrIndex] ? "NewCount" : "OldCount";
+ $LogFailure->{$Count}++;
+ }
+ }
+ }
+
+ return $LogFailures;
+}
+
+=pod
+=over 12
+
+=item C<SaveLogFailures()>
+
+Updates the TaskFailures objects for the task and log specified by the
+$LogFailures structure.
+
+Note that this implies deleting any preexisting TaskFailure to avoid leaving
+obsolete data.
+
+=back
+=cut
+
+sub SaveLogFailures($)
+{
+ my ($LogFailures) = @_;
+
+ # FIXME $Task->Failures->AddFilter() cannot be undone and impacts every
+ # future use of $Task->Failures. So add the filter on a throw away
+ # clone to not end up with a nonsensical filter.
+ my $TaskFailures = $LogFailures->{Task}->Failures->Clone();
+ $TaskFailures->AddFilter("TaskLog", [$LogFailures->{LogName}]);
+ my $ErrMessage = $TaskFailures->DeleteAll();
+ return $ErrMessage if (defined $ErrMessage);
+ return undef if (!$LogFailures->{Failures});
+
+ foreach my $LogFailure (values %{$LogFailures->{Failures}})
+ {
+ my $TaskFailure = $LogFailure->{Failure}->TaskFailures->Add();
+ my $OldKey = $TaskFailure->GetKey();
+ $TaskFailure->Task($LogFailures->{Task});
+ $TaskFailure->TaskLog($LogFailures->{LogName});
+ $TaskFailure->KeyChanged($OldKey, $TaskFailure->GetKey());
+ $TaskFailure->NewCount($LogFailure->{NewCount});
+ $TaskFailure->OldCount($LogFailure->{OldCount});
+ }
+
+ (my $_ErrKey, my $_ErrProperty, $ErrMessage) = $LogFailures->{Collection}->Save();
+ return $ErrMessage;
+}
+
#
# Log errors caching [Part 2]
@@ -1322,8 +1495,11 @@ sub CreateLogErrorsCache($;$)
# Don't mark the errors as new if there is no reference WineTest report
# as this would cause false positives.
}
+ my $LogFailures = MatchLogFailures($LogInfo, $Task) if ($Task);
- return _SaveLogErrors($LogInfo);
+ my $ErrMessage = _SaveLogErrors($LogInfo);
+ $ErrMessage ||= SaveLogFailures($LogFailures) if ($Task);
+ return $ErrMessage;
}