Module: tools Branch: master Commit: f1fe0ca977f17a1cb5a631e4d4c1c2f0c77a244f URL: https://gitlab.winehq.org/winehq/tools/-/commit/f1fe0ca977f17a1cb5a631e4d4c1...
Author: Francois Gouget fgouget@codeweavers.com Date: Wed Oct 19 19:19:07 2022 +0200
testbot: Track which build warnings are new.
This requires keeping the past build logs as a reference and comparing the new build logs to them. Note that because of the build parallelism the warnings may not always appear in the same order in the reference build logs so there could still be some false positives. If that happens the false positives can be avoided by tracking the warnings as known failures. Regardless, even new warnings do not cause a patch to be reported as bad.
---
testbot/bin/Janitor.pl | 2 +- testbot/bin/UpdateTaskLogs | 8 ++++---- testbot/bin/WineRunBuild.pl | 6 +++++- testbot/bin/WineRunReconfig.pl | 14 +++++++++++++- testbot/bin/WineRunWineTest.pl | 6 +++++- testbot/lib/WineTestBot/LogUtils.pm | 17 ++++++++--------- 6 files changed, 36 insertions(+), 17 deletions(-)
diff --git a/testbot/bin/Janitor.pl b/testbot/bin/Janitor.pl index 069ffb76..169bc3cb 100755 --- a/testbot/bin/Janitor.pl +++ b/testbot/bin/Janitor.pl @@ -347,7 +347,7 @@ if (opendir(my $dh, "$DataDir/latest")) my $TTL = $JobPurgeDays ? $JobPurgeDays - $Age : undef;
# Keep the regexp in sync with WineTestBot::Task::GetRefReportName() - if ($Entry !~ /^[a-zA-Z0-9_]+-job[0-9]+-[a-zA-Z0-9_]+.report(?:.errors)?$/) + if ($Entry !~ /^[a-zA-Z0-9_]+-job[0-9]+-[a-zA-Z0-9_]+.(?:log|report)(?:.errors)?$/) { my $Deletion = defined $TTL ? " (deletion in $TTL days)" : ""; Error "Found a suspicious file$Deletion: latest/$Entry\n"; diff --git a/testbot/bin/UpdateTaskLogs b/testbot/bin/UpdateTaskLogs index e50e449c..8a74fd7b 100755 --- a/testbot/bin/UpdateTaskLogs +++ b/testbot/bin/UpdateTaskLogs @@ -402,8 +402,6 @@ sub ProcessTaskLogs($$$$) # Take a snapshot of the reference reports older than this Task foreach my $ReportName (@{GetLogFileNames($TaskDir)}) { - next if ($ReportName !~ /.report$/); - Debug(TaskKeyStr($Task) .": Snapshotting the reference reports for $ReportName\n"); my $ErrMessages = SnapshotLatestReport($Task, $ReportName); foreach my $ErrMessage (@$ErrMessages) @@ -456,7 +454,8 @@ sub ProcessTaskLogs($$$$) } }
- if (!$OptDelete and $Task->Status eq "completed" and $Step->Type eq "suite") + if (!$OptDelete and $Task->Status eq "completed" and + ($Step->Type eq "suite" or $Step->Type eq "reconfig")) { # Update the latest reference reports # WineTest runs that timed out are missing results which would cause false @@ -468,7 +467,8 @@ sub ProcessTaskLogs($$$$) { foreach my $ReportName (@{GetLogFileNames($TaskDir)}) { - next if ($ReportName !~ /.report$/); + next if ($Step->Type eq "suite" and $ReportName !~ /.report$/); + next if ($Step->Type eq "reconfig" and $ReportName !~ /.log$/); next if (-z "$TaskDir/$ReportName"); $Rc += DoUpdateLatestReport($Task, $ReportName, "$TaskDir/$ReportName"); } diff --git a/testbot/bin/WineRunBuild.pl b/testbot/bin/WineRunBuild.pl index c664a8b8..934a9aa3 100755 --- a/testbot/bin/WineRunBuild.pl +++ b/testbot/bin/WineRunBuild.pl @@ -415,6 +415,10 @@ if ($TA->GetFile("Build.log", "$TaskDir/task.log")) $TAError = $ErrMessage = undef; # ...but keep keep the task.log ones $TaskFailures = $LogInfo->{eCount}; + + # Grab a copy of the reference report + my $ErrMessages = SnapshotLatestReport($Task, "task.log"); + LogTaskError("$_\n") for (@$ErrMessages); } elsif ($LogInfo->{Task} eq "badpatch") { @@ -432,7 +436,7 @@ if ($TA->GetFile("Build.log", "$TaskDir/task.log")) # that explains why. $NewStatus = "badbuild"; } - my $LogErrMsg = CreateLogErrorsCache($LogInfo); + my $LogErrMsg = CreateLogErrorsCache($LogInfo, $Task); LogTaskError("$LogErrMsg\n") if (defined $LogErrMsg); } elsif (!defined $TAError) diff --git a/testbot/bin/WineRunReconfig.pl b/testbot/bin/WineRunReconfig.pl index 889af4bd..90004545 100755 --- a/testbot/bin/WineRunReconfig.pl +++ b/testbot/bin/WineRunReconfig.pl @@ -254,6 +254,14 @@ sub WrapUpAndExit($;$$$$) $VM->Save(); }
+ # Update the 'latest/' reference WineTest results + # Ignore the results from failed or timed out builds. + if ($Status eq 'completed' and !$TimedOut) + { + my $ErrMessages = UpdateLatestReports($Task, ["task.log"]); + Error("$_\n") for (@$ErrMessages); + } + my $Result = $VM->Name .": ". $VM->Status ." Status: $Status Failures: ". (defined $TestFailures ? $TestFailures : "unset"); LogMsg "Task $JobId/$StepNo/$TaskNo done ($Result)\n"; Debug(Elapsed($Start), " Done. $Result\n"); @@ -417,6 +425,10 @@ if ($TA->GetFile("Reconfig.log", "$TaskDir/task.log")) $TAError = $ErrMessage = undef; # ...but keep keep the task.log ones $TaskFailures = $LogInfo->{eCount}; + + # Grab a copy of the reference report + my $ErrMessages = SnapshotLatestReport($Task, "task.log"); + LogTaskError("$_\n") for (@$ErrMessages); } elsif (defined $LogInfo->{BadLog}) { @@ -435,7 +447,7 @@ if ($TA->GetFile("Reconfig.log", "$TaskDir/task.log")) MakeOfficialURL(GetTaskURL($JobId, $StepNo, $TaskNo)) ."\n"); $NewStatus = "badbuild"; } - my $LogErrMsg = CreateLogErrorsCache($LogInfo); + my $LogErrMsg = CreateLogErrorsCache($LogInfo, $Task); LogTaskError("$LogErrMsg\n") if (defined $LogErrMsg); } elsif (!defined $TAError) diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl index 985fdc2a..8bc8e356 100755 --- a/testbot/bin/WineRunWineTest.pl +++ b/testbot/bin/WineRunWineTest.pl @@ -517,6 +517,10 @@ if ($TA->GetFile("Task.log", "$TaskDir/task.log")) $TAError = $ErrMessage = $PossibleCrash = undef; # Reset the timeout error but keep the task.log ones $TaskFailures = $LogInfo->{eCount}; + + # Grab a copy of the reference report + my $ErrMessages = SnapshotLatestReport($Task, "task.log"); + LogTaskError("$_\n") for (@$ErrMessages); } elsif ($LogInfo->{Task} eq "badpatch") { @@ -543,7 +547,7 @@ if ($TA->GetFile("Task.log", "$TaskDir/task.log")) $TaskFailures = undef; $PossibleCrash = 1; } - my $LogErrMsg = CreateLogErrorsCache($LogInfo); + my $LogErrMsg = CreateLogErrorsCache($LogInfo, $Task); LogTaskError("$LogErrMsg\n") if (defined $LogErrMsg); } elsif (!defined $TAError) diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm index e60dec8a..56129e10 100644 --- a/testbot/lib/WineTestBot/LogUtils.pm +++ b/testbot/lib/WineTestBot/LogUtils.pm @@ -1585,17 +1585,16 @@ sub CreateLogErrorsCache($;$)
return $LogInfo->{BadLog} if (defined $LogInfo->{BadLog});
- if ($LogInfo->{LogName} !~ /.report$/) - { - # Only test reports have reference WineTest results. - # So for other logs all errors are new. - MarkAllMessagesAsNew($LogInfo); - } - elsif ($Task and @{$LogInfo->{MsgGroupNames}}) + if ($Task and @{$LogInfo->{MsgGroupNames}}) { TagNewMessages($LogInfo, $Task); - # Don't mark the errors as new if there is no reference WineTest report - # as this would cause false positives. + if ($LogInfo->{NoRef} and $LogInfo->{LogName} !~ /.report$/) + { + # In the absence of a reference report, consider all non-test errors + # to be new, mostly for the benefit of build errors. + # But don't do so for test errors to avoid false positives. + MarkAllMessagesAsNew($LogInfo); + } } my $LogFailures = MatchLogFailures($LogInfo, $Task) if ($Task);