Also take the latest WineTest reports snapshot when the corresponding task completes rather than when the build task they depend on completes. This way the snapshot is taken closest to when the reference reports are needed.
Signed-off-by: Francois Gouget fgouget@codeweavers.com --- testbot/bin/UpdateTaskLogs | 12 ++-- testbot/bin/WineRunBuild.pl | 31 --------- testbot/bin/WineRunTask.pl | 34 ++++----- testbot/bin/WineRunWineTest.pl | 65 +++-------------- testbot/lib/WineTestBot/LogUtils.pm | 104 +++++++++++++++++++++++++++- testbot/lib/WineTestBot/Tasks.pm | 30 ++++++++ 6 files changed, 160 insertions(+), 116 deletions(-)
diff --git a/testbot/bin/UpdateTaskLogs b/testbot/bin/UpdateTaskLogs index f5205f23d8..5f858dece6 100755 --- a/testbot/bin/UpdateTaskLogs +++ b/testbot/bin/UpdateTaskLogs @@ -230,15 +230,11 @@ sub DoUpdateLatestReport($$$) # Add the reference report to latest/ Debug("latest: Adding $RefReportName from ". Path2TaskKey($SrcReportPath) ."\n");
- foreach my $Suffix ("", ".err") + my $ErrMessages = UpdateLatestReport($Task, $ReportName, $SrcReportPath); + foreach my $ErrMessage (@$ErrMessages) { - unlink("$LatestReportPath$Suffix"); - if (-f "$SrcReportPath$Suffix" and - !link("$SrcReportPath$Suffix", "$LatestReportPath$Suffix")) - { - Error "Could not replace 'latest/$RefReportName$Suffix': $!\n"; - $Rc = 1; - } + Error "$ErrMessage\n"; + $Rc = 1; } $LatestReports{$RefReportName} = $LatestReportPath; } diff --git a/testbot/bin/WineRunBuild.pl b/testbot/bin/WineRunBuild.pl index 9fc5bffe24..6b68c0dc47 100755 --- a/testbot/bin/WineRunBuild.pl +++ b/testbot/bin/WineRunBuild.pl @@ -496,37 +496,6 @@ Debug(Elapsed($Start), " Disconnecting\n"); $TA->Disconnect();
-# -# Grab a copy of the reference logs -# - -# Note that this may be a bit inaccurate right after a Wine commit. -# See WineSendLog.pl for more details. -my $LatestDir = "$DataDir/latest"; -foreach my $TestStep (@{$Job->Steps->GetItems()}) -{ - if (($TestStep->PreviousNo || 0) == $Step->No and - $TestStep->FileType =~ /^exe/) - { - foreach my $TestTask (@{$TestStep->Tasks->GetItems()}) - { - my $RefReport = $TestTask->GetRefReportName($TestStep->FileType .".report"); - for my $Suffix ("", ".err") - { - if (-f "$LatestDir/$RefReport$Suffix") - { - unlink "$StepDir/$RefReport$Suffix"; - if (!link "$LatestDir/$RefReport$Suffix", "$StepDir/$RefReport$Suffix") - { - Error "Could not link '$RefReport$Suffix': $!\n"; - } - } - } - } - } -} - - # # Wrap up # diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl index 7c8c5704fe..2b54048bac 100755 --- a/testbot/bin/WineRunTask.pl +++ b/testbot/bin/WineRunTask.pl @@ -45,7 +45,6 @@ use WineTestBot::Engine::Notify; use WineTestBot::Jobs; use WineTestBot::Log; use WineTestBot::LogUtils; -use WineTestBot::Missions; use WineTestBot::Utils; use WineTestBot::VMs;
@@ -195,7 +194,6 @@ my $OldUMask = umask(002); my $TaskDir = $Task->CreateDir(); umask($OldUMask); my $VM = $Task->VM; -my $RptFileName = $Step->FileType .".report";
my $Start = Time(); @@ -224,6 +222,8 @@ sub LogTaskError($) umask($OldUMask); }
+my $ReportNames; + sub WrapUpAndExit($;$$$$) { my ($Status, $TestFailures, $Retry, $TimedOut, $Reason) = @_; @@ -299,22 +299,11 @@ sub WrapUpAndExit($;$$$$) $VM->Save(); }
+ # Update the 'latest/' reference WineTest results if ($Step->Type eq 'suite' and $Status eq 'completed' and !$TimedOut) { - # Keep the old report if the new one is missing - if (-f "$TaskDir/$RptFileName" and !-z "$TaskDir/$RptFileName") - { - # Update the VM's reference WineTest results for WineSendLog.pl - my $RefReport = "$DataDir/latest/". $Task->GetRefReportName($RptFileName); - unlink($RefReport); - link("$TaskDir/$RptFileName", $RefReport); - - unlink("$RefReport.err"); - if (-f "$TaskDir/$RptFileName.err" and !-z "$TaskDir/$RptFileName.err") - { - link("$TaskDir/$RptFileName.err", "$RefReport.err"); - } - } + my $ErrMessages = UpdateLatestReports($Task, $ReportNames); + Error("$_\n") for (@$ErrMessages); }
my $Result = $VM->Name .": ". $VM->Status ." Status: $Status Failures: ". (defined $TestFailures ? $TestFailures : "unset"); @@ -420,12 +409,11 @@ if ($Step->FileType ne "exe32" and $Step->FileType ne "exe64") FatalError("Unexpected file type '". $Step->FileType ."' found for ". $Step->Type ." step\n"); }
-my ($ErrMessage, $Missions) = ParseMissionStatement($Task->Missions); +(my $ErrMessage, $ReportNames, my $TaskMissions) = $Task->GetReportNames(); FatalError "$ErrMessage\n" if (defined $ErrMessage); -FatalError "Empty mission statement\n" if (!@$Missions); -FatalError "Cannot specify missions for multiple tasks\n" if (@$Missions > 1); -FatalError "Cannot specify multiple missions\n" if (@{$Missions->[0]->{Missions}} > 1); -my $Mission = $Missions->[0]->{Missions}->[0]; +FatalError "Cannot specify multiple missions\n" if (@{$TaskMissions->{Missions}} > 1); +my $Mission = $TaskMissions->{Missions}->[0]; +my $RptFileName = $ReportNames->[0];
# @@ -567,6 +555,10 @@ if ($TA->GetFile($RptFileName, "$TaskDir/$RptFileName")) } else { + # Grab a copy of the reference report + my $ErrMessages = SnapshotLatestReport($Task, $RptFileName); + LogTaskError("$_\n") for (@$ErrMessages); + # $LogInfo->{Failures} can legitimately be undefined in case of a timeout $TaskFailures += $LogInfo->{Failures} || 0; if (@{$LogInfo->{Extra}} and open(my $Log, ">", "$TaskDir/$RptFileName.err")) diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl index 6ca1d284fe..9a0a4ebafc 100755 --- a/testbot/bin/WineRunWineTest.pl +++ b/testbot/bin/WineRunWineTest.pl @@ -43,7 +43,6 @@ $Name0 =~ s+^.*/++; use WineTestBot::Config; use WineTestBot::Engine::Notify; use WineTestBot::Jobs; -use WineTestBot::Missions; use WineTestBot::PatchUtils; use WineTestBot::Log; use WineTestBot::LogUtils; @@ -218,7 +217,7 @@ sub LogTaskError($) } }
-my $TaskMissions; +my $ReportNames;
sub WrapUpAndExit($;$$$$) { @@ -295,26 +294,11 @@ sub WrapUpAndExit($;$$$$) $VM->Save(); }
+ # Update the 'latest/' reference WineTest results if ($Step->Type eq 'suite' and $Status eq 'completed' and !$TimedOut) { - foreach my $Mission (@{$TaskMissions->{Missions}}) - { - # Keep the old report if the new one is missing - my $RptFileName = GetMissionBaseName($Mission) .".report"; - if (-f "$TaskDir/$RptFileName" and !-z "$TaskDir/$RptFileName") - { - # Update the VM's reference WineTest results for WineSendLog.pl - my $RefReport = "$DataDir/latest/". $Task->VM->Name ."_$RptFileName"; - unlink($RefReport); - link("$TaskDir/$RptFileName", $RefReport); - - unlink("$RefReport.err"); - if (-f "$TaskDir/$RptFileName.err" and !-z "$TaskDir/$RptFileName.err") - { - link("$TaskDir/$RptFileName.err", "$RefReport.err"); - } - } - } + my $ErrMessages = UpdateLatestReports($Task, $ReportNames); + Error("$_\n") for (@$ErrMessages); }
my $Result = $VM->Name .": ". $VM->Status ." Status: $Status Failures: ". (defined $TestFailures ? $TestFailures : "unset"); @@ -421,11 +405,8 @@ if (($Step->Type eq "suite" and $Step->FileType ne "none") or FatalError("Unexpected file type '". $Step->FileType ."' found for ". $Step->Type ." step\n"); }
-my ($ErrMessage, $Missions) = ParseMissionStatement($Task->Missions); +(my $ErrMessage, $ReportNames, my $_TaskMissions) = $Task->GetReportNames(); FatalError "$ErrMessage\n" if (defined $ErrMessage); -FatalError "Empty mission statement\n" if (!@$Missions); -FatalError "Cannot specify missions for multiple tasks\n" if (@$Missions > 1); -$TaskMissions = $Missions->[0];
# @@ -570,9 +551,8 @@ elsif (!defined $TAError) # Grab the test reports if any #
-foreach my $Mission (@{$TaskMissions->{Missions}}) +foreach my $RptFileName (@$ReportNames) { - my $RptFileName = GetMissionBaseName($Mission) .".report"; Debug(Elapsed($Start), " Retrieving '$RptFileName'\n"); if ($TA->GetFile($RptFileName, "$TaskDir/$RptFileName")) { @@ -589,6 +569,10 @@ foreach my $Mission (@{$TaskMissions->{Missions}}) } else { + # Grab a copy of the reference report + my $ErrMessages = SnapshotLatestReport($Task, $RptFileName); + LogTaskError("$_\n") for (@$ErrMessages); + # $LogInfo->{Failures} can legitimately be undefined in case of a timeout $TaskFailures += $LogInfo->{Failures} || 0; if (@{$LogInfo->{Extra}} and open(my $Log, ">", "$TaskDir/$RptFileName.err")) @@ -617,35 +601,6 @@ Debug(Elapsed($Start), " Disconnecting\n"); $TA->Disconnect();
-# -# Grab a copy of the reference logs -# - -# Note that this may be a bit inaccurate right after a Wine commit. -# See WineSendLog.pl for more details. -if ($NewStatus eq 'completed') -{ - my $LatestDir = "$DataDir/latest"; - my $StepDir = $Step->GetDir(); - foreach my $Mission (@{$TaskMissions->{Missions}}) - { - my $RptFileName = GetMissionBaseName($Mission) .".report"; - my $RefReport = $Task->GetRefReportName($RptFileName); - for my $Suffix ("", ".err") - { - if (-f "$LatestDir/$RefReport$Suffix") - { - unlink "$StepDir/$RefReport$Suffix"; - if (!link "$LatestDir/$RefReport$Suffix", "$StepDir/$RefReport$Suffix") - { - Error "Could not link '$RefReport$Suffix': $!\n"; - } - } - } - } -} - - # # Wrap up # diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm index 4d0d99bfab..5b567bc23b 100644 --- a/testbot/lib/WineTestBot/LogUtils.pm +++ b/testbot/lib/WineTestBot/LogUtils.pm @@ -29,7 +29,8 @@ WineTestBot::LogUtils - Provides functions to parse task logs use Exporter 'import'; our @EXPORT = qw(GetLogFileNames GetLogLabel GetLogErrors TagNewErrors GetLogLineCategory GetReportLineCategory - ParseTaskLog ParseWineTestReport); + ParseTaskLog ParseWineTestReport + SnapshotLatestReport UpdateLatestReport UpdateLatestReports);
use Algorithm::Diff; use File::Basename; @@ -873,6 +874,11 @@ sub GetLogErrors($) return $LogInfo; }
+ +# +# New error detection +# + sub _DumpDiff($$) { my ($Label, $Diff) = @_; @@ -1031,4 +1037,100 @@ sub TagNewErrors($$) } }
+ +# +# Reference report management +# + +=pod +=over 12 + +=item C<SnapshotLatestReport()> + +Takes a snapshot of the reference WineTest results for the specified Task. + +The reference report is used to identify new failures, even long after the task has been +run (and the reference report replaced by a newer version). + +Note also that comparing reports in this way may be a bit inaccurate right +after a Wine commit due to delays in getting new WineTest results, etc. +See WineSendLog.pl for more details. + +=back +=cut + +sub SnapshotLatestReport($$) +{ + my ($Task, $ReportName) = @_; + + my @ErrMessages; + my $TaskDir = $Task->GetDir(); + my $RefReportName = $Task->GetRefReportName($ReportName); + foreach my $Suffix ("", ".err") + { + next if (!-f "$DataDir/latest/$RefReportName$Suffix"); + + # FIXME: The reference reports are stored at the step level! + if (!link("$DataDir/latest/$RefReportName$Suffix", + "$TaskDir/../$RefReportName$Suffix")) + { + push @ErrMessages, "Could not create the '../$RefReportName$Suffix' link: $!"; + } + } + + return @ErrMessages; +} + +sub UpdateLatestReport($$$) +{ + my ($Task, $ReportName, $SrcReportPath) = @_; + my @ErrMessages; + + my $RefReportName = $Task->GetRefReportName($ReportName); + foreach my $Suffix ("", ".err") + { + # Add the new reference file even if it is empty. + next if (!-f "$SrcReportPath$Suffix"); + + unlink "$DataDir/latest/$RefReportName$Suffix"; + if (!link("$SrcReportPath$Suffix", + "$DataDir/latest/$RefReportName$Suffix")) + { + push @ErrMessages, "Could not create the '$RefReportName$Suffix' link: $!"; + } + } + + return @ErrMessages; +} + +=pod +=over 12 + +=item C<UpdateLatestReports()> + +Adds the Task's WineTest results to the set of reference reports. + +The reference reports will then be used to detect new failures in the other +tasks. + +This must be called after SnapshotLatestReport() otherwise a WineTest task +would compare its results to itself. + +=back +=cut + +sub UpdateLatestReports($$) +{ + my ($Task, $ReportNames) = @_; + + my @ErrMessages; + my $TaskDir = $Task->GetDir(); + foreach my $ReportName (@$ReportNames) + { + next if (!-f "$TaskDir/$ReportName" or -z _); + push @ErrMessages, @{UpdateLatestReport($Task, $ReportName, "$TaskDir/$ReportName")}; + } + return @ErrMessages; +} + 1; diff --git a/testbot/lib/WineTestBot/Tasks.pm b/testbot/lib/WineTestBot/Tasks.pm index 0c26904e15..1a46f0d57c 100644 --- a/testbot/lib/WineTestBot/Tasks.pm +++ b/testbot/lib/WineTestBot/Tasks.pm @@ -70,6 +70,7 @@ our @ISA = qw(WineTestBot::WineTestBotItem); use File::Path; use ObjectModel::BackEnd; use WineTestBot::Config; +use WineTestBot::Missions;
sub InitializeNew($$) @@ -108,6 +109,35 @@ sub RmTree($) rmtree($Dir); }
+sub GetReportNames($) +{ + my ($self) = @_; + + my ($ErrMessage, $Missions) = ParseMissionStatement($self->Missions); + return ($ErrMessage, undef, undef) if (defined $ErrMessage); + if (!@$Missions) + { + my @TaskKey = $self->GetMasterKey(); + return ("Task @TaskKey has no mission", undef, undef); + } + if (@$Missions > 1) + { + my @TaskKey = $self->GetMasterKey(); + return ("Task @TaskKey should not have missions for multiple tasks\n", undef); + } + my $TaskMissions = $Missions->[0]; + + my @ReportNames; + foreach my $Mission (@{$TaskMissions->{Missions}}) + { + if ($Mission->{test} ne "build") + { + push @ReportNames, GetMissionBaseName($Mission) .".report"; + } + } + return (undef, @ReportNames, $TaskMissions); +} + sub GetRefReportName($$) { my ($self, $ReportName) = @_;