[PATCH] testbot/UpdateTaskLogs: Delete or recreate files derived from task logs.
UpdateTaskLogs can rebuild the reference WineTest reports used for the detection of new failures, and the .err files that store extra errors. The reference reports are created from the reports of the WineTest tasks, based on their completion time. Note that it is not possible to recreate reference reports for tasks that are older than the first completed WineTest task. But such issues should go away when these tasks get expired on the next day. This allows fixing up the reference reports after an upgrade or a downgrade. For instance for a downgrade one could run 'UpdateTaskLogs --delete' first to delete files specific to the new code, then downgrade and run 'UpdateTaskLogs' to regenerate the files needed by the old code. Note however that this does not undo any rename operations done when upgrading. Those would need to be reverted by hand. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/bin/UpdateTaskLogs | 468 +++++++++++++++++++++++++++++++++++++ 1 file changed, 468 insertions(+) create mode 100755 testbot/bin/UpdateTaskLogs diff --git a/testbot/bin/UpdateTaskLogs b/testbot/bin/UpdateTaskLogs new file mode 100755 index 0000000000..ae3b305388 --- /dev/null +++ b/testbot/bin/UpdateTaskLogs @@ -0,0 +1,468 @@ +#!/usr/bin/perl -Tw +# -*- Mode: Perl; perl-indent-level: 2; indent-tabs-mode: nil -*- +# +# Updates or recreates the reference reports for the specified tasks and the +# latest directory. +# +# Copyright 2019 Francois Gouget +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +use strict; + +sub BEGIN +{ + if ($0 !~ m=^/=) + { + # Turn $0 into an absolute path so it can safely be used in @INC + require Cwd; + $0 = Cwd::cwd() . "/$0"; + } + if ($0 =~ m=^(/.*)/[^/]+/[^/]+$=) + { + $::RootDir = $1; + unshift @INC, "$::RootDir/lib"; + } +} +my $Name0 = $0; +$Name0 =~ s+^.*/++; + + +use File::Basename; + +use WineTestBot::Config; +use WineTestBot::Jobs; +use WineTestBot::Log; +use WineTestBot::LogUtils; + + +# +# Logging and error handling helpers +# + +my $Debug; +sub Debug(@) +{ + print STDERR @_ if ($Debug); +} + +my $LogOnly; +sub Error(@) +{ + print STDERR "$Name0:error: ", @_ if (!$LogOnly); + LogMsg @_; +} + + +# +# Setup and command line processing +# + +my $Usage; +sub ValidateNumber($$) +{ + my ($Name, $Value) = @_; + + # Validate and untaint the value + return $1 if ($Value =~ /^(\d+)$/); + Error "$Value is not a valid $Name\n"; + $Usage = 2; + return undef; +} + +my ($OptDelete, $OptRebuild, $OptJobId, $OptStepNo, $OptTaskNo); +while (@ARGV) +{ + my $Arg = shift @ARGV; + if ($Arg eq "--delete") + { + $OptDelete = 1; + } + elsif ($Arg eq "--rebuild") + { + $OptRebuild = 1; + } + elsif ($Arg eq "--debug") + { + $Debug = 1; + } + elsif ($Arg eq "--log-only") + { + $LogOnly = 1; + } + elsif ($Arg =~ /^(?:-\?|-h|--help)$/) + { + $Usage = 0; + last; + } + elsif ($Arg =~ /^-/) + { + Error "unknown option '$Arg'\n"; + $Usage = 2; + last; + } + elsif (!defined $OptJobId) + { + $OptJobId = ValidateNumber('job id', $Arg); + } + elsif (!defined $OptStepNo) + { + $OptStepNo = ValidateNumber('step number', $Arg); + } + elsif (!defined $OptTaskNo) + { + $OptTaskNo = ValidateNumber('task number', $Arg); + } + else + { + Error "unexpected argument '$Arg'\n"; + $Usage = 2; + last; + } +} + +# Check parameters +if (!defined $Usage) +{ + if ($OptDelete and $OptRebuild) + { + Error "--delete and --rebuild are mutually exclusive\n"; + $Usage = 2; + } +} +if (defined $Usage) +{ + print "Usage: $Name0 [--rebuild] [--log-only] [--debug] [--help] [JOBID [STEPNO [TASKNO]]]\n"; + print "\n"; + print "Deletes, upgrades or recreates the reference reports and .err files for the specified tasks and the latest directory.\n"; + print "\n"; + print "Where:\n"; + print " --delete Delete generated files of both the latest directory and the\n"; + print " tasks.\n"; + print " --rebuild Delete and recreate the generated files of both the latest\n"; + print " directory and the tasks from the reports of the WineTest tasks,\n"; + print " based on their completion time. For tasks older than the first\n"; + print " completed WineTest task, the individual tasks' reference\n"; + print " reports are preserved. If these tasks don't have reference\n"; + print " reports, an error is shown.\n"; + print " Without this option only the missing reference reports and\n"; + print " .err files are added.\n"; + print " JOBID STEPNO TASKNO Only the tasks matching the specified JOBID, STEPNO and\n"; + print " TASKNO will be modified.\n"; + print " --log-only Only print errors to the log. By default they also go to stderr.\n"; + print " --debug Print additional debugging information.\n"; + print " --help Shows this usage message.\n"; + + exit $Usage; +} + +sub TaskKeyStr($) +{ + my ($Task) = @_; + return join("/", @{$Task->GetMasterKey()}); +} + +sub Path2TaskKey($) +{ + my ($Path) = @_; + + return "latest" if (!defined $Path); + return "latest" if ($Path =~ m~/latest(?:/|$)~); + $Path =~ s~^.*/jobs/(\d+/\d+/\d+)(/.*)?$~$1~; + return $Path; +} + +sub BuildErrFile($$$$) +{ + my ($Dir, $ReportName, $IsWineTest, $TaskTimedOut) = @_; + + my $TaskKey = Path2TaskKey($Dir); + + my ($TestUnitCount, $TimeoutCount, $LogFailures, $LogErrors) = ParseWineTestReport("$Dir/$ReportName", $IsWineTest, $TaskTimedOut); + if (!defined $LogFailures and @$LogErrors == 1) + { + return "Unable to open '$TaskKey/$ReportName' for reading: $!"; + } + return undef if (!@$LogErrors); + + Debug("$TaskKey: Creating $ReportName.err\n"); + if (open(my $Log, ">", "$Dir/$ReportName.err")) + { + # Save the extra errors detected by ParseWineTestReport() in + # $ReportName.err (see WineRunWineTest.pl). + print $Log "$_\n" for (@$LogErrors); + close($Log); + return undef; + } + return "Unable to open '$TaskKey/$ReportName' for reading: $!"; +} + +# Maps the reference report name to its full pathname. +# Note that the file may not be in latest/. +my %LatestReports; + +# The set of latest/ reference report files to rebuild. +my %LatestRebuilds; + +sub DoUpdateLatestReport($$$) +{ + my ($Task, $ReportName, $SrcReportPath) = @_; + + my $RefReportName = $Task->GetRefReportName($ReportName); + my $LatestReportPath = "$DataDir/latest/$RefReportName"; + if (!defined $OptJobId and !$OptDelete and !-f $LatestReportPath) + { + $LatestRebuilds{$RefReportName} = 1; + } + + my $Rc = 0; + if ($LatestRebuilds{$RefReportName}) + { + # Add the reference report to latest/ + Debug("latest: Adding $RefReportName from ". Path2TaskKey($SrcReportPath) ."\n"); + + foreach my $Suffix ("", ".err") + { + unlink("$LatestReportPath$Suffix"); + if (-f "$SrcReportPath$Suffix" and + !link("$SrcReportPath$Suffix", "$LatestReportPath$Suffix")) + { + Error "Could not replace 'latest/$RefReportName$Suffix': $!\n"; + $Rc = 1; + } + } + $LatestReports{$RefReportName} = $LatestReportPath; + } + # else only add task reports, not their own reference reports + elsif ($SrcReportPath =~ m~/\Q$ReportName\E$~) + { + $LatestReports{$RefReportName} = $SrcReportPath; + } + return $Rc; +} + +sub ProcessTaskLogs($$$) +{ + my ($Step, $Task, $CollectOnly) = @_; + + my $Rc = 0; + my $StepDir = $Step->GetDir(); + my $TaskDir = $Task->GetDir(); + + if (($OptDelete or $OptRebuild) and !$CollectOnly) + { + # Save / delete the task's reference reports + foreach my $ReportName (@{GetLogFileNames($TaskDir)}) + { + next if ($ReportName !~ /\.report$/); + + my $RefReportName = $Task->GetRefReportName($ReportName); + my $RefReportPath = "$StepDir/$RefReportName"; + + if (-f $RefReportPath or -f "$RefReportPath.err") + { + if (!-f "$RefReportPath.err") + { + # (Re)Build the .err file before adding the reference report to + # latest/. + my $ErrMessage = BuildErrFile($StepDir, $RefReportName, 1, 0); + if (defined $ErrMessage) + { + Error "$ErrMessage\n"; + $Rc = 1; + } + } + + # Save this report to latest/ in case it's not already present there + # (this would be the case for the oldest tasks with --rebuild) + $Rc += DoUpdateLatestReport($Task, $ReportName, $RefReportPath); + + Debug(TaskKeyStr($Task) .": Deleting ../$RefReportName\n"); + } + foreach my $Suffix ("", ".err") + { + if (!unlink "$RefReportPath$Suffix" and -e "$RefReportPath$Suffix") + { + Error "Could not delete '$RefReportPath$Suffix': $!\n"; + $Rc = 1; + } + } + + next if (!-f "$TaskDir/$ReportName.err"); + Debug(TaskKeyStr($Task) .": Deleting $ReportName.err\n"); + if (!unlink "$TaskDir/$ReportName.err") + { + Error "Could not delete '$ReportName.err': $!\n"; + $Rc = 1; + } + } + } + + if (!$OptDelete and !$CollectOnly and $Task->Status eq "completed") + { + # Take a snapshot of the latest reference reports + foreach my $ReportName (@{GetLogFileNames($TaskDir)}) + { + next if ($ReportName !~ /\.report$/); + my $RefReportName = $Task->GetRefReportName($ReportName); + next if (-f "$StepDir/$RefReportName"); + + my $LatestReportPath = $LatestReports{$RefReportName}; + if (!defined $LatestReportPath) + { + Error TaskKeyStr($Task) .": Missing '$RefReportName' reference report\n"; + $Rc = 1; + } + else + { + Debug(TaskKeyStr($Task) .": Snapshotting $RefReportName from ". Path2TaskKey($LatestReportPath) ."\n"); + foreach my $Suffix ("", ".err") + { + unlink "$StepDir/$RefReportName$Suffix"; + if (-f "$LatestReportPath$Suffix" and + !link("$LatestReportPath$Suffix", "$StepDir/$RefReportName$Suffix")) + { + Error "Could not link '$RefReportName$Suffix': $!\n"; + $Rc = 1; + } + } + } + } + + # And (re)build the .err files + if ($Task->Status !~ /^(?:queued|running)$/) + { + my ($IsWineTest, $TaskTimedOut); + if ($Task->Started and $Task->Ended) + { + my $Duration = $Task->Ended - $Task->Started; + $TaskTimedOut = $Duration > $Task->Timeout; + $IsWineTest = ($Step->Type eq "patch" or $Step->Type eq "suite"); + } + foreach my $ReportName (@{GetLogFileNames($TaskDir)}) + { + next if ($ReportName !~ /\.report$/); + next if (-f "$TaskDir/$ReportName.err"); + my $ErrMessage = BuildErrFile($TaskDir, $ReportName, $IsWineTest, $TaskTimedOut); + if (defined $ErrMessage) + { + Error "$ErrMessage\n"; + $Rc = 1; + } + } + } + } + + if (!$OptDelete and $Task->Status eq "completed" and $Step->Type eq "suite") + { + # Update the latest reference reports + # WineTest runs that timed out are missing results which would cause false + # positives. So don't use them as reference results. Also note that + # WineTest tasks have a single WineTest run so a task timeout is the same + # as a WineTest timeout. + my $Duration = $Task->Ended - $Task->Started; + if ($Duration < $Task->Timeout) + { + foreach my $ReportName (@{GetLogFileNames($TaskDir)}) + { + next if ($ReportName !~ /\.report$/); + next if (-z "$TaskDir/$ReportName"); + $Rc += DoUpdateLatestReport($Task, $ReportName, "$TaskDir/$ReportName"); + } + } + } + + return $Rc; +} + +sub ProcessLatestReports() +{ + my $Rc = 0; + my $LatestGlob = "$DataDir/latest/*.report"; + + # Perform cleanups and updates + foreach my $LatestReportPath (glob("$LatestGlob $LatestGlob.err")) + { + my $RefReportName = basename($LatestReportPath); + # Keep the regexp in sync with WineTestBot::Task::GetRefReportName() + next if ($RefReportName !~ /^([a-zA-Z0-9_]+_[a-zA-Z0-9_]+\.report)(?:\.err)?$/); + $RefReportName = $1; # untaint + $LatestReportPath = "$DataDir/latest/$RefReportName"; + + if ($OptDelete or $OptRebuild) + { + # Delete the reports that should be deleted / rebuilt + foreach my $Suffix ("", ".err") + { + next if (!-f "$LatestReportPath$Suffix"); + Debug("latest: Deleting $RefReportName$Suffix\n"); + next if (unlink "$LatestReportPath$Suffix"); + Error "Could not delete '$LatestReportPath$Suffix': $!\n"; + $Rc = 1; + } + } + elsif (!-f "$LatestReportPath") + { + Debug("latest: Deleting orphaned $RefReportName.err\n"); + if (!unlink "$LatestReportPath.err") + { + Error "Could not delete orphaned '$LatestReportPath.err': $!\n"; + $Rc = 1; + } + } + elsif (!$OptDelete and !-f "$LatestReportPath.err") + { + # Build the missing .err file + my $ErrMessage = BuildErrFile("$DataDir/latest", $RefReportName, 1, 0); + if (defined $ErrMessage) + { + Error "$ErrMessage\n"; + $Rc = 1; + } + } + } + + return $Rc; +} + +my $Rc = 0; +$Rc = ProcessLatestReports() if (!defined $OptJobId); + +my @AllTasks; +foreach my $Job (@{CreateJobs()->GetItems()}) +{ + foreach my $Step (@{$Job->Steps->GetItems()}) + { + foreach my $Task (@{$Step->Tasks->GetItems()}) + { + push @AllTasks, $Task if ($Task->Status !~ /^(?:queued|running)$/); + } + } +} + +# Process the tasks in completion order so the reference logs +# are updated in the right order. +my $Jobs = CreateJobs(); +foreach my $Task (sort { ($a->Ended || 0) <=> ($b->Ended || 0) } @AllTasks) +{ + my ($JobId, $StepNo, $TaskNo) = @{$Task->GetMasterKey()}; + my $Step = $Jobs->GetItem($JobId)->Steps->GetItem($StepNo); + my $CollectOnly = ((defined $OptJobId and $OptJobId ne $JobId) or + (defined $OptStepNo and $OptStepNo ne $StepNo) or + (defined $OptTaskNo and $OptTaskNo ne $TaskNo)); + $Rc += ProcessTaskLogs($Step, $Task, $CollectOnly); +} + +exit $Rc ? 1 : 0; -- 2.20.1
participants (1)
-
Francois Gouget