The Wine TestBot test Suite is a series of Wine pactches that can be used to verify that the TestBot handles test failures, timeouts, etc as expected. The TestWTBS script looks for directives in the commit message of these patches to automate a range of checks on the corresponding jobs and tasks. To simplify parsing the directives, reuse the TestBot's simple errors cache format.
For instance: ----- TestWTBS ----- p job.Status completed p tasks.Status completed p tests.TestFailures 1 ----- TestWTBS -----
Signed-off-by: Francois Gouget fgouget@codeweavers.com ---
The TestWTBS script does not include patches to send to the TestBot for testing. There is also no 'official' set of patches but you can find the ones I use there:
https://github.com/fgouget/wine/tree/wtbsuite
This test suite covers many aspects such as checking that the TestBot rebuilds Wine correctly no matter which file is patched, runs the right set of tests, handles timeouts correctly, keeps track of concurrent patch series, etc. As a result this test suite is quite large, resulting in about 127 TestBot jobs where, for each one, one should check each task's status, list of errors, reported failure count, etc. That's tedious, time counsuming, error prone, and thus lead to the need for automation.
testbot/tests/TestWTBS | 459 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100755 testbot/tests/TestWTBS
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS new file mode 100755 index 00000000..786d3692 --- /dev/null +++ b/testbot/tests/TestWTBS @@ -0,0 +1,459 @@ +#!/usr/bin/perl +# -*- Mode: Perl; perl-indent-level: 2; indent-tabs-mode: nil -*- +# +# Automate checking the Wine TestBot test Suite results. +# +# Copyright 2020 Francois Gouget +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +use strict; +use warnings; + +sub BEGIN +{ + if ($0 !~ m=^/=) + { + # Turn $0 into an absolute path so it can safely be used in @INC + require Cwd; + $0 = Cwd::cwd() . "/$0"; + } + if ($0 =~ m=^(/.*)/[^/]+/[^/]+$=) + { + $::RootDir = $1; + unshift @INC, "$::RootDir/lib"; + } +} + +my $name0 = $0; +$name0 =~ s+^.*/++; + +use Algorithm::Diff; +use File::Basename; +use Test::More; + +use WineTestBot::Config; # For $PatchesMailingList +use WineTestBot::Jobs; +use WineTestBot::Log; +use WineTestBot::LogUtils; +use WineTestBot::VMs; + + +sub error(@) +{ + print STDERR "$name0:error: ", @_; +} + + +# +# Process the command line +# + +=pod + +This script automates checking the Wine TestBot test Suite results. + +The Wine TestBot test Suite is a series of Wine patches that can be used to +verify that the TestBot handles test failures, timeouts, and other conditions +as expected. + +This script looks for directives in the commit message of these patches to +automate a range of checks on the corresponding jobs and tasks. + +To simplify parsing the directives this script reuses the TestBot's errors +cache format (see LogUtils::LoadLogErrorsFromFh()). For each property the +special value 'ignore' causes that check to be skipped rather than using the +defaults. + +The directives must be enclosed in "----- TestWTBS -----" lines. For instance: +----- TestWTBS ----- +p job.Status completed +p tasks.Status completed +p tests.TestFailures 1 +----- TestWTBS ----- + +=cut + +my $Usage; +while (@ARGV) +{ + my $Arg = shift @ARGV; + if ($Arg eq "--help") + { + $Usage = 0; + } + else + { + error("unexpected argument '$Arg'\n"); + $Usage = 2; + } +} + +if (defined $Usage) +{ + if ($Usage) + { + error("try '$name0 --help' for more information\n"); + exit $Usage; + } + print "Usage: $name0 [--help]\n"; + print "\n"; + print "Tests the Patches subject parser.\n"; + print "\n"; + print "Where:\n"; + print " --help Shows this usage message.\n"; + exit 0; +} + + +# +# Test data retrieval +# + +=pod +=item <Categories> + +Property names are of the form 'category.name' where the category is one of: +- patch, job or tasks for checks to perform on the Patch, Job or Task objects + respectively. +- build for checks to perform on the build task. +- win32, win64 for checks to perform on the Windows 32- or 64-bit test results + respectively. +- win for the checks to perform on both the 32- and 64-bit test results; that + is equivalent to duplicating the directive for the win32 and win64 + categories. +- wine for the checks to perform on the Wine test results. +- tests for checks to perform on all test results, that is equivalent to + duplicating the directive for the win32, win64 and wine categories (but not + build since it does not run the tests). + +A corollary is that any property which is documented as being valid for a +category can also be specified for any of its subcategories. So for instance if +tasks.Foo is valid, then tests.Foo and win32.Foo are also valid. + +=cut + +sub DumpTestInfo($) +{ + my ($TestInfo) = @_; + + foreach my $Category ("patch", "job", "tasks", "tests", "win", + "build", "win32", "win64", "wine") + { + WineTestBot::LogUtils::_WriteLogErrorsToFh(*STDERR, $TestInfo->{$Category}); + WineTestBot::LogUtils::_DumpErrors($Category, $TestInfo->{$Category}); + } +} + +sub SetDefault($$$$) +{ + my ($TestInfo, $Category, $Field, $Default) = @_; + if (!defined $TestInfo->{$Category}->{$Field}) + { + $TestInfo->{$Category}->{$Field} = $Default; + } +} + +sub CheckValue($) +{ + my ($Value) = @_; + return (defined $Value and $Value ne "ignore"); +} + +sub LoadTestInfo($) +{ + my ($FileName) = @_; + + my $RawInfo; + if (open(my $TestFh, "<", $FileName)) + { + parser: while (my $Line = <$TestFh>) + { + $RawInfo->{LineNo}++; + chomp $Line; + + if ($Line eq "----- TestWTBS -----") + { + # Patch series may have multiple TestWTBS sections, one per part. + # Only the last one counts. + $RawInfo = { + LogName => basename($FileName), + LogPath => $FileName, + LineNo => $RawInfo->{LineNo}, + }; + while (1) + { + $Line = LoadLogErrorsFromFh($RawInfo, $TestFh); + if (!defined $Line) + { + $RawInfo->{BadLog} = "Reached an unexpected end-of-file at line $RawInfo->{LineNo}"; + last; + } + if ($Line eq "----- TestWTBS -----") + { + delete $RawInfo->{BadLog}; + last; + } + last parser if ($Line !~ /^\s*(?:;|$)/); + delete $RawInfo->{BadLog}; + } + } + } + close($TestFh); + + if ($RawInfo->{BadLog}) + { + fail("$FileName is not valid: $RawInfo->{BadLog}"); + return undef; + } + } + else + { + fail("Could not open $FileName for reading: $!"); + return undef; + } + + # Split up the information to jobs, tasks, etc. + my $TestInfo = { + patch => {}, job => {}, + tasks => {}, tests => {}, win => {}, + build => {}, win32 => {}, win64 => {}, wine => {}, + }; + my $HasTestInfo; + foreach my $Entry (keys %{$RawInfo}) + { + my $Field = lcfirst($Entry); + if ($Field =~ s/^(patch|job|tasks|tests|win|build|win32|win64|wine).//) + { + my $TaskType = $1; + $TestInfo->{$TaskType}->{$Field} = $RawInfo->{$Entry}; + $HasTestInfo = 1; + } + elsif ($Entry !~ /^(?:CurGroup|ErrCount|ErrGroups|ErrGroupNames|LineNo|LogName|LogPath|NewCount)$/) + { + fail("$FileName: $Entry is not a valid property name"); + } + } + return undef if (!$HasTestInfo); + + # Fill in some useful defaults + SetDefault($TestInfo, "job", "Status", "completed"); + SetDefault($TestInfo, "tasks", "Status", "completed"); + SetDefault($TestInfo, "tasks", "HasTask", 1); + + # Then propagate the defaults + foreach my $Pair (["win", ["win32", "win64"]], + ["tests", ["win32", "win64", "wine"]], + ["tasks", ["build", "win32", "win64", "wine"]]) + { + my ($Src, $TaskTypes) = @$Pair; + foreach my $Field (keys %{$TestInfo->{$Src}}) + { + foreach my $TaskType (@$TaskTypes) + { + my $TaskInfo = $TestInfo->{$TaskType}; + if (!defined $TaskInfo->{$Field}) + { + $TaskInfo->{$Field} = $TestInfo->{$Src}->{$Field}; + } + } + } + } + + return $TestInfo; +} + + +# +# Verify the Jobs and Tasks +# + +my $HasBaseVM; + +sub TaskKeyStr($) +{ + my ($Task) = @_; + return join("/", @{$Task->GetMasterKey()}); +} + +sub IsMailingListJob($) +{ + my ($Job) = @_; + return $Job->Remarks =~ /^[\Q$PatchesMailingList\E] /; +} + +=pod + +=item <job.Status> +=item <tasks.Status> + +Checks the value of the job or tasks' Status field. Note that jobs that are +still queued or running or have been canceled are automatically skipped to +avoid false positives. +By default the Status field should be 'completed'. + +=item <tasks.TestFailures> + +Checks the value of the task's TestFailures field. +By default the TestFailures field is not checked. + +=cut + +sub CheckTask($$$) +{ + my ($Task, $TaskType, $TestInfo) = @_; + + my $TaskInfo = $TestInfo->{$TaskType}; + if (CheckValue($TaskInfo->{Status})) + { + is($Task->Status, $TaskInfo->{Status}, "Check Status of task ". TaskKeyStr($Task)); + } + + my $ReportCount = 0; + foreach my $LogName (@{GetLogFileNames($Task->GetDir())}) + { + my $LogPath = $Task->GetDir() ."/$LogName"; + my $LogInfo = LoadLogErrors($LogPath); + + if ($LogName =~ /.report$/) + { + $ReportCount++; + } + } + if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures})) + { + # Scale the expected TestFailures count with the number of times the test + # was run, i.e. $ReportCount, or take it as is if no report is available. + is($Task->TestFailures, $TaskInfo->{TestFailures} * ($ReportCount || 1), "Check Failures of task ". TaskKeyStr($Task)); + } +} + +=pod + +=item <job.Remarks> + +Checks that the job's Remarks field matches the specified value. By default +it should match the patch's subject, modulo the mailing list name. Specifying +job.Remarks is only necessary to deal with some patch series. + +=cut + +sub CheckJob($$) +{ + my ($Job, $TestInfo) = @_; + + print $Job->Id ." Checking ". $Job->Remarks ."\n"; + my $JobInfo = $TestInfo->{job}; + if (CheckValue($JobInfo->{Remarks})) + { + my $Remarks = $Job->Remarks; + # Remove the mailing-list prefix if present + $Remarks =~ s/^[\Q$PatchesMailingList\E] //; + is($Remarks, $JobInfo->{Remarks}, "Check Remarks for job ". $Job->Id); + } + if (CheckValue($JobInfo->{Status})) + { + is($Job->Status, $JobInfo->{Status}, "Check Status for job ". $Job->Id); + } +} + +=pod + +=item <tasks.HasTask> + +Set to 1 if the job must have a task matching the specified category if the +TestBot has the relevant type of VM. Set to 0 if it must not have such a task. +By default all types of task are expected to be present. + +For instance if the patch does not impact the Windows test results add the +lines below to make sure the TestBot did not try to run the tests on Windows. + +p build.HasTask 0 +p win.HasTask 0 + +=cut + +sub CheckJobTree($) +{ + my ($Job) = @_; + + my ($TestInfo, $HasTask); + + my $Steps = $Job->Steps; + foreach my $Step (sort { $a->No <=> $b->No } @{$Job->Steps->GetItems()}) + { + if (!$TestInfo) + { + # WTBS jobs all have a patch + return if ($Step->FileType ne "patch"); + + $TestInfo = LoadTestInfo($Step->GetFullFileName()); + return if (!$TestInfo); + } + + my $TaskType = $Step->Type eq "build" ? "build" : + $Step->FileType eq "patch" ? "wine" : + $Step->FileType eq "exe32" ? "win32" : "win64"; + + foreach my $Task (sort { $a->No <=> $b->No } @{$Step->Tasks->GetItems()}) + { + $HasTask->{$TaskType} = 1; + CheckTask($Task, $TaskType, $TestInfo); + } + } + CheckJob($Job, $TestInfo); + + # Ignore manually submitted jobs because they allow the developer to pick + # which VMs to run the test on. + if (IsMailingListJob($Job)) + { + foreach my $Type ("win32", "win64", "wine") + { + next if (!$HasBaseVM->{$Type}); + + my $TypeInfo = $TestInfo->{$Type}; + if (CheckValue($TypeInfo->{HasTask})) + { + $HasTask->{$Type} ||= 0; + is($HasTask->{$Type}, $TypeInfo->{HasTask}, "Check the presence of $Type tasks for job ". $Job->Id); + } + } + } +} + +sub CheckJobs() +{ + my $Jobs = CreateJobs(); + $Jobs->AddFilter("Status", ['completed', 'badpatch', 'badbuild', 'boterror']); + foreach my $Job (sort { $b->Id <=> $a->Id } @{$Jobs->GetItems()}) + { + CheckJobTree($Job); + } +} + + +# +# Run the tests +# + +my $VMs = CreateVMs(); +foreach my $VM (@{$VMs->GetItems()}) +{ + $HasBaseVM->{$VM->Type} = 1 if ($VM->Role eq "base"); +} + +CheckJobs(); + +done_testing();