Module: tools
Branch: master
Commit: 18965f76c01c7cb6bb2729fea49ca4a24120b704
URL: https://source.winehq.org/git/tools.git/?a=commit;h=18965f76c01c7cb6bb2729f…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Mon Jan 25 14:52:34 2021 +0100
testbot/TestWTBS: Allow checking the patches disposition.
This allows verifying that patches that should not result in a TestBot
job are indeed ignored for the right reason (aka disposition).
For instance:
----- TestWTBS -----
p patch.Disposition Does not impact the Wine build
----- TestWTBS -----
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 115 +++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 106 insertions(+), 9 deletions(-)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 58bf8a7..5686428 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -48,6 +48,7 @@ use WineTestBot::Config; # For $PatchesMailingList
use WineTestBot::Jobs;
use WineTestBot::Log;
use WineTestBot::LogUtils;
+use WineTestBot::Patches;
use WineTestBot::VMs;
@@ -161,9 +162,10 @@ if (defined $Usage)
print "\n";
print "Where:\n";
print " --jobs RANGES Only check the jobs in one of the specified comma-separated\n";
- print " job id ranges. Each range is either a single job id, or of the\n";
- print " form 'FIRST..LAST' where FIRST and LAST are either the empty\n";
- print " string or a job id.\n";
+ print " job id ranges. This also causes patches not related to one of\n";
+ print " these jobs to be ignored. Each range is either a single job id,\n";
+ print " or of the form 'FIRST..LAST' where FIRST and LAST are either the\n";
+ print " empty string or a job id.\n";
print " --help Shows this usage message.\n";
exit 0;
}
@@ -318,9 +320,12 @@ sub LoadTestInfo($)
return undef if (!$HasTestInfo);
# Fill in some useful defaults
- SetDefault($TestInfo, "job", "Status", "completed");
- SetDefault($TestInfo, "tasks", "Status", "completed");
- SetDefault($TestInfo, "tasks", "HasTask", 1);
+ if (!defined $TestInfo->{patch}->{Disposition})
+ {
+ SetDefault($TestInfo, "job", "Status", "completed");
+ SetDefault($TestInfo, "tasks", "Status", "completed");
+ SetDefault($TestInfo, "tasks", "HasTask", 1);
+ }
# Then propagate the defaults
foreach my $Pair (["tasks", ["build", "tests"]],
@@ -666,6 +671,10 @@ sub CheckJob($$)
{
is($Job->Status, $JobInfo->{Status}, "Check Status for job ". $Job->Id);
}
+ elsif (CheckValue($TestInfo->{patch}->{Disposition}))
+ {
+ fail("The '$TestInfo->{patch}->{Disposition}' patch disposition is incompatible with job ". $Job->Id);
+ }
}
sub IsJobInRange($)
@@ -681,6 +690,8 @@ sub IsJobInRange($)
return 0;
}
+my %CheckedJobs;
+
=pod
=item <tasks.HasTask>
@@ -705,12 +716,15 @@ p win.TestUnits cabinet:extract cabinet:fdi
=cut
-sub CheckJobTree($)
+sub CheckJobTree($;$)
{
- my ($Job) = @_;
+ my ($Job, $TestInfo) = @_;
return if (!IsJobInRange($Job));
- my ($TestInfo, $HasTask);
+ return if ($CheckedJobs{$Job->Id});
+ $CheckedJobs{$Job->Id} = 1;
+
+ my $HasTask;
my $TestUnits = { wine => {} };
my $Steps = $Job->Steps;
@@ -783,6 +797,85 @@ sub CheckJobs()
}
+#
+# Verify the Patches and the patches site
+#
+
+=pod
+
+=item <patch.Disposition>
+
+Checks the patch's disposition.
+By default the disposition is supposed to point to the relevant job, in which
+case default values are provided for the job.Status, tasks.Status and
+tasks.HasTask properties. If a specific disposition is specified, then these
+defaults are not provided since no job should be created in that case.
+
+For instance:
+p patch.Disposition Does not impact the Wine build
+
+=cut
+
+sub CheckPatch($$$)
+{
+ my ($Jobs, $Patch, $TestInfo) = @_;
+
+ my $Job;
+ if ($Patch->Disposition =~ /^Submitted job (\d+)$/)
+ {
+ $Job = $Jobs->GetItem($1);
+ }
+ # The job range restriction extends to their associated patches...
+ return if ($Job and !IsJobInRange($Job));
+ # ..and to patches with no job
+ return if (!$Job and @JobRanges);
+
+ my $PatchInfo = $TestInfo->{patch};
+ if (CheckValue($PatchInfo->{Disposition}))
+ {
+ is($Patch->Disposition, $PatchInfo->{Disposition}, "Check Disposition for patch ". $Patch->Id .": ". $Patch->Subject);
+ }
+ elsif ($Patch->Disposition =~ /^Submitted job (.*)$/)
+ {
+ isnt($Job, undef, "Check job $1 for patch ". $Patch->Id .": ". $Patch->Subject);
+ if (!$Job or $Job->Status eq "canceled")
+ {
+ ; # Nothing to check
+ }
+ elsif ($Job->Status =~ /^(?:queued|running)$/)
+ {
+ print "...skipping job ". $Job->Id ." because it is not complete yet (". $Job->Status .")\n";
+ }
+ else
+ {
+ # Provide defaults for the job checks
+ my $Remarks = $Patch->Subject;
+ $Remarks =~ s~^\[\d+/\d+\]\s~~; # In case submitted as a patch series
+ $Remarks =~ s~\s+$~~;
+ SetDefault($TestInfo, "job", "Remarks", $Remarks);
+ CheckJobTree($Job, $TestInfo);
+ }
+ }
+ else
+ {
+ fail("Patch ". $Patch->Id ."(". $Patch->Subject .") should have an associated job but Disposition is: ". $Patch->Disposition);
+ }
+}
+
+sub CheckPatches()
+{
+ my $Jobs = CreateJobs();
+ my $Patches = CreatePatches();
+ foreach my $Patch (sort { $b->Id <=> $a->Id } @{$Patches->GetItems()})
+ {
+ my $PatchFileName = "$DataDir/patches/". $Patch->Id;
+ next if (!-f $PatchFileName);
+ my $TestInfo = LoadTestInfo($PatchFileName);
+ CheckPatch($Jobs, $Patch, $TestInfo) if ($TestInfo);
+ }
+}
+
+
#
# Run the tests
#
@@ -799,6 +892,10 @@ if (!$HasBaseVM->{build})
delete $HasBaseVM->{win64};
}
+print "***** Checking Patches *****\n";
+CheckPatches();
+
+print "\n***** Checking Jobs *****\n";
CheckJobs();
done_testing();
Module: tools
Branch: master
Commit: 5ae56250113e8dbe7a4e0abbbb7519e1ed552a64
URL: https://source.winehq.org/git/tools.git/?a=commit;h=5ae56250113e8dbe7a4e0ab…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Mon Jan 25 14:52:31 2021 +0100
testbot/TestWTBS: Allow checking for specific strings in logs.
Either for their presence or their absence.
This is particularly useful to check that specific build actions were
performed (or skipped) as expected.
For instance:
----- TestWTBS -----
a build.log.Grep ^Applying patch
a build.log.GrepV ^Running make_requests
a build.log.GrepV ^Running make_opengl
----- TestWTBS -----
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 84 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 83 insertions(+), 1 deletion(-)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 01f46bf..58bf8a7 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -3,7 +3,7 @@
#
# Automate checking the Wine TestBot test Suite results.
#
-# Copyright 2020 Francois Gouget
+# Copyright 2020-2021 Francois Gouget
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
@@ -353,6 +353,22 @@ sub LoadTestInfo($)
}
}
+ # Validate and fix the Grep* fields
+ foreach my $GrepType ("Grep", "GrepV")
+ {
+ foreach my $TaskType ("tasks", "tests", "build", "win", "win32", "win64", "wine")
+ {
+ foreach my $LogType ("report", "log", "testbot")
+ {
+ my $Array = $TestInfo->{$TaskType}->{"$LogType.$GrepType"};
+ next if (!defined $Array);
+ next if (ref($Array) eq "ARRAY");
+ fail("$TaskType.$LogType.$GrepType should be an array (a lines)");
+ $TestInfo->{$TaskType}->{"$LogType.$GrepType"} = [ $Array ];
+ }
+ }
+ }
+
return $TestInfo;
}
@@ -488,6 +504,51 @@ sub CheckLogErrors($$$)
}
}
+sub GrepTaskLog($$$$)
+{
+ my ($Task, $LogName, $TaskInfo, $LogType) = @_;
+
+ my @Grep = @{$TaskInfo->{"$LogType.Grep"} || []};
+ my @GrepV = @{$TaskInfo->{"$LogType.GrepV"} || []};
+ return if (!@Grep and !@GrepV);
+
+ if (open(my $LogFile, "<", $Task->GetDir() ."/$LogName"))
+ {
+ my $LineNo;
+ foreach my $Line (<$LogFile>)
+ {
+ $LineNo++;
+ my $i = 0;
+ while ($i < @GrepV)
+ {
+ if ($Line =~ /$GrepV[$i]/)
+ {
+ fail(TaskKeyStr($Task) ."/$LogName:$LineNo should not match $GrepV[$i]");
+ splice @GrepV, $i, 1;
+ next;
+ }
+ $i++;
+ }
+ $i = 0;
+ while ($i < @Grep)
+ {
+ if ($Line =~ /$Grep[$i]/)
+ {
+ splice @Grep, $i, 1;
+ next;
+ }
+ $i++;
+ }
+ last if (!@Grep and !@GrepV);
+ }
+ close($LogFile);
+ }
+ foreach my $RegExp (@Grep)
+ {
+ fail(TaskKeyStr($Task) ."/$LogName should match $RegExp");
+ }
+}
+
=pod
=item <job.Status>
@@ -503,6 +564,26 @@ By default the Status field should be 'completed'.
Checks the value of the task's TestFailures field.
By default the TestFailures field is not checked.
+=item <tasks.(report|log|testbot).Grep>
+=item <tasks.(report|log|testbot).GrepV>
+
+Verifies the presence or absence of specific regular expressions in the
+specified error log or reports of the tasks in the specified category.
+Multiple regular expressions can be specified so this uses the array type of
+line ('a').
+
+For instance:
+a build.log.Grep ^Applying patch
+a build.log.GrepV ^Running make_requests
+a build.log.GrepV ^Running make_opengl
+
+Checks that the TestBot applied the patch and did not find it necessary to
+run make_requests and make_opengl.
+
+Finally, note that while the Grep* directives are inherited by subcategories,
+they are not merged. So if if both win and win32 have Grep directives, win32
+will not inherit anything from the win category.
+
=cut
sub CheckTask($$$$)
@@ -548,6 +629,7 @@ sub CheckTask($$$$)
CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"},
TaskKeyStr($Task) ."/$LogName");
}
+ GrepTaskLog($Task, $LogName, $TaskInfo, $LogType);
}
if ($Task->Status eq "completed" and CheckValue($TaskInfo->{TestFailures}))
{
Module: tools
Branch: master
Commit: 2f043aaf3b5aff2478343ddc2b7cf4391182e72a
URL: https://source.winehq.org/git/tools.git/?a=commit;h=2f043aaf3b5aff2478343dd…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Mon Jan 25 14:52:28 2021 +0100
testbot/LogUtils: Add support for arrays in .errors files.
Array lines allow providing multiple values for a given 'property' name.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/lib/WineTestBot/LogUtils.pm | 30 +++++++++++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
diff --git a/testbot/lib/WineTestBot/LogUtils.pm b/testbot/lib/WineTestBot/LogUtils.pm
index 3488c7b..7971435 100644
--- a/testbot/lib/WineTestBot/LogUtils.pm
+++ b/testbot/lib/WineTestBot/LogUtils.pm
@@ -848,6 +848,10 @@ Property lines contain (name, value) pairs.
Note that properties which can be calculated while reading the errors file
are not saved (e.g. ErrCount and NewCount).
+=item a <name> <value>
+Array lines contain (name, value) pairs and can appear multiple times for a
+given name, resulting in a corresponding list of values.
+
=item g <lineno> <groupname>
Group lines contain the group name and the line number of the first line of
the group in the log. Note that the first line would typically not be an
@@ -897,6 +901,22 @@ sub LoadLogErrorsFromFh($$)
return $Line;
}
}
+ elsif ($Type eq "a")
+ {
+ if (!defined $LogInfo->{$Property})
+ {
+ $LogInfo->{$Property} = [ $Value ];
+ }
+ elsif (ref($LogInfo->{$Property}) eq "ARRAY")
+ {
+ push @{$LogInfo->{$Property}}, $Value;
+ }
+ else
+ {
+ $LogInfo->{BadLog} = "$LogInfo->{LineNo}: $Property is not an array, its value is $LogInfo->{$Property}";
+ return $Line;
+ }
+ }
elsif ($Type eq "g")
{
$LogInfo->{CurGroup} = _AddLogGroup($LogInfo, $Value, $Property);
@@ -1025,7 +1045,15 @@ sub _DumpErrors
next;
}
- print STDERR "+ $Key $LogInfo->{$Key}\n";
+ if (ref($LogInfo->{$Key}) eq "ARRAY")
+ {
+ print STDERR "+ $Key\n";
+ map { print STDERR " | $_\n" } (@{$LogInfo->{$Key}});
+ }
+ else
+ {
+ print STDERR "+ $Key $LogInfo->{$Key}\n";
+ }
}
_WriteLogErrorsToFh(*STDERR, $LogInfo);
map { _DumpErrors("$Label.$_", $LogInfo->{$_}) } (@SubKeys);
Module: tools
Branch: master
Commit: b47c3dd0180168c53c267972a32821c16cca4a30
URL: https://source.winehq.org/git/tools.git/?a=commit;h=b47c3dd0180168c53c26797…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Fri Jan 22 17:02:47 2021 +0100
testbot/TestWTBS: Fix the TestUnits check in case of build errors.
In case of a build error the TestUnits check must be skipped on Wine VMs
since the test units list is built from the test reports.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index cc3037f..01f46bf 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -514,6 +514,12 @@ sub CheckTask($$$$)
{
is($Task->Status, $TaskInfo->{Status}, "Check Status of task ". TaskKeyStr($Task));
}
+ if ($Task->Status =~ /^bad/)
+ {
+ # It makes no sense to check which test units were run in case of a build
+ # error.
+ $TestUnits->{$TaskType}->{"*skipped*"} = 1;
+ }
my $ReportCount = 0;
foreach my $LogName (@{GetLogFileNames($Task->GetDir())})
@@ -670,6 +676,8 @@ sub CheckJobTree($)
$HasTask->{$Type} ||= 0;
is($HasTask->{$Type}, $TypeInfo->{HasTask}, "Check the presence of $Type tasks for job ". $Job->Id);
}
+
+ next if ($TestUnits->{$Type}->{"*skipped*"});
if (CheckValue($TypeInfo->{TestUnits}))
{
foreach my $TestUnit (split / +/, $TypeInfo->{TestUnits})
Module: tools
Branch: master
Commit: 9b028fed31426847eb847bb0e3e22873bfdeca49
URL: https://source.winehq.org/git/tools.git/?a=commit;h=9b028fed31426847eb847bb…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Fri Jan 22 17:02:38 2021 +0100
testbot/TestWTBS: Allow matching errors in testbot.log.
Tasks have three types of logs:
- *.report files that collect the test output.
- task.log which collects the messages from the VM-side process that
starts the Wine test / build.
- testbot.log which collects the server-side messages when some
inconsistency is detected or the connection to the VM is lost.
Each get their own .errors file but so far only errors in the first two
types of logs could be matched.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 5cadc5d..e3909a2 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -300,7 +300,7 @@ sub LoadTestInfo($)
foreach my $RawGroupName (@{$RawInfo->{ErrGroupNames}})
{
my $GroupName = lcfirst($RawGroupName);
- if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine)\.(log|report)\.//)
+ if ($GroupName =~ s/^(tasks|build|tests|win|win32|win64|wine)\.(report|log|testbot)\.//)
{
my $ErrInfo = ($TestInfo->{$1}->{"$2.errors"} ||= {});
push @{$ErrInfo->{ErrGroupNames}}, $GroupName;
@@ -395,12 +395,13 @@ sub IsMailingListJob($)
}
=pod
-=item <tasks.(log|report).groupname>
+=item <tasks.(report|log|testbot).groupname>
Verifies the presence of new errors in the specified error log or report
of the tasks in the specified category. For this, list the group and errors
that are expected to appear; where the group name is prefixed by the task
-category and either 'log' or 'report'.
+category and either 'report' for the test reports, 'log' for 'task.log', or
+'testbot' for 'testbot.log'.
For instance:
g 0 tests.report.kernel32
@@ -531,6 +532,11 @@ sub CheckTask($$$$)
map { $TestUnits->{wine}->{$_} = 1 } (keys %$ReportTestUnits);
}
}
+ elsif ($LogName =~ /^testbot\./)
+ {
+ $LogType = "testbot";
+ }
+
if ($TaskInfo->{"$LogType.errors"})
{
CheckLogErrors($LogInfo, $TaskInfo->{"$LogType.errors"},
Module: tools
Branch: master
Commit: 0d988959e645b52d4c22213132e7343e88171d24
URL: https://source.winehq.org/git/tools.git/?a=commit;h=0d988959e645b52d4c22213…
Author: Francois Gouget <fgouget(a)codeweavers.com>
Date: Fri Jan 22 17:02:27 2021 +0100
testbot/TestWTBS: Skip the base Windows VMs if there is no base build VM.
Do not report errors about missing Windows tasks if the TestBot only has
Wine VMs.
Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com>
Signed-off-by: Alexandre Julliard <julliard(a)winehq.org>
---
testbot/tests/TestWTBS | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/testbot/tests/TestWTBS b/testbot/tests/TestWTBS
index 5f62d63..c5ac9e1 100755
--- a/testbot/tests/TestWTBS
+++ b/testbot/tests/TestWTBS
@@ -696,6 +696,12 @@ foreach my $VM (@{$VMs->GetItems()})
{
$HasBaseVM->{$VM->Type} = 1 if ($VM->Role eq "base");
}
+if (!$HasBaseVM->{build})
+{
+ # Cannot run the tests on Windows if there is no build VM
+ delete $HasBaseVM->{win32};
+ delete $HasBaseVM->{win64};
+}
CheckJobs();