[PATCH 1/4] testbot: Build the per-module test unit list from the list of Wine files.
This also caches winefiles.txt so it is only reloaded when it changes. Also stop generating and retrieving testlist.txt since it is not needed anymore. If desired testlist.txt can be deleted for cleanup. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/bin/WineRunReconfig.pl | 10 ----- testbot/lib/WineTestBot/PatchUtils.pm | 62 +++++++++++++-------------- 2 files changed, 31 insertions(+), 41 deletions(-) diff --git a/testbot/bin/WineRunReconfig.pl b/testbot/bin/WineRunReconfig.pl index 2950799d5..2e338c975 100755 --- a/testbot/bin/WineRunReconfig.pl +++ b/testbot/bin/WineRunReconfig.pl @@ -414,16 +414,6 @@ if ($NewStatus eq "completed") } } - Debug(Elapsed($Start), " Retrieving the updated test list '$TaskDir/testlist.txt'\n"); - if ($TA->GetFile("testlist.txt", "$TaskDir/testlist.txt")) - { - copy "$TaskDir/testlist.txt", "$DataDir/latest/testlist.txt"; - } - elsif (!defined $TAError) - { - $TAError = "An error occurred while retrieving the test list: ". $TA->GetLastError(); - } - Debug(Elapsed($Start), " Retrieving the list of Wine files '$TaskDir/winefiles.txt'\n"); if ($TA->GetFile("latest/winefiles.txt", "$TaskDir/winefiles.txt")) { diff --git a/testbot/lib/WineTestBot/PatchUtils.pm b/testbot/lib/WineTestBot/PatchUtils.pm index f83df6087..3bf172125 100644 --- a/testbot/lib/WineTestBot/PatchUtils.pm +++ b/testbot/lib/WineTestBot/PatchUtils.pm @@ -58,8 +58,8 @@ my $IgnoredPathsRe = join('|', =item C<UpdateWineData()> -Updates the summary information about the Wine source such as the list of -tests (testlist.txt) and the full list of Wine files. +Updates information about the Wine source, such as the list of Wine files, +for use by the TestBot server. =back =cut @@ -72,51 +72,51 @@ sub UpdateWineData($) my $ErrMessage = `cd '$WineDir' && git ls-tree -r --name-only HEAD 2>&1 >'$DataDir/latest/winefiles.txt'`; return $ErrMessage if ($? != 0); - - if (open(my $fh, ">", "$DataDir/testlist.txt")) - { - foreach my $TestFile (glob("$WineDir/*/*/tests/*.c"), - glob("$WineDir/*/*/tests/*.spec")) - { - next if ($TestFile =~ m=/testlist\.c$=); - $TestFile =~ s=^$WineDir/==; - print $fh "$TestFile\n"; - } - close($fh); - return undef; - } - - return "Could not open 'testlist.txt' for writing: $!"; } +my $_TimeStamp; +my $_TestList; + =pod =over 12 -=item C<GetTestList()> +=item C<_LoadWineFiles()> -Returns a hashtable containing the list of the source files for a given module. -This structure is built from the latest/testlist.txt file. +Reads latest/winefiles.txt to build a per-module hashtable of the test unit +files. =back =cut -sub GetTestList() +sub _LoadWineFiles() { - my $TestList = {}; - if (open(my $File, "<", "$DataDir/latest/testlist.txt")) + my $FileName = "$DataDir/latest/winefiles.txt"; + my $MTime = (stat($FileName))[9] || 0; + + if ($_TestList and $_TimeStamp == $MTime) { - while (my $TestFileName = <$File>) + # The file has not changed since we loaded it + return; + } + + $_TimeStamp = $MTime; + $_TestList = {}; + if (open(my $fh, "<", $FileName)) + { + while (my $Line = <$fh>) { - chomp $TestFileName; - if ($TestFileName =~ m~^\w+/([^/]+)/tests/([^/]+)$~) + chomp $Line; + + if ($Line =~ m~^\w+/([^/]+)/tests/([^/]+)$~) { my ($Module, $File) = ($1, $2); - $TestList->{$Module}->{$File} = 1; + next if ($File eq "testlist.c"); + next if ($File !~ /\.(?:c|spec)$/); + $_TestList->{$Module}->{$File} = 1; } } - close($File); + close($fh); } - return $TestList; } sub _HandleFile($$$) @@ -154,8 +154,7 @@ sub _HandleFile($$$) if (!$Tests->{$Module}->{Files}) { - my $TestList = ( $Impacts->{TestList} ||= GetTestList() ); - foreach my $File (keys %{$TestList->{$Module}}) + foreach my $File (keys %{$_TestList->{$Module}}) { $Tests->{$Module}->{Files}->{$File} = 0; # not modified } @@ -192,6 +191,7 @@ sub GetPatchImpact($;$$) NoUnits => $NoUnits, Tests => {}, }; + _LoadWineFiles(); if ($PastImpacts) { -- 2.17.1
Check that the VM is of the expected type. Also the VM should always be powered on (for powered off snapshots LibvirtTool.pl is responsible for powering on the VM), even when debugging. Move the Step validation next to the VM one. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/bin/WineRunBuild.pl | 15 ++++++++++++--- testbot/bin/WineRunReconfig.pl | 15 ++++++++++++--- testbot/bin/WineRunTask.pl | 24 ++++++++++++++---------- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/testbot/bin/WineRunBuild.pl b/testbot/bin/WineRunBuild.pl index 60af32c73..ff04ded0c 100755 --- a/testbot/bin/WineRunBuild.pl +++ b/testbot/bin/WineRunBuild.pl @@ -309,18 +309,27 @@ sub FatalTAError($$) # -# Check the VM +# Check the VM and Step # -if (!$Debug and $VM->Status ne "running") +if ($VM->Type ne "build") +{ + FatalError("This is not a build VM! (" . $VM->Type . ")\n"); +} +elsif (!$Debug and $VM->Status ne "running") { FatalError("The VM is not ready for use (" . $VM->Status . ")\n"); } -elsif ($Debug and !$VM->GetDomain()->IsPoweredOn()) +elsif (!$VM->GetDomain()->IsPoweredOn()) { FatalError("The VM is not powered on\n"); } +if ($Step->FileType ne "patchdlls") +{ + FatalError("Unexpected file type '". $Step->FileType ."' found\n"); +} + # # Figure out what to build diff --git a/testbot/bin/WineRunReconfig.pl b/testbot/bin/WineRunReconfig.pl index 2e338c975..23b038a1f 100755 --- a/testbot/bin/WineRunReconfig.pl +++ b/testbot/bin/WineRunReconfig.pl @@ -309,18 +309,27 @@ sub FatalTAError($$) # -# Check the VM +# Check the VM and Step # -if (!$Debug and $VM->Status ne "running") +if ($VM->Type ne "build") +{ + FatalError("This is not a build VM! (" . $VM->Type . ")\n"); +} +elsif (!$Debug and $VM->Status ne "running") { FatalError("The VM is not ready for use (" . $VM->Status . ")\n"); } -elsif ($Debug and !$VM->GetDomain()->IsPoweredOn()) +elsif (!$VM->GetDomain()->IsPoweredOn()) { FatalError("The VM is not powered on\n"); } +if ($Step->FileType ne "none") +{ + FatalError("Unexpected file type '". $Step->FileType ."' found\n"); +} + # # Run the build diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl index 3f5a3ee8d..7604e73ad 100755 --- a/testbot/bin/WineRunTask.pl +++ b/testbot/bin/WineRunTask.pl @@ -364,18 +364,27 @@ sub FatalTAError($$;$) # -# Check the VM +# Check the VM and Step # -if (!$Debug and $VM->Status ne "running") +if ($VM->Type ne "win32" and $VM->Type ne "win64") +{ + FatalError("This is not a Windows VM! (" . $VM->Type . ")\n"); +} +elsif (!$Debug and $VM->Status ne "running") { FatalError("The VM is not ready for use (" . $VM->Status . ")\n"); } -elsif ($Debug and !$VM->GetDomain()->IsPoweredOn()) +elsif (!$VM->GetDomain()->IsPoweredOn()) { FatalError("The VM is not powered on\n"); } +if ($Step->FileType ne "exe32" and $Step->FileType ne "exe64") +{ + FatalError("Unexpected file type '". $Step->FileType ."' found\n"); +} + # # Setup the VM @@ -395,11 +404,6 @@ if (!$TA->SetTime()) } } -my $FileType = $Step->FileType; -if ($FileType ne "exe32" && $FileType ne "exe64") -{ - FatalError("Unexpected file type $FileType found\n"); -} my $FileName = $Step->FileName; Debug(Elapsed($Start), " Sending '". $Step->GetFullFileName() ."'\n"); if (!$TA->SendFile($Step->GetFullFileName(), $FileName, 0)) @@ -418,7 +422,7 @@ if ($Step->ReportSuccessfulTests) my $IsWineTest = 1; if ($Step->Type eq "single") { - my $TestLauncher = "TestLauncher" . ($FileType eq "exe64" ? "64" : "32") . ".exe"; + my $TestLauncher = "TestLauncher" . ($Step->FileType eq "exe64" ? "64" : "32") . ".exe"; Debug(Elapsed($Start), " Sending 'latest/$TestLauncher'\n"); if (!$TA->SendFile("$DataDir/latest/$TestLauncher", $TestLauncher, 0)) { @@ -450,7 +454,7 @@ elsif ($Step->Type eq "suite") $Tag =~ s/[^a-zA-Z0-9]/-/g; if ($VM->Type eq "win64") { - $Tag .= "-" . ($FileType eq "exe64" ? "64" : "32"); + $Tag .= "-" . ($Step->FileType eq "exe64" ? "64" : "32"); } if (defined($WebHostName)) { -- 2.17.1
This adds the base infrastructure for later testing the patches on Wine. This adds a "wine" VM type which applies to all VMs that can compile and run Wine, regardless of the underlying operating system. Adapting to the underlying operating system is thus the responsibility of the VM-side scripts. This also updates GetPatchImpacts() to detect non-Wine scripts (such as TestBot patches) so the TestBot does not report them as bad patches for failing to apply on top of Wine. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- Note: This requires updating the database with update37.sql, and restarting the TestBot engine and web server. testbot/bin/CheckForWinetestUpdate.pl | 47 ++- testbot/bin/WineRunReconfig.pl | 36 +- testbot/bin/WineRunWineTest.pl | 423 ++++++++++++++++++++ testbot/bin/build/WineReconfig.pl | 258 ++++++++++++ testbot/bin/build/WineTest.pl | 282 +++++++++++++ testbot/ddl/update37.sql | 4 + testbot/ddl/winetestbot.sql | 2 +- testbot/lib/WineTestBot/Config.pm | 9 +- testbot/lib/WineTestBot/Engine/Scheduler.pm | 3 +- testbot/lib/WineTestBot/PatchUtils.pm | 80 +++- testbot/lib/WineTestBot/Patches.pm | 213 +++++----- testbot/lib/WineTestBot/PendingPatchSets.pm | 14 +- testbot/lib/WineTestBot/StepsTasks.pm | 2 +- testbot/lib/WineTestBot/Tasks.pm | 5 +- testbot/lib/WineTestBot/VMs.pm | 8 +- 15 files changed, 1234 insertions(+), 152 deletions(-) create mode 100755 testbot/bin/WineRunWineTest.pl create mode 100755 testbot/bin/build/WineReconfig.pl create mode 100755 testbot/bin/build/WineTest.pl create mode 100644 testbot/ddl/update37.sql diff --git a/testbot/bin/CheckForWinetestUpdate.pl b/testbot/bin/CheckForWinetestUpdate.pl index 80c57b9e1..74c59955f 100755 --- a/testbot/bin/CheckForWinetestUpdate.pl +++ b/testbot/bin/CheckForWinetestUpdate.pl @@ -63,7 +63,8 @@ my %WineTestUrls = ( 64 => "http://test.winehq.org/builds/winetest64-latest.exe" ); -my %TaskTypes = (build => 1, base32 => 1, winetest32 => 1, all64 => 1); +my %TaskTypes = (build => 1, base32 => 1, winetest32 => 1, all64 => 1, + wine => 1); my $Debug; @@ -212,7 +213,7 @@ sub AddJob($$$) my $Tasks = $NewStep->Tasks; foreach my $VMKey (@{$VMs->SortKeysBySortOrder($VMs->GetKeys())}) { - Debug(" $VMKey\n"); + Debug(" $VMKey exe$Bits\n"); my $Task = $Tasks->Add(); $Task->VM($VMs->GetItem($VMKey)); $Task->Timeout($SuiteTimeout); @@ -246,11 +247,23 @@ sub AddJob($$$) return 1; } -sub AddReconfigJob() +sub AddReconfigJob($) { - my $Remarks = "Update Wine to latest git"; + my ($VMType) = @_; + + my $Remarks = "Update the $VMType VMs"; Debug("Creating the '$Remarks' job\n"); + my $VMs = CreateVMs(); + $VMs->AddFilter("Type", [$VMType]); + $VMs->FilterEnabledRole(); + if ($VMs->GetItemsCount() == 0) + { + # There is nothing to do + Debug(" Found no VM\n"); + return 1; + } + # First create a new job my $Jobs = CreateJobs(); my $NewJob = $Jobs->Add(); @@ -265,15 +278,16 @@ sub AddReconfigJob() $NewStep->FileType("none"); $NewStep->InStaging(!1); - # Add a task for the build VM - my $VMs = CreateVMs(); - $VMs->AddFilter("Type", ["build"]); - $VMs->AddFilter("Role", ["base"]); - my $BuildVM = ${$VMs->GetItems()}[0]; - Debug(" ", $BuildVM->GetKey(), "\n"); - my $Task = $NewStep->Tasks->Add(); - $Task->VM($BuildVM); - $Task->Timeout($ReconfigTimeout); + # And a task for each VM + my $SortedKeys = $VMs->SortKeysBySortOrder($VMs->GetKeys()); + foreach my $VMKey (@$SortedKeys) + { + my $VM = $VMs->GetItem($VMKey); + Debug(" $VMKey $VMType reconfig\n"); + my $Task = $NewStep->Tasks->Add(); + $Task->VM($VM); + $Task->Timeout($ReconfigTimeout); + } # Save it all my ($ErrKey, $ErrProperty, $ErrMessage) = $Jobs->Save(); @@ -358,7 +372,8 @@ if (defined $Usage) # my $Rc = 0; -if ($OptTypes{build} or $OptTypes{base32} or $OptTypes{winetest32}) +if ($OptTypes{build} or $OptTypes{base32} or $OptTypes{winetest32} or + $OptTypes{wine}) { my ($Create, $LatestBaseName) = UpdateWineTest($OptCreate, 32); if ($Create < 0) @@ -370,9 +385,11 @@ if ($OptTypes{build} or $OptTypes{base32} or $OptTypes{winetest32}) # A new executable means there have been commits so update Wine. Create # this job first purely to make the WineTestBot job queue look nice, and # arbitrarily do it only for 32-bit executables to avoid redundant updates. - $Rc = 1 if ($OptTypes{build} and !AddReconfigJob()); + $Rc = 1 if ($OptTypes{build} and !AddReconfigJob("build")); $Rc = 1 if ($OptTypes{base32} and !AddJob("base", $LatestBaseName, 32)); $Rc = 1 if ($OptTypes{winetest32} and !AddJob("", $LatestBaseName, 32)); + + $Rc = 1 if ($OptTypes{wine} and !AddReconfigJob("wine")); } } diff --git a/testbot/bin/WineRunReconfig.pl b/testbot/bin/WineRunReconfig.pl index 23b038a1f..e418e5bb9 100755 --- a/testbot/bin/WineRunReconfig.pl +++ b/testbot/bin/WineRunReconfig.pl @@ -5,7 +5,7 @@ # See the bin/build/Reconfig.pl script. # # Copyright 2009 Ge van Geldorp -# Copyright 2013-2016 Francois Gouget +# Copyright 2013-2018 Francois Gouget # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -312,9 +312,9 @@ sub FatalTAError($$) # Check the VM and Step # -if ($VM->Type ne "build") +if ($VM->Type ne "build" and $VM->Type ne "wine") { - FatalError("This is not a build VM! (" . $VM->Type . ")\n"); + FatalError("This is neither a build nor a Wine VM! (" . $VM->Type . ")\n"); } elsif (!$Debug and $VM->Status ne "running") { @@ -337,11 +337,12 @@ if ($Step->FileType ne "none") # Use our own log so it can be used for reference # even after another task has run. -my $Script = "#!/bin/sh\n". - "( set -x\n". - " git pull &&\n". - " ../bin/build/Reconfig.pl\n". - ") >Reconfig.log 2>&1\n"; +my $Script = $VM->Type eq "wine" ? "WineReconfig.pl" : "Reconfig.pl"; +$Script = "#!/bin/sh\n". + "( set -x\n". + " git pull &&\n". + " ../bin/build/$Script\n". + ") >Reconfig.log 2>&1\n"; my $TA = $VM->GetAgent(); Debug(Elapsed($Start), " Sending the script: [$Script]\n"); if (!$TA->SendFileFromString($Script, "task", $TestAgent::SENDFILE_EXE)) @@ -410,16 +411,19 @@ elsif (!defined $TAError) if ($NewStatus eq "completed") { use File::Copy; - for my $Bitness ("32", "64") + if ($VM->Type eq "build") { - Debug(Elapsed($Start), " Retrieving the $Bitness bit TestLauncher to '$TaskDir/TestLauncher$Bitness.exe'\n"); - if ($TA->GetFile("../src/TestLauncher/TestLauncher$Bitness.exe", "$TaskDir/TestLauncher$Bitness.exe")) + for my $Bitness ("32", "64") { - copy "$TaskDir/TestLauncher$Bitness.exe", "$DataDir/latest/TestLauncher$Bitness.exe"; - } - elsif (!defined $TAError) - { - $TAError = "An error occurred while retrieving the $Bitness bit TestLauncher: ". $TA->GetLastError(); + Debug(Elapsed($Start), " Retrieving the $Bitness bit TestLauncher to '$TaskDir/TestLauncher$Bitness.exe'\n"); + if ($TA->GetFile("../src/TestLauncher/TestLauncher$Bitness.exe", "$TaskDir/TestLauncher$Bitness.exe")) + { + copy "$TaskDir/TestLauncher$Bitness.exe", "$DataDir/latest/TestLauncher$Bitness.exe"; + } + elsif (!defined $TAError) + { + $TAError = "An error occurred while retrieving the $Bitness bit TestLauncher: ". $TA->GetLastError(); + } } } diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl new file mode 100755 index 000000000..0aba7ecfc --- /dev/null +++ b/testbot/bin/WineRunWineTest.pl @@ -0,0 +1,423 @@ +#!/usr/bin/perl -Tw +# -*- Mode: Perl; perl-indent-level: 2; indent-tabs-mode: nil -*- +# +# Makes sure the Wine patches compile. +# See the bin/build/WineTest.pl script. +# +# Copyright 2018 Francois Gouget +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +use strict; + +sub BEGIN +{ + if ($0 !~ m=^/=) + { + # Turn $0 into an absolute path so it can safely be used in @INC + require Cwd; + $0 = Cwd::cwd() . "/$0"; + } + if ($0 =~ m=^(/.*)/[^/]+/[^/]+$=) + { + $::RootDir = $1; + unshift @INC, "$::RootDir/lib"; + } +} +my $Name0 = $0; +$Name0 =~ s+^.*/++; + + +use WineTestBot::Config; +use WineTestBot::Jobs; +use WineTestBot::PatchUtils; +use WineTestBot::VMs; +use WineTestBot::Log; +use WineTestBot::LogUtils; +use WineTestBot::Engine::Notify; + + +# +# Logging and error handling helpers +# + +my $Debug; +sub Debug(@) +{ + print STDERR @_ if ($Debug); +} + +my $LogOnly; +sub Error(@) +{ + print STDERR "$Name0:error: ", @_ if (!$LogOnly); + LogMsg @_; +} + + +# +# Setup and command line processing +# + +$ENV{PATH} = "/usr/bin:/bin"; +delete $ENV{ENV}; + +my $Usage; +sub ValidateNumber($$) +{ + my ($Name, $Value) = @_; + + # Validate and untaint the value + return $1 if ($Value =~ /^(\d+)$/); + Error "$Value is not a valid $Name\n"; + $Usage = 2; + return undef; +} + +my ($JobId, $StepNo, $TaskNo); +while (@ARGV) +{ + my $Arg = shift @ARGV; + if ($Arg eq "--debug") + { + $Debug = 1; + } + elsif ($Arg eq "--log-only") + { + $LogOnly = 1; + } + elsif ($Arg =~ /^(?:-\?|-h|--help)$/) + { + $Usage = 0; + last; + } + elsif ($Arg =~ /^-/) + { + Error "unknown option '$Arg'\n"; + $Usage = 2; + last; + } + elsif (!defined $JobId) + { + $JobId = ValidateNumber('job id', $Arg); + } + elsif (!defined $StepNo) + { + $StepNo = ValidateNumber('step number', $Arg); + } + elsif (!defined $TaskNo) + { + $TaskNo = ValidateNumber('task number', $Arg); + } + else + { + Error "unexpected argument '$Arg'\n"; + $Usage = 2; + last; + } +} + +# Check parameters +if (!defined $Usage) +{ + if (!defined $JobId || !defined $StepNo || !defined $TaskNo) + { + Error "you must specify the job id, step number and task number\n"; + $Usage = 2; + } +} +if (defined $Usage) +{ + print "Usage: $Name0 [--debug] [--log-only] [--help] JobId StepNo TaskNo\n"; + exit $Usage; +} + +my $Job = CreateJobs()->GetItem($JobId); +if (!defined $Job) +{ + Error "Job $JobId does not exist\n"; + exit 1; +} +my $Step = $Job->Steps->GetItem($StepNo); +if (!defined $Step) +{ + Error "Step $StepNo of job $JobId does not exist\n"; + exit 1; +} +my $Task = $Step->Tasks->GetItem($TaskNo); +if (!defined $Task) +{ + Error "Step $StepNo task $TaskNo of job $JobId does not exist\n"; + exit 1; +} +my $TaskDir = $Task->CreateDir(); +my $VM = $Task->VM; + + +my $Start = Time(); +LogMsg "Task $JobId/$StepNo/$TaskNo started\n"; + + +# +# Error handling helpers +# + +sub LogTaskError($) +{ + my ($ErrMessage) = @_; + Debug("$Name0:error: ", $ErrMessage); + + if (open(my $ErrFile, ">>", "$TaskDir/err")) + { + print $ErrFile $ErrMessage; + close($ErrFile); + } + else + { + Error "Unable to open 'err' for writing: $!\n"; + } +} + +sub WrapUpAndExit($;$$) +{ + my ($Status, $Retry, $Timeout) = @_; + my $NewVMStatus = $Status eq 'queued' ? 'offline' : 'dirty'; + my $VMResult = $Status eq "boterror" ? "boterror" : + $Status eq "queued" ? "error" : + $Timeout ? "timeout" : ""; + + my $TestFailures; + my $Tries = $Task->TestFailures || 0; + if ($Retry) + { + # This may be a transient error (e.g. a network glitch) + # so retry a few times to improve robustness + $Tries++; + if ($Task->CanRetry()) + { + $Status = 'queued'; + $TestFailures = $Tries; + } + else + { + LogTaskError("Giving up after $Tries run(s)\n"); + } + } + elsif ($Tries >= 1) + { + LogTaskError("The previous $Tries run(s) terminated abnormally\n"); + } + + # Record result details that may be lost or overwritten by a later run + if ($VMResult) + { + $VMResult .= " $Tries $MaxTaskTries" if ($Retry); + $VM->RecordResult(undef, $VMResult); + } + + # Update the Task and Job + $Task->Status($Status); + $Task->TestFailures($TestFailures); + if ($Status eq 'queued') + { + $Task->Started(undef); + $Task->Ended(undef); + # Leave the Task files around so they can be seen until the next run + } + else + { + $Task->Ended(time()); + } + $Task->Save(); + $Job->UpdateStatus(); + + # Get the up-to-date VM status and update it if nobody else changed it + $VM = CreateVMs()->GetItem($VM->GetKey()); + if ($VM->Status eq 'running') + { + $VM->Status($NewVMStatus); + $VM->ChildDeadline(undef); + $VM->ChildPid(undef); + $VM->Save(); + } + + my $Result = $VM->Name .": ". $VM->Status ." Status: $Status Failures: ". (defined $TestFailures ? $TestFailures : "unset"); + LogMsg "Task $JobId/$StepNo/$TaskNo done ($Result)\n"; + Debug(Elapsed($Start), " Done. $Result\n"); + exit($Status eq 'completed' ? 0 : 1); +} + +sub FatalError($;$) +{ + my ($ErrMessage, $Retry) = @_; + + LogMsg "$JobId/$StepNo/$TaskNo $ErrMessage"; + LogTaskError($ErrMessage); + + WrapUpAndExit('boterror', $Retry); +} + +sub FatalTAError($$) +{ + my ($TA, $ErrMessage) = @_; + $ErrMessage .= ": ". $TA->GetLastError() if (defined $TA); + + # A TestAgent operation failed, see if the VM is still accessible + my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn(); + if (!defined $IsPoweredOn) + { + # The VM host is not accessible anymore so mark the VM as offline and + # requeue the task. This does not count towards the task's tries limit + # since neither the VM nor the task are at fault. + Error("$ErrMessage\n"); + WrapUpAndExit('queued'); + } + + my $Retry; + if ($IsPoweredOn) + { + LogMsg("$ErrMessage\n"); + LogTaskError("$ErrMessage\n"); + $ErrMessage = "The test VM has crashed, rebooted or lost connectivity (or the TestAgent server died)\n"; + # Retry in case it was a temporary network glitch + $Retry = 1; + } + else + { + # Ignore the TestAgent error, it's irrelevant + $ErrMessage = "The test VM is powered off!\n"; + } + FatalError($ErrMessage, $Retry); +} + + +# +# Check the VM and Step +# + +if ($VM->Type ne "wine") +{ + FatalError("This is not a Wine VM! (" . $VM->Type . ")\n"); +} +elsif (!$Debug and $VM->Status ne "running") +{ + FatalError("The VM is not ready for use (" . $VM->Status . ")\n"); +} +elsif (!$VM->GetDomain()->IsPoweredOn()) +{ + FatalError("The VM is not powered on\n"); +} + +if ($Step->FileType ne "patchdlls") +{ + FatalError("Unexpected file type '". $Step->FileType ."' found\n"); +} + + +# +# Run the task +# + +my $FileName = $Step->GetFullFileName(); +my $TA = $VM->GetAgent(); +Debug(Elapsed($Start), " Sending '$FileName'\n"); +if (!$TA->SendFile($FileName, "staging/patch.diff", 0)) +{ + FatalTAError($TA, "Could not copy the patch to the VM"); +} +my $Script = "#!/bin/sh\n". + "( set -x\n" . + " ../bin/build/WineTest.pl ". $Task->CmdLineArg ." build patch.diff\n". + ") >Task.log 2>&1\n"; +Debug(Elapsed($Start), " Sending the script: [$Script]\n"); +if (!$TA->SendFileFromString($Script, "task", $TestAgent::SENDFILE_EXE)) +{ + FatalTAError($TA, "Could not send the task script to the VM"); +} + +Debug(Elapsed($Start), " Starting the script\n"); +my $Pid = $TA->Run(["./task"], 0); +if (!$Pid) +{ + FatalTAError($TA, "Failed to start the task"); +} + + +# +# From that point on we want to at least try to grab the task +# log before giving up +# + +my ($NewStatus, $ErrMessage, $TAError, $TaskTimedOut); +Debug(Elapsed($Start), " Waiting for the script (", $Task->Timeout, "s timeout)\n"); +if (!defined $TA->Wait($Pid, $Task->Timeout, 60)) +{ + $ErrMessage = $TA->GetLastError(); + if ($ErrMessage =~ /timed out waiting for the child process/) + { + $ErrMessage = "The task timed out\n"; + $NewStatus = "badbuild"; + $TaskTimedOut = 1; + } + else + { + $TAError = "An error occurred while waiting for the task to complete: $ErrMessage"; + $ErrMessage = undef; + } +} + +Debug(Elapsed($Start), " Retrieving 'Task.log'\n"); +if ($TA->GetFile("Task.log", "$TaskDir/log")) +{ + my $Result = ParseTaskLog("$TaskDir/log", "Task"); + if ($Result eq "ok") + { + # We must have gotten the full log and the build did succeed. + # So forget any prior error. + $NewStatus = "completed"; + $TAError = $ErrMessage = undef; + } + elsif ($Result eq "badpatch") + { + $NewStatus = "badpatch"; + } + elsif ($Result =~ s/^nolog://) + { + FatalError("$Result\n", "retry"); + } + else + { + # If the result line is missing we probably already have an error message + # that explains why. + $NewStatus = "badbuild"; + } +} +elsif (!defined $TAError) +{ + $TAError = "An error occurred while retrieving the task log: ". $TA->GetLastError(); +} + +# Report the task errors even though they may have been caused by +# TestAgent trouble. +LogTaskError($ErrMessage) if (defined $ErrMessage); +FatalTAError(undef, $TAError) if (defined $TAError); +$TA->Disconnect(); + + +# +# Wrap up +# + +WrapUpAndExit($NewStatus, undef, $TaskTimedOut); diff --git a/testbot/bin/build/WineReconfig.pl b/testbot/bin/build/WineReconfig.pl new file mode 100755 index 000000000..b030051c6 --- /dev/null +++ b/testbot/bin/build/WineReconfig.pl @@ -0,0 +1,258 @@ +#!/usr/bin/perl -Tw +# -*- Mode: Perl; perl-indent-level: 2; indent-tabs-mode: nil -*- +# +# Updates the Wine source from Git and rebuilds it. +# +# Copyright 2018 Francois Gouget +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +use strict; + +sub BEGIN +{ + if ($0 !~ m=^/=) + { + # Turn $0 into an absolute path so it can safely be used in @INC + require Cwd; + $0 = Cwd::cwd() . "/$0"; + } + if ($0 =~ m=^(/.*)/[^/]+/[^/]+/[^/]+$=) + { + $::RootDir = $1; + unshift @INC, "$::RootDir/lib"; + } + $::BuildEnv = 1; +} +my $Name0 = $0; +$Name0 =~ s+^.*/++; + + +use WineTestBot::Config; +use WineTestBot::PatchUtils; + + +# +# Logging and error handling helpers +# + +sub InfoMsg(@) +{ + print @_; +} + +sub LogMsg(@) +{ + print "Reconfig: ", @_; +} + +sub Error(@) +{ + print STDERR "$Name0:error: ", @_; +} + + +# +# Build helpers +# + +my $ncpus; +sub CountCPUs() +{ + if (open(my $fh, "<", "/proc/cpuinfo")) + { + # Linux + map { $ncpus++ if (/^processor/); } <$fh>; + close($fh); + } + $ncpus ||= 1; +} + +sub BuildTestAgentd() +{ + # If testagentd already exists it's likely already running + # so don't rebuild it. + if (! -x "$BinDir/build/testagentd") + { + InfoMsg "\nBuilding the native testagentd\n"; + system("cd '$::RootDir/src/testagentd' && set -x && ". + "time make -j$ncpus build"); + if ($? != 0) + { + LogMsg "Build testagentd failed\n"; + return !1; + } + } + + return 1; +} + +sub GitPull($) +{ + my ($Targets) = @_; + return 1 if (!$Targets->{update}); + + InfoMsg "\nUpdating the Wine source\n"; + system("cd '$DataDir/wine' && git pull"); + if ($? != 0) + { + LogMsg "Git pull failed\n"; + return !1; + } + + my $ErrMessage = UpdateWineData("$DataDir/wine"); + if ($ErrMessage) + { + LogMsg "$ErrMessage\n"; + return !1; + } + + return 1; +} + +sub BuildWine($$$$) +{ + my ($Targets, $NoRm, $Build, $Extras) = @_; + + return 1 if (!$Targets->{$Build}); + mkdir "$DataDir/build-$Build" if (!-d "$DataDir/build-$Build"); + + # If $NoRm is not set, rebuild from scratch to make sure cruft will not + # accumulate + InfoMsg "\nRebuilding the $Build Wine\n"; + system("cd '$DataDir/build-$Build' && set -x && ". + ($NoRm ? "" : "rm -rf * && ") . + "time ../wine/configure $Extras --disable-winetest && ". + "time make -j$ncpus"); + if ($? != 0) + { + LogMsg "The $Build build failed\n"; + return !1; + } + + return 1; +} + + +# +# Setup and command line processing +# + +$ENV{PATH} = "/usr/lib/ccache:/usr/bin:/bin"; +delete $ENV{ENV}; + +my %AllTargets; +map { $AllTargets{$_} = 1 } qw(update win32 wow32 wow64); + +my ($Usage, $TargetList, $NoRm); +while (@ARGV) +{ + my $Arg = shift @ARGV; + if ($Arg eq "--no-rm") + { + $NoRm = 1; + } + elsif ($Arg =~ /^(?:-\?|-h|--help)$/) + { + $Usage = 0; + last; + } + elsif ($Arg =~ /^-/) + { + Error "unknown option '$Arg'\n"; + $Usage = 2; + last; + } + elsif (!defined $TargetList) + { + $TargetList = $Arg; + } + else + { + Error "unexpected argument '$Arg'\n"; + $Usage = 2; + last; + } +} + +# Check and untaint parameters +my $Targets; +if (!defined $Usage) +{ + $TargetList = join(",", keys %AllTargets) if (!defined $TargetList); + foreach my $Target (split /,/, $TargetList) + { + if (!$AllTargets{$Target}) + { + Error "invalid target name $Target\n"; + $Usage = 2; + last; + } + $Targets->{$Target} = 1; + } +} +if (defined $Usage) +{ + if ($Usage) + { + Error "try '$Name0 --help' for more information\n"; + exit $Usage; + } + print "Usage: $Name0 [--no-rm] [--help] [TARGETS]\n"; + print "\n"; + print "Performs all the tasks needed for the host to be ready to test new patches: update the Wine source and rebuild the Wine binaries.\n"; + print "\n"; + print "Where:\n"; + print " TARGETS Is a comma-separated list of targets to process. By default all\n"; + print " targets are processed.\n"; + print " - update: Update Wine's source code.\n"; + print " - win32: Rebuild the regular 32 bit Wine.\n"; + print " - wow32: Rebuild the 32 bit WoW Wine.\n"; + print " - wow64: Rebuild the 64 bit WoW Wine.\n"; + print " --no-rm Don't rebuild from scratch.\n"; + print " --help Shows this usage message.\n"; + exit 0; +} + +if (! -d "$DataDir/staging" and ! mkdir "$DataDir/staging") +{ + LogMsg "Unable to create '$DataDir/staging': $!\n"; + exit(1); +} + +if ($DataDir =~ /'/) +{ + LogMsg "The install path contains invalid characters\n"; + exit(1); +} + + +# +# Run the builds +# + +CountCPUs(); + +if (!BuildTestAgentd() || + !GitPull($Targets) || + !BuildWine($Targets, $NoRm, "win32", "") || + !BuildWine($Targets, $NoRm, "wow64", "--enable-win64") || + !BuildWine($Targets, $NoRm, "wow32", "--with-wine64='$DataDir/build-wow64'")) +{ + exit(1); +} + +LogMsg "ok\n"; +exit; diff --git a/testbot/bin/build/WineTest.pl b/testbot/bin/build/WineTest.pl new file mode 100755 index 000000000..01396f9bf --- /dev/null +++ b/testbot/bin/build/WineTest.pl @@ -0,0 +1,282 @@ +#!/usr/bin/perl +# -*- Mode: Perl; perl-indent-level: 2; indent-tabs-mode: nil -*- +# +# Applies the patch and rebuilds Wine. +# +# This script does not use tainting (-T) because its whole purpose is to run +# arbitrary user-provided code anyway. +# +# Copyright 2018 Francois Gouget +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +use warnings; +use strict; + +sub BEGIN +{ + if ($0 !~ m=^/=) + { + # Turn $0 into an absolute path so it can safely be used in @INC + require Cwd; + $0 = Cwd::cwd() . "/$0"; + } + if ($0 =~ m=^(/.*)/[^/]+/[^/]+/[^/]+$=) + { + $::RootDir = $1; + unshift @INC, "$::RootDir/lib"; + } + $::BuildEnv = 1; +} +my $Name0 = $0; +$Name0 =~ s+^.*/++; + + +use WineTestBot::Config; +use WineTestBot::PatchUtils; +use WineTestBot::Utils; + + +# +# Logging and error handling helpers +# + +sub InfoMsg(@) +{ + print @_; +} + +sub LogMsg(@) +{ + print "Task: ", @_; +} + +sub Error(@) +{ + print STDERR "$Name0:error: ", @_; +} + + +# +# Build helpers +# + +my $ncpus; +sub CountCPUs() +{ + if (open(my $fh, "<", "/proc/cpuinfo")) + { + # Linux + map { $ncpus++ if (/^processor/); } <$fh>; + close($fh); + } + $ncpus ||= 1; +} + +sub ApplyPatch($) +{ + my ($PatchFile) = @_; + + InfoMsg "Applying patch\n"; + system("cd '$DataDir/wine' && set -x && ". + "git apply --verbose ". ShQuote($PatchFile) ." && ". + "git add -A"); + if ($? != 0) + { + LogMsg "Patch failed to apply\n"; + return undef; + } + + my $Impacts = GetPatchImpact($PatchFile, "nounits"); + if ($Impacts->{Makefiles}) + { + InfoMsg "\nRunning make_makefiles\n"; + system("cd '$DataDir/wine' && set -x && ./tools/make_makefiles"); + if ($? != 0) + { + LogMsg "make_makefiles failed\n"; + return undef; + } + } + + if ($Impacts->{Autoconf} && !$Impacts->{HasConfigure}) + { + InfoMsg "\nRunning autoconf\n"; + system("cd '$DataDir/wine' && set -x && autoconf"); + if ($? != 0) + { + LogMsg "Autoconf failed\n"; + return undef; + } + } + + return $Impacts; +} + +sub BuildWine($$) +{ + my ($Targets, $Build) = @_; + + return 1 if (!$Targets->{$Build}); + + InfoMsg "\nRebuilding the $Build Wine\n"; + system("cd '$DataDir/build-$Build' && set -x && ". + "time make -j$ncpus"); + if ($? != 0) + { + LogMsg "The $Build build failed\n"; + return !1; + } + + return 1; +} + + +# +# Setup and command line processing +# + +$ENV{PATH} = "/usr/lib/ccache:/usr/bin:/bin"; +delete $ENV{ENV}; + +my %AllTargets; +map { $AllTargets{$_} = 1 } qw(win32 wow32 wow64); + +my ($Usage, $TargetList, $Action, $FileName); +while (@ARGV) +{ + my $Arg = shift @ARGV; + if ($Arg =~ /^(?:-\?|-h|--help)$/) + { + $Usage = 0; + last; + } + elsif ($Arg =~ /^-/) + { + Error "unknown option '$Arg'\n"; + $Usage = 2; + last; + } + elsif (!defined $TargetList) + { + $TargetList = $Arg; + } + elsif (!defined $Action) + { + $Action = $Arg; + } + elsif (!defined $FileName) + { + if (IsValidFileName($Arg)) + { + $FileName = "$DataDir/staging/$Arg"; + if (!-r $FileName) + { + Error "'$Arg' is not readable\n"; + $Usage = 2; + } + } + else + { + Error "invalid file name '$Arg'\n"; + $Usage = 2; + last; + } + } + else + { + Error "unexpected argument '$Arg'\n"; + $Usage = 2; + last; + } +} + +# Check and untaint parameters +my $Targets; +if (!defined $Usage) +{ + if (defined $TargetList) + { + foreach my $Target (split /,/, $TargetList) + { + if (!$AllTargets{$Target}) + { + Error "invalid target name $Target\n"; + $Usage = 2; + } + $Targets->{$Target} = 1; + } + } + else + { + Error "specify at least one target\n"; + $Usage = 2; + } + + if (!defined $Action) + { + Error "you must specify the action to perform\n"; + $Usage = 2; + } + elsif ($Action ne "build") + { + Error "invalid '$Action' action\n"; + $Usage = 2; + } + + if (!defined $FileName and $Action eq "build") + { + Error "you must provide a patch file\n"; + $Usage = 2; + } +} +if (defined $Usage) +{ + print "Usage: $Name0 [--help] TARGETS build PATCH\n"; + print "\n"; + print "Applies the specified patch and rebuilds Wine.\n"; + print "\n"; + print "Where:\n"; + print " TARGETS Is a comma-separated list of targets for the build.\n"; + print " - win32: The regular 32 bit Wine build.\n"; + print " - wow32: The 32 bit WoW Wine build.\n"; + print " - wow64: The 64 bit WoW Wine build.\n"; + print " build Verify that the patch compiles.\n"; + print " PATCH Is the staging file containing the patch to test.\n"; + print " --help Shows this usage message.\n"; + exit $Usage; +} + +if ($DataDir =~ /'/) +{ + LogMsg "The install path contains invalid characters\n"; + exit(1); +} + + +# +# Run the builds +# + +CountCPUs(); + +my $Impacts = ApplyPatch($FileName); +exit(1) if (!$Impacts or + !BuildWine($Targets, "win32") or + !BuildWine($Targets, "wow64") or + !BuildWine($Targets, "wow32")); + +LogMsg "ok\n"; +exit; diff --git a/testbot/ddl/update37.sql b/testbot/ddl/update37.sql new file mode 100644 index 000000000..cfa50ce2e --- /dev/null +++ b/testbot/ddl/update37.sql @@ -0,0 +1,4 @@ +USE winetestbot; + +ALTER TABLE VMs + MODIFY Type ENUM('win32', 'win64', 'build', 'wine') NOT NULL; diff --git a/testbot/ddl/winetestbot.sql b/testbot/ddl/winetestbot.sql index 1e7b1edd1..12b5dea1a 100644 --- a/testbot/ddl/winetestbot.sql +++ b/testbot/ddl/winetestbot.sql @@ -46,7 +46,7 @@ CREATE TABLE VMs ( Name VARCHAR(20) NOT NULL, SortOrder INT(3) NOT NULL, - Type ENUM('win32', 'win64', 'build') NOT NULL, + Type ENUM('win32', 'win64', 'build', 'wine') NOT NULL, Role ENUM('extra', 'base', 'winetest', 'retired', 'deleted') NOT NULL, Status ENUM('dirty', 'reverting', 'sleeping', 'idle', 'running', 'off', 'offline', 'maintenance') NOT NULL, Errors INT(2) NULL, diff --git a/testbot/lib/WineTestBot/Config.pm b/testbot/lib/WineTestBot/Config.pm index 66cb813ff..1ffff262f 100644 --- a/testbot/lib/WineTestBot/Config.pm +++ b/testbot/lib/WineTestBot/Config.pm @@ -31,7 +31,8 @@ use vars qw (@ISA @EXPORT @EXPORT_OK $UseSSL $LogDir $DataDir $BinDir $MaxVMsWhenIdle $SleepAfterRevert $WaitForToolsInVM $VMToolTimeout $MaxVMErrors $MaxTaskTries $AdminEMail $RobotEMail $WinePatchToOverride $WinePatchCc $SuiteTimeout $SingleTimeout - $BuildTimeout $ReconfigTimeout $TimeoutMargin $TagPrefix + $BuildTimeout $ReconfigTimeout $WineReconfigTimeout $TimeoutMargin + $TagPrefix $MaxUnitSize $ProjectName $PatchesMailingList $LDAPServer $LDAPBindDN $LDAPSearchBase $LDAPSearchFilter $LDAPRealNameAttribute $LDAPEMailAttribute $AgentPort $Tunnel @@ -45,7 +46,8 @@ require Exporter; $MaxRunningVMs $MaxVMsWhenIdle $SleepAfterRevert $WaitForToolsInVM $VMToolTimeout $MaxVMErrors $MaxTaskTries $AdminEMail $RobotEMail $WinePatchToOverride $WinePatchCc $SuiteTimeout - $SingleTimeout $BuildTimeout $ReconfigTimeout $TimeoutMargin + $SingleTimeout $BuildTimeout $ReconfigTimeout $WineReconfigTimeout + $TimeoutMargin $TagPrefix $MaxUnitSize $ProjectName $PatchesMailingList $LDAPServer $LDAPBindDN $LDAPSearchBase $LDAPSearchFilter $LDAPRealNameAttribute $LDAPEMailAttribute $AgentPort $Tunnel @@ -101,6 +103,9 @@ $BuildTimeout = 5 * 60; # (in seconds). Note that this includes building the native Wine build tools, # and the 32 and 64 bit test executables. $ReconfigTimeout = (1 + 2 * 5) * 60; +# How long to let a full Wine recompilation run before forcibly shutting it +# down (in seconds). +$WineReconfigTimeout = 20 * 60; # How much to add to the task timeout to account for file transfers, etc. $TimeoutMargin = 2 * 60; # Maximum amount of traces for a test unit. diff --git a/testbot/lib/WineTestBot/Engine/Scheduler.pm b/testbot/lib/WineTestBot/Engine/Scheduler.pm index 159b10cf8..3ed86e28d 100644 --- a/testbot/lib/WineTestBot/Engine/Scheduler.pm +++ b/testbot/lib/WineTestBot/Engine/Scheduler.pm @@ -392,7 +392,8 @@ sub _CheckAndClassifyVMs() 20) + # extra ($VM->Type eq "build" ? 0 : $VM->Type eq "win64" ? 1 : - 2); # win32 + $VM->Type eq "win32" ? 2 : + 3); # wine } # If a VM was in an inconsistent state, update the jobs status fields before diff --git a/testbot/lib/WineTestBot/PatchUtils.pm b/testbot/lib/WineTestBot/PatchUtils.pm index 3bf172125..1aeed6740 100644 --- a/testbot/lib/WineTestBot/PatchUtils.pm +++ b/testbot/lib/WineTestBot/PatchUtils.pm @@ -36,6 +36,22 @@ our @EXPORT = qw(GetPatchImpact UpdateWineData); use WineTestBot::Config; +# These paths are too generic to be proof that this is a Wine patch. +my $AmbiguousPathsRe = join('|', + 'Makefile\.in$', + # aclocal.m4 gets special treatment + # configure gets special treatment + # configure.ac gets special treatment + 'include/Makefile\.in$', + 'include/config\.h\.in$', + 'po/', + 'tools/Makefile.in', + 'tools/config.guess', + 'tools/config.sub', + 'tools/install-sh', + 'tools/makedep.c', +); + # Patches to these paths don't impact the Wine build. So ignore them. my $IgnoredPathsRe = join('|', '\.mailmap$', @@ -53,6 +69,7 @@ my $IgnoredPathsRe = join('|', 'tools/winemaker/', ); + =pod =over 12 @@ -75,6 +92,7 @@ sub UpdateWineData($) } my $_TimeStamp; +my $_WineFiles; my $_TestList; =pod @@ -83,7 +101,7 @@ my $_TestList; =item C<_LoadWineFiles()> Reads latest/winefiles.txt to build a per-module hashtable of the test unit -files. +files and a hashtable of all the Wine files. =back =cut @@ -101,11 +119,13 @@ sub _LoadWineFiles() $_TimeStamp = $MTime; $_TestList = {}; + $_WineFiles = {}; if (open(my $fh, "<", $FileName)) { while (my $Line = <$fh>) { chomp $Line; + $_WineFiles->{$Line} = 1; if ($Line =~ m~^\w+/([^/]+)/tests/([^/]+)$~) { @@ -121,11 +141,23 @@ sub _LoadWineFiles() sub _HandleFile($$$) { - my ($Impacts, $Path, $Change) = @_; + my ($Impacts, $FilePath, $Change) = @_; - if ($Path =~ m~^(dlls|programs)/([^/]+)/tests/([^/\s]+)$~) + if ($Change eq "new") + { + delete $Impacts->{DeletedFiles}->{$FilePath}; + $Impacts->{NewFiles}->{$FilePath} = 1; + } + elsif ($Change eq "rm") + { + delete $Impacts->{NewFiles}->{$FilePath}; + $Impacts->{DeletedFiles}->{$FilePath} = 1; + } + + if ($FilePath =~ m~^(dlls|programs)/([^/]+)/tests/([^/\s]+)$~) { my ($Root, $Module, $File) = ($1, $2, $3); + $Impacts->{IsWinePatch} = 1; $Impacts->{TestBuild} = 1; my $Tests = $Impacts->{Tests}; @@ -163,8 +195,29 @@ sub _HandleFile($$$) } else { - # Figure out if this patch impacts the Wine build - $Impacts->{WineBuild} = 1 if ($Path !~ /^(?:$IgnoredPathsRe)/); + my $WineFiles = $Impacts->{WineFiles} || $_WineFiles; + if ($WineFiles->{$FilePath}) + { + if ($FilePath !~ /^(?:$AmbiguousPathsRe)/) + { + $Impacts->{IsWinePatch} = 1; + } + # Else this file exists in Wine but has a very common name so it may just + # as well belong to another repository. Still update WineBuild in case + # this patch really is for Wine. + + if ($FilePath !~ /^(?:$IgnoredPathsRe)/) + { + $Impacts->{WineBuild} = 1; + } + # Else patches to this file don't impact the Wine build. + } + elsif ($FilePath =~ m~/Makefile.in$~ and $Change eq "new") + { + # This may or may not be a Wine patch but the new Makefile.in will be + # added to the build by make_makefiles. + $Impacts->{WineBuild} = $Impacts->{Makefiles} = 1; + } } } @@ -195,6 +248,21 @@ sub GetPatchImpact($;$$) if ($PastImpacts) { + if ($PastImpacts->{WineBuild} or $PastImpacts->{TestBuild}) + { + # Update the list of Wine files so we correctly recognize patchset parts + # that modify new Wine files. + my $WineFiles = $PastImpacts->{WineFiles} || $_WineFiles; + map { $Impacts->{WineFiles}->{$_} = 1 } keys %{$WineFiles}; + map { $Impacts->{WineFiles}->{$_} = 1 } keys %{$PastImpacts->{NewFiles}}; + map { delete $Impacts->{WineFiles}->{$_} } keys %{$PastImpacts->{DeletedFiles}}; + } + else + { + $Impacts->{NewFiles} = $PastImpacts->{NewFiles}; + $Impacts->{DeletedFiles} = $PastImpacts->{DeletedFiles}; + } + foreach my $PastInfo (values %{$PastImpacts->{Tests}}) { if ($PastInfo->{Files}) @@ -207,6 +275,7 @@ sub GetPatchImpact($;$$) } } } + my ($Path, $Change); while (my $Line = <$fh>) { @@ -221,6 +290,7 @@ sub GetPatchImpact($;$$) elsif ($Line =~ m=^--- \w+/tools/make_makefiles$=) { $Impacts->{WineBuild} = $Impacts->{Makefiles} = 1; + $Impacts->{IsWinePatch} = 1; } elsif ($Line =~ m=^--- /dev/null$=) { diff --git a/testbot/lib/WineTestBot/Patches.pm b/testbot/lib/WineTestBot/Patches.pm index 258aa2c19..03a0ebf16 100644 --- a/testbot/lib/WineTestBot/Patches.pm +++ b/testbot/lib/WineTestBot/Patches.pm @@ -135,32 +135,24 @@ sub Submit($$$) my $PastImpacts; $PastImpacts = GetPatchImpact($PatchFileName) if ($IsSet); my $Impacts = GetPatchImpact("$DataDir/patches/" . $self->Id, undef, $PastImpacts); - if (!$Impacts->{UnitCount}) - { - $self->Disposition(($IsSet ? "Part" : "Patch") . - " doesn't affect tests"); - return undef; - } - my $User; - my $Users = CreateUsers(); - if (defined($self->FromEMail)) + if (!$Impacts->{WineBuild} and !$Impacts->{TestBuild}) { - $Users->AddFilter("EMail", [$self->FromEMail]); - if (! $Users->IsEmpty()) + if ($Impacts->{IsWinePatch}) { - $User = @{$Users->GetItems()}[0]; + $self->Disposition(($IsSet ? "Part does" : "Does") + ." not impact the Wine build"); } - } - if (! defined($User)) - { - $User = GetBatchUser(); + else + { + $self->Disposition(($IsSet ? "Part is not" : "Not") ." a Wine patch"); + } + return undef; } # Create a new job for this patch my $Jobs = CreateJobs(); my $NewJob = $Jobs->Add(); - $NewJob->User($User); $NewJob->Priority(6); my $PropertyDescriptor = $Jobs->GetPropertyDescriptorByName("Remarks"); my $Subject = $self->Subject; @@ -171,84 +163,137 @@ sub Submit($$$) $PropertyDescriptor->GetMaxLength())); $NewJob->Patch($self); - # Add build step to the job - my $BuildStep = $NewJob->Steps->Add(); - $BuildStep->FileName("patch.diff"); - $BuildStep->FileType("patchdlls"); # This is irrelevant now - $BuildStep->InStaging(!1); - $BuildStep->Type("build"); - $BuildStep->DebugLevel(0); - - # Add build task - my $VMs = CreateVMs(); - $VMs->AddFilter("Type", ["build"]); - $VMs->AddFilter("Role", ["base"]); - my $BuildVM = ${$VMs->GetItems()}[0]; - my $Task = $BuildStep->Tasks->Add(); - $Task->VM($BuildVM); - $Task->Timeout($BuildTimeout); - - # Save the build step so the others can reference it. - my ($ErrKey, $ErrProperty, $ErrMessage) = $Jobs->Save(); - if (defined($ErrMessage)) + my $User; + my $Users = CreateUsers(); + if (defined $self->FromEMail) { - $self->Disposition("Failed to submit build step"); - return $ErrMessage; + $Users->AddFilter("EMail", [$self->FromEMail]); + $User = @{$Users->GetItems()}[0] if (!$Users->IsEmpty()); } + $NewJob->User($User || GetBatchUser()); - # Stage the patch so it can be picked up by the job - if (!link($PatchFileName, "$DataDir/staging/job". $NewJob->Id ."_patch.diff")) + my $BuildVMs = CreateVMs(); + $BuildVMs->AddFilter("Type", ["build"]); + $BuildVMs->AddFilter("Role", ["base"]); + if ($Impacts->{UnitCount} and !$BuildVMs->IsEmpty()) { - $self->Disposition("Failed to stage the patch file"); - return $!; - } + # Create the Build Step + my $BuildStep = $NewJob->Steps->Add(); + $BuildStep->FileName("patch.diff"); + $BuildStep->FileType("patchdlls"); + $BuildStep->InStaging(!1); + $BuildStep->Type("build"); + $BuildStep->DebugLevel(0); + + # Add build task + my $BuildVM = ${$BuildVMs->GetItems()}[0]; + my $Task = $BuildStep->Tasks->Add(); + $Task->VM($BuildVM); + $Task->Timeout($BuildTimeout); + + # Save the build step so the others can reference it. + my ($ErrKey, $ErrProperty, $ErrMessage) = $Jobs->Save(); + if (defined($ErrMessage)) + { + $self->Disposition("Failed to submit build step"); + return $ErrMessage; + } - foreach my $Module (sort keys %{$Impacts->{Tests}}) - { - my $TestInfo = $Impacts->{Tests}->{$Module}; - foreach my $Unit (sort keys %{$TestInfo->{Units}}) + # Create steps for the Windows tests + foreach my $Module (sort keys %{$Impacts->{Tests}}) { - # Add 32 and 64-bit tasks - foreach my $Bits ("32", "64") + my $TestInfo = $Impacts->{Tests}->{$Module}; + foreach my $Unit (sort keys %{$TestInfo->{Units}}) { - $VMs = CreateVMs(); - $VMs->AddFilter("Type", $Bits eq "32" ? ["win32", "win64"] : ["win64"]); - $VMs->AddFilter("Role", ["base"]); - if (@{$VMs->GetKeys()}) + foreach my $Bits ("32", "64") { - # Create the corresponding Step - my $NewStep = $NewJob->Steps->Add(); - $NewStep->PreviousNo($BuildStep->No); - my $FileName = $TestInfo->{ExeBase}; - $FileName .= "64" if ($Bits eq "64"); - $NewStep->FileName("$FileName.exe"); - $NewStep->FileType("exe$Bits"); - $NewStep->InStaging(!1); - - # And a task for each VM - my $Tasks = $NewStep->Tasks; - my $SortedKeys = $VMs->SortKeysBySortOrder($VMs->GetKeys()); - foreach my $VMKey (@$SortedKeys) + my $WinVMs = CreateVMs(); + $WinVMs->AddFilter("Type", $Bits eq "32" ? ["win32", "win64"] : ["win64"]); + $WinVMs->AddFilter("Role", ["base"]); + if (!$WinVMs->IsEmpty()) { - my $VM = $VMs->GetItem($VMKey); - my $Task = $Tasks->Add(); - $Task->VM($VM); - $Task->Timeout($SingleTimeout); - $Task->CmdLineArg($Unit); + # Create one Step per (module, unit, bitness) combination + my $NewStep = $NewJob->Steps->Add(); + $NewStep->PreviousNo($BuildStep->No); + my $FileName = $TestInfo->{ExeBase}; + $FileName .= "64" if ($Bits eq "64"); + $NewStep->FileName("$FileName.exe"); + $NewStep->FileType("exe$Bits"); + $NewStep->InStaging(!1); + + # And a task for each VM + my $Tasks = $NewStep->Tasks; + my $SortedKeys = $WinVMs->SortKeysBySortOrder($WinVMs->GetKeys()); + foreach my $VMKey (@$SortedKeys) + { + my $VM = $WinVMs->GetItem($VMKey); + my $Task = $Tasks->Add(); + $Task->VM($VM); + $Task->Timeout($SingleTimeout); + $Task->CmdLineArg($Unit); + } } } } } } + my $WineVMs = CreateVMs(); + $WineVMs->AddFilter("Type", ["wine"]); + $WineVMs->AddFilter("Role", ["base"]); + if (!$WineVMs->IsEmpty()) + { + # Add a Wine step to the job + my $NewStep = $NewJob->Steps->Add(); + $NewStep->FileName("patch.diff"); + $NewStep->FileType("patchdlls"); + $NewStep->InStaging(!1); + $NewStep->DebugLevel(0); + + # And a task for each VM + my $Tasks = $NewStep->Tasks; + my $SortedKeys = $WineVMs->SortKeysBySortOrder($WineVMs->GetKeys()); + foreach my $VMKey (@$SortedKeys) + { + my $VM = $WineVMs->GetItem($VMKey); + my $Task = $Tasks->Add(); + $Task->VM($VM); + # Only verify that the win32 version compiles + $Task->Timeout($WineReconfigTimeout); + $Task->CmdLineArg("win32"); + } + } + + if ($NewJob->Steps->IsEmpty()) + { + # This may be a Wine patch but there is no suitable VM to test it! + if ($Impacts->{UnitCount}) + { + $self->Disposition("No build or test VM!"); + } + else + { + $self->Disposition(($IsSet ? "Part does" : "Does") ." not impact the ". + ($WineVMs->IsEmpty() ? "Windows " : "") ."tests"); + } + return undef; + } + # Save it all - ($ErrKey, $ErrProperty, $ErrMessage) = $Jobs->Save(); + my ($ErrKey, $ErrProperty, $ErrMessage) = $Jobs->Save(); if (defined $ErrMessage) { $self->Disposition("Failed to submit job"); return $ErrMessage; } + # Stage the patch so it can be picked up by the job + if (!link($PatchFileName, "$DataDir/staging/job". $NewJob->Id ."_patch.diff")) + { + $self->Disposition("Failed to stage the patch file"); + return $!; + } + # Switch Status to staging to indicate we are done setting up the job $NewJob->Status("staging"); ($ErrKey, $ErrProperty, $ErrMessage) = $Jobs->Save(); @@ -348,27 +393,6 @@ sub IsPatch($$) return !1; } -sub IsTestPatch($$) -{ - my ($self, $Body) = @_; - - if (open(BODY, "<" . $Body->path)) - { - my $Line; - while (defined($Line = <BODY>)) - { - if ($Line =~ m/^\+\+\+ .*\/(dlls|programs)\/[^\/]+\/tests\/[^\/\s]+/) - { - close BODY; - return 1; - } - } - close BODY; - } - - return !1; -} - =pod =over 12 @@ -412,7 +436,6 @@ sub NewPatch($$$) my $ErrMessage; if (scalar(@PatchBodies) == 1) { - $Patch->AffectsTests($self->IsTestPatch($PatchBodies[0])); my $Subject = $Patch->Subject; $Subject =~ s/32\/64//; $Subject =~ s/64\/32//; diff --git a/testbot/lib/WineTestBot/PendingPatchSets.pm b/testbot/lib/WineTestBot/PendingPatchSets.pm index bfd05e4a3..fc22a8687 100644 --- a/testbot/lib/WineTestBot/PendingPatchSets.pm +++ b/testbot/lib/WineTestBot/PendingPatchSets.pm @@ -248,8 +248,7 @@ sub NewSubmission($$) if (! $Set->CheckSubsetComplete($PartNo)) { - $Patch->Disposition($Patch->AffectsTests ? "Set not complete yet" : - "Patch doesn't affect tests"); + $Patch->Disposition("Set not complete yet"); } else { @@ -260,16 +259,11 @@ sub NewSubmission($$) my $Part = $Parts->GetItem($PartNo); if (defined($Part)) { - if ($Part->Patch->AffectsTests) + $ErrMessage = $Set->SubmitSubset($PartNo, $Part->Patch); + if (!defined $ErrMessage) { - $ErrMessage = $Set->SubmitSubset($PartNo, $Part->Patch); + (my $ErrProperty, $ErrMessage) = $Part->Patch->Save(); } - else - { - $Part->Patch->Disposition("Patch doesn't affect tests"); - } - my $ErrProperty; - ($ErrProperty, $ErrMessage) = $Part->Patch->Save(); } else { diff --git a/testbot/lib/WineTestBot/StepsTasks.pm b/testbot/lib/WineTestBot/StepsTasks.pm index 84c5a005f..05854bca7 100644 --- a/testbot/lib/WineTestBot/StepsTasks.pm +++ b/testbot/lib/WineTestBot/StepsTasks.pm @@ -78,7 +78,7 @@ sub GetTitle($) { $Title .= "64 bit "; } - $Title .= $self->CmdLineArg; + $Title .= $self->CmdLineArg || ""; } elsif ($self->Type eq "build") { diff --git a/testbot/lib/WineTestBot/Tasks.pm b/testbot/lib/WineTestBot/Tasks.pm index afd4fb4d3..54ebf5360 100644 --- a/testbot/lib/WineTestBot/Tasks.pm +++ b/testbot/lib/WineTestBot/Tasks.pm @@ -177,8 +177,9 @@ sub Run($$) my ($self, $Step) = @_; my ($JobId, $StepNo, $TaskNo) = @{$self->GetMasterKey()}; - my $Script = $Step->Type eq "build" ? "Build" : - $Step->Type eq "reconfig" ? "Reconfig" : + my $Script = $Step->Type eq "reconfig" ? "Reconfig" : + $self->VM->Type eq "wine" ? "WineTest" : + $Step->Type eq "build" ? "Build" : "Task"; my $Args = ["$BinDir/${ProjectName}Run$Script.pl", "--log-only", $JobId, $StepNo, $TaskNo]; diff --git a/testbot/lib/WineTestBot/VMs.pm b/testbot/lib/WineTestBot/VMs.pm index 2f494cb81..cf7bd5ab7 100644 --- a/testbot/lib/WineTestBot/VMs.pm +++ b/testbot/lib/WineTestBot/VMs.pm @@ -306,10 +306,10 @@ sub Validate($) { my ($self) = @_; - if ($self->Type ne "win32" && $self->Type ne "win64" && - ($self->Role eq "winetest" || $self->Role eq "extra")) + if ($self->Type !~ /^(?:win32|win64|wine)$/ and + $self->Role =~ /^(?:extra|winetest)$/) { - return ("Role", "Only win32 and win64 VMs can have a role of '" . $self->Role . "'"); + return ("Role", "Only win32, win64 and wine VMs can have a role of '" . $self->Role . "'"); } return $self->SUPER::Validate(); } @@ -664,7 +664,7 @@ sub CreateItem($) my @PropertyDescriptors = ( CreateBasicPropertyDescriptor("Name", "VM name", 1, 1, "A", 20), CreateBasicPropertyDescriptor("SortOrder", "Display order", !1, 1, "N", 3), - CreateEnumPropertyDescriptor("Type", "Type of VM", !1, 1, ['win32', 'win64', 'build']), + CreateEnumPropertyDescriptor("Type", "Type of VM", !1, 1, ['win32', 'win64', 'build', 'wine']), CreateEnumPropertyDescriptor("Role", "VM Role", !1, 1, ['extra', 'base', 'winetest', 'retired', 'deleted']), CreateEnumPropertyDescriptor("Status", "Current status", !1, 1, ['dirty', 'reverting', 'sleeping', 'idle', 'running', 'off', 'offline', 'maintenance']), CreateBasicPropertyDescriptor("Errors", "Errors", !1, !1, "N", 2), -- 2.17.1
For now the Wine VMs are unchecked by default since all they do is a compilation check which is mostly redundant with the regular build step. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> --- testbot/web/Submit.pl | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/testbot/web/Submit.pl b/testbot/web/Submit.pl index afebc722b..9fa252a11 100644 --- a/testbot/web/Submit.pl +++ b/testbot/web/Submit.pl @@ -244,10 +244,14 @@ sub GenerateFields($) { $VMs->AddFilter("Type", ["win64"]); } - else + elsif ($self->{FileType} eq "exe32") { $VMs->AddFilter("Type", ["win32", "win64"]); } + else + { + $VMs->AddFilter("Type", ["win32", "win64", "wine"]); + } if ($self->{ShowAll}) { # All but the retired and deleted ones @@ -272,6 +276,10 @@ sub GenerateFields($) $Status = " [". $VM->Status ."]"; $Checked = undef; } + elsif ($VM->Type eq "wine") + { + $Checked = undef; + } if ($Checked and ($self->GetParam("Page") == 1 || $self->GetParam($FieldName))) { @@ -844,6 +852,39 @@ sub OnSubmit($) } } + if ($FileType eq "patchdlls") + { + my $Tasks; + my $VMs = CreateVMs(); + $VMs->AddFilter("Type", ["wine"]); + my $SortedKeys = $VMs->SortKeysBySortOrder($VMs->GetKeys()); + foreach my $VMKey (@$SortedKeys) + { + my $VM = $VMs->GetItem($VMKey); + my $FieldName = "vm_" . $self->CGI->escapeHTML($VMKey); + next if (!$self->GetParam($FieldName)); # skip unselected VMs + + if (!$Tasks) + { + # First create the Wine test step + my $WineStep = $Steps->Add(); + $WineStep->FileName($BaseName); + $WineStep->FileType($FileType); + $WineStep->InStaging(!1); + $WineStep->Type("build"); + $WineStep->DebugLevel($self->GetParam("DebugLevel")); + $WineStep->ReportSuccessfulTests(defined($self->GetParam("ReportSuccessfulTests"))); + $Tasks = $WineStep->Tasks; + } + + # Then add a task for this VM + my $Task = $Tasks->Add(); + $Task->VM($VM); + $Task->CmdLineArg("win32"); + $Task->Timeout($WineReconfigTimeout); + } + } + # Now save it all (or whatever's left to save) my ($ErrKey, $ErrProperty, $ErrMessage) = $Jobs->Save(); if (defined($ErrMessage)) -- 2.17.1
participants (1)
-
Francois Gouget