Francois Gouget : testbot: Always notify the administrator when putting a VM offline.
Module: tools Branch: master Commit: 48a24be93225a51f8caa4df6b10bf89096560e99 URL: https://source.winehq.org/git/tools.git/?a=commit;h=48a24be93225a51f8caa4df6... Author: Francois Gouget <fgouget(a)codeweavers.com> Date: Tue Sep 10 11:07:57 2019 +0200 testbot: Always notify the administrator when putting a VM offline. Include the reason why the VM was put offline in the notification email. Add NotifyAdministrator() to the Utils module to simplify sending the notification. Signed-off-by: Francois Gouget <fgouget(a)codeweavers.com> Signed-off-by: Alexandre Julliard <julliard(a)winehq.org> --- testbot/bin/LibvirtTool.pl | 25 +-------------------- testbot/bin/WineRunBuild.pl | 6 ++++- testbot/bin/WineRunReconfig.pl | 6 ++++- testbot/bin/WineRunTask.pl | 6 ++++- testbot/bin/WineRunWineTest.pl | 6 ++++- testbot/lib/WineTestBot/Engine/Scheduler.pm | 5 +++++ testbot/lib/WineTestBot/Utils.pm | 34 ++++++++++++++++++++++++++++- testbot/lib/WineTestBot/VMs.pm | 5 +++++ 8 files changed, 64 insertions(+), 29 deletions(-) diff --git a/testbot/bin/LibvirtTool.pl b/testbot/bin/LibvirtTool.pl index 4000c14..e348834 100755 --- a/testbot/bin/LibvirtTool.pl +++ b/testbot/bin/LibvirtTool.pl @@ -43,6 +43,7 @@ $Name0 =~ s+^.*/++; use WineTestBot::Config; use WineTestBot::Log; +use WineTestBot::Utils; use WineTestBot::VMs; my $Debug; @@ -58,30 +59,6 @@ sub Error(@) LogMsg @_; } -sub NotifyAdministrator($$) -{ - my ($Subject, $Body) = @_; - - if (open(my $fh, "|/usr/sbin/sendmail -oi -t -odq")) - { - LogMsg "Notifying administrator: $Subject\n"; - print $fh <<"EOF"; -From: $RobotEMail -To: $AdminEMail -Subject: $Subject - -$Body -EOF - close($fh); - } - else - { - LogMsg "Could not send administrator notification: $!\n"; - LogMsg " Subject: $Subject\n"; - LogMsg " Body: $Body\n"; - } -} - # # Setup and command line processing diff --git a/testbot/bin/WineRunBuild.pl b/testbot/bin/WineRunBuild.pl index fdcf770..1d57c44 100755 --- a/testbot/bin/WineRunBuild.pl +++ b/testbot/bin/WineRunBuild.pl @@ -280,10 +280,14 @@ sub FatalTAError($$) my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn(); if (!defined $IsPoweredOn) { - # The VM host is not accessible anymore so mark the VM as offline and + # The VM host is not accessible anymore so put the VM offline and # requeue the task. This does not count towards the task's tries limit # since neither the VM nor the task are at fault. Error("$ErrMessage\n"); + NotifyAdministrator("Putting the ". $VM->Name ." VM offline", + "A TestAgent operation to the ". $VM->Name ." VM failed:\n". + "\n$ErrMessage\n". + "So the VM has been put offline and the TestBot will try to regain access to it."); WrapUpAndExit('queued'); } diff --git a/testbot/bin/WineRunReconfig.pl b/testbot/bin/WineRunReconfig.pl index 493ab10..bad5f76 100755 --- a/testbot/bin/WineRunReconfig.pl +++ b/testbot/bin/WineRunReconfig.pl @@ -282,10 +282,14 @@ sub FatalTAError($$) my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn(); if (!defined $IsPoweredOn) { - # The VM host is not accessible anymore so mark the VM as offline and + # The VM host is not accessible anymore so put the VM offline and # requeue the task. This does not count towards the task's tries limit # since neither the VM nor the task are at fault. Error("$ErrMessage\n"); + NotifyAdministrator("Putting the ". $VM->Name ." VM offline", + "A TestAgent operation to the ". $VM->Name ." VM failed:\n". + "\n$ErrMessage\n". + "So the VM has been put offline and the TestBot will try to regain access to it."); WrapUpAndExit('queued'); } diff --git a/testbot/bin/WineRunTask.pl b/testbot/bin/WineRunTask.pl index 6a71849..53e9e50 100755 --- a/testbot/bin/WineRunTask.pl +++ b/testbot/bin/WineRunTask.pl @@ -332,10 +332,14 @@ sub FatalTAError($$;$) my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn(); if (!defined $IsPoweredOn) { - # The VM host is not accessible anymore so mark the VM as offline and + # The VM host is not accessible anymore so put the VM offline and # requeue the task. This does not count towards the task's tries limit # since neither the VM nor the task are at fault. Error("$ErrMessage\n"); + NotifyAdministrator("Putting the ". $VM->Name ." VM offline", + "A TestAgent operation to the ". $VM->Name ." VM failed:\n". + "\n$ErrMessage\n". + "So the VM has been put offline and the TestBot will try to regain access to it."); WrapUpAndExit('queued'); } diff --git a/testbot/bin/WineRunWineTest.pl b/testbot/bin/WineRunWineTest.pl index 74b974a..508605b 100755 --- a/testbot/bin/WineRunWineTest.pl +++ b/testbot/bin/WineRunWineTest.pl @@ -332,10 +332,14 @@ sub FatalTAError($$;$) my $IsPoweredOn = $VM->GetDomain()->IsPoweredOn(); if (!defined $IsPoweredOn) { - # The VM host is not accessible anymore so mark the VM as offline and + # The VM host is not accessible anymore so put the VM offline and # requeue the task. This does not count towards the task's tries limit # since neither the VM nor the task are at fault. Error("$ErrMessage\n"); + NotifyAdministrator("Putting the ". $VM->Name ." VM offline", + "A TestAgent operation to the ". $VM->Name ." VM failed:\n". + "\n$ErrMessage\n". + "So the VM has been put offline and the TestBot will try to regain access to it."); WrapUpAndExit('queued'); } diff --git a/testbot/lib/WineTestBot/Engine/Scheduler.pm b/testbot/lib/WineTestBot/Engine/Scheduler.pm index 1b42ea9..39d2342 100644 --- a/testbot/lib/WineTestBot/Engine/Scheduler.pm +++ b/testbot/lib/WineTestBot/Engine/Scheduler.pm @@ -37,6 +37,7 @@ use WineTestBot::Engine::Events; use WineTestBot::Jobs; use WineTestBot::Log; use WineTestBot::RecordGroups; +use WineTestBot::Utils; use WineTestBot::VMs; @@ -277,6 +278,10 @@ sub _CheckAndClassifyVMs() if ($Errors >= $MaxVMErrors) { $NewStatus = "maintenance"; + NotifyAdministrator("Putting the $VMKey VM in maintenance mode", + "The last $Errors revert operations timed out.\n\n". + "No further operation will be attempted until an administrator has put\n". + "the VM back online."); $Sched->{busyvms}->{$VMKey} = 1; } } diff --git a/testbot/lib/WineTestBot/Utils.pm b/testbot/lib/WineTestBot/Utils.pm index e28a34b..83f60fa 100644 --- a/testbot/lib/WineTestBot/Utils.pm +++ b/testbot/lib/WineTestBot/Utils.pm @@ -30,7 +30,8 @@ use Exporter 'import'; our @EXPORT = qw(MakeSecureURL SecureConnection GenerateRandomString OpenNewFile CreateNewFile CreateNewLink CreateNewDir DurationToString BuildEMailRecipient IsValidFileName - BuildTag SanitizeTag LocaleName ShQuote ShArgv2Cmd); + BuildTag SanitizeTag LocaleName NotifyAdministrator + ShQuote ShArgv2Cmd); use Fcntl; @@ -276,6 +277,37 @@ sub BuildTag($;$) } +# +# EMail helper +# + +sub NotifyAdministrator($$) +{ + my ($Subject, $Body) = @_; + + if (open(my $fh, "|/usr/sbin/sendmail -oi -t -odq")) + { + require WineTestBot::Log; + WineTestBot::Log::LogMsg("Notifying administrator: $Subject\n"); + print $fh <<"EOF"; +From: $RobotEMail +To: $AdminEMail +Subject: $Subject + +$Body +EOF + close($fh); + } + else + { + require WineTestBot::Log; + WineTestBot::Log::LogMsg("Could not send administrator notification: $!\n"); + WineTestBot::Log::LogMsg(" Subject: $Subject\n"); + WineTestBot::Log::LogMsg(" Body: $Body\n"); + } +} + + # # Shell helpers # diff --git a/testbot/lib/WineTestBot/VMs.pm b/testbot/lib/WineTestBot/VMs.pm index 5d738e2..78e066f 100644 --- a/testbot/lib/WineTestBot/VMs.pm +++ b/testbot/lib/WineTestBot/VMs.pm @@ -452,6 +452,11 @@ sub Run($$$$$$) # Reset the Status and ChildPid since the exec failed $self->Status("offline"); + NotifyAdministrator("Putting the ". $self->Name ." VM offline", + "Could not run $Tool on the ". $self->Name ." VM. The failed command is:\n\n". + ShArgv2Cmd(@$Args) ."\n\n". + "An administrator should investigate why the command failed but the\n". + "TestBot will still try to continue using the VM.\n"); $self->ChildDeadline(undef); $self->ChildPid(undef); my ($ErrProperty, $ErrMessage) = $self->Save();
participants (1)
-
Alexandre Julliard