[tools 2/2] testbot/LibvirtTool: Limit the administrator notifications for offline VMs.

[Francois Gouget]

Tweak the notification to reflect that.
This also means the LibvirtTool monitor should not announce that the VM
is working again since it may very well still be broken, for instance if
the required snapshot is missing. Instead announce the VM works again
only when a revert was successful.

So now the administrator gets:
* $MaxVMErrors-1 messages that the VM is offline
* 1 message that the VM needs maintenance
* 1 message when the VM works again

Signed-off-by: Francois Gouget 
---
 testbot/bin/LibvirtTool.pl | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/testbot/bin/LibvirtTool.pl b/testbot/bin/LibvirtTool.pl
index a025085d44..c027d6bd68 100755
--- a/testbot/bin/LibvirtTool.pl
+++ b/testbot/bin/LibvirtTool.pl
@@ -196,19 +196,22 @@ sub FatalError($)
   {
     LogMsg "Could not put the $VMKey VM offline: $SaveErrMessage ($ErrProperty)\n";
   }
-  elsif ($Errors >= $MaxVMErrors)
+  elsif ($Errors < $MaxVMErrors)
   {
-    NotifyAdministrator("The $VMKey VM needs maintenance",
-                        "Got $Errors consecutive errors working on the $VMKey VM:\n".
+    NotifyAdministrator("Putting the $VMKey VM offline ($Errors)",
+                        "Could not perform the $Action operation on the $VMKey VM:\n".
                         "\n$ErrMessage\n".
-                        "It probably needs fixing to get back online.");
+                        "Got $Errors consecutive errors.\n".
+                        "The VM has been put offline and the TestBot will try to regain access to it.");
   }
-  else
+  elsif ($Errors == $MaxVMErrors)
   {
-    NotifyAdministrator("Putting the $VMKey VM offline",
-                        "Could not perform the $Action operation on the $VMKey VM:\n".
+    NotifyAdministrator("The $VMKey VM needs maintenance",
+                        "Got $Errors consecutive errors working on the $VMKey VM:\n".
                         "\n$ErrMessage\n".
-                        "The VM has been put offline and the TestBot will try to regain access to it.");
+                        "It probably needs fixing to get back online.\n".
+                        "* Spacing out attempts to revive it.\n".
+                        "* No further error notification will be sent until it starts working again.\n");
   }
   exit 1;
 }
@@ -405,10 +408,11 @@ sub Monitor()
     }
     if ($IsReady)
     {
+      # The VM may well still be broken, for instance if the required snapshot
+      # does not exist. But only revert can tell. For this reason, let revert
+      # announce when the VM works again.
       return 1 if (ChangeStatus("offline", "off", "done"));
-      NotifyAdministrator("The $VMKey VM is working again",
-                          "The $VMKey VM started working again after ".
-                          PrettyElapsed($Start) .".");
+      LogMsg("Switching $VMKey from offline to off after ". PrettyElapsed($Start) .".");
       return 0;
     }

@@ -821,9 +825,15 @@ sub Revert()
   Debug(Elapsed($Start), " Sleeping ${Sleep}s\n");
   LogMsg "Letting $VMKey settle down for ${Sleep}s\n";
   sleep($Sleep);
-
   CheckBootCount($TA);
-  return ChangeStatus($CurrentStatus, "idle", "done");
+
+  return 1 if (ChangeStatus($CurrentStatus, "idle", "done"));
+  if ($VM->Errors)
+  {
+    NotifyAdministrator("The $VMKey VM is working again",
+                        "After ". $VM->Errors ." consecutive errors $VMKey was successfully reverted to $Config->{snapshot} in ". PrettyElapsed($Start) ." and is now idle.");
+  }
+  return 0;
 }