[PATCH 3/3] testbot: Reschedule at the latest when the next task times out.

[Francois Gouget]

This ensures we catch stuck tasks in a timely fashion.
Note that we still reschedule every 10 minutes to catch any issues but
the scheduler handles this itself instead of relying on SafetyNet().

Signed-off-by: Francois Gouget 
---
 testbot/bin/Engine.pl                       | 14 ++++++------
 testbot/lib/WineTestBot/Engine/Scheduler.pm | 24 +++++++++++++++++++++
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/testbot/bin/Engine.pl b/testbot/bin/Engine.pl
index c3db46b12..0f8de82af 100755
--- a/testbot/bin/Engine.pl
+++ b/testbot/bin/Engine.pl
@@ -611,11 +611,10 @@ sub ClientRead($)

This is called on startup and regularly after that to catch things that fall
 through the cracks, possibly because of an Engine restart.
-Specifically it updates the status of all the current Jobs, Steps and
-Tasks, then schedules Tasks to be run, checks the staging directory for
-wine-patches emails dropped by WinePatchesMLSubmit.pl, for notifications of
-changes on Wine's Patches web site dropped by WinePatchesWebSubmit.pl, and
-checks whether any pending patchsets are now complete and thus can be scheduled.
+Specifically it checks the staging directory for wine-patches emails dropped
+by WinePatchesMLSubmit.pl, for notifications of changes on Wine's Patches web
+site dropped by WinePatchesWebSubmit.pl, and checks whether any pending
+patchsets are now complete and thus can be scheduled.

=back
 =cut
@@ -623,7 +622,6 @@ checks whether any pending patchsets are now complete and thus can be scheduled.
 sub SafetyNet()
 {
   CheckJobs();
-  ScheduleJobs();
   HandleWinePatchWebSubmission();

my $Set = CreatePendingPatchSets();
@@ -760,6 +758,10 @@ sub main()
   SaveRecord('engine', 'start');
   Cleanup(1);

+  # Cleanup starts processes to update the VMs. Run ScheduleJobs() now so it
+  # can detect if these processes get stuck.
+  ScheduleJobs();
+
   # Check for patches that arrived while the server was off.
   HandleWinePatchMLSubmission();

diff --git a/testbot/lib/WineTestBot/Engine/Scheduler.pm b/testbot/lib/WineTestBot/Engine/Scheduler.pm
index 52e7619dd..a191f0e1c 100644
--- a/testbot/lib/WineTestBot/Engine/Scheduler.pm
+++ b/testbot/lib/WineTestBot/Engine/Scheduler.pm
@@ -31,6 +31,7 @@ use Exporter 'import';
 our @EXPORT = qw(ScheduleJobs CheckJobs);

use WineTestBot::Config;
+use WineTestBot::Engine::Events;
 use WineTestBot::Jobs;
 use WineTestBot::Log;
 use WineTestBot::RecordGroups;
@@ -971,6 +972,29 @@ sub ScheduleJobs()
   {
     $Sched->{recordgroups}->DeleteItem($Sched->{recordgroup});
   }
+
+  # Reschedule at the latest when the next task times out
+  my $FirstDeadline;
+  foreach my $VM (@{$Sched->{VMs}->GetItems()})
+  {
+    if (defined $VM->ChildDeadline and
+        (!defined $FirstDeadline or $VM->ChildDeadline < $FirstDeadline))
+    {
+      $FirstDeadline = $VM->ChildDeadline;
+    }
+  }
+  my $Timeout;
+  if ($FirstDeadline)
+  {
+    $Timeout = $FirstDeadline - time();
+    $Timeout = 1 if ($Timeout <= 0);
+  }
+  if (!$Timeout or $Timeout > 600)
+  {
+    # Reschedule regularly as a safety net
+    $Timeout = 600;
+  }
+  AddEvent("ScheduleJobs", $Timeout, 0, &ScheduleJobs);
 }