Merge ~sylvain-pineau/checkbox-ng:remote-auto-retry into checkbox-ng:master

[Sylvain Pineau]

Support for auto-retry of failed jobs, as set in launchers:

[ui]
auto_retry = yes
max_attempts = 5
delay_before_retry = 30

Nota: Since the delay_before_retry is handled on master side, disconnecting the master when rerunning jobs does not let them run on slave. We have to reconnect to continue.

[Maciej Kisielewski (kissiel)]

Code looks good. I'm +1 on landing it. It'll get tested heavily on my extended ctrl+c support branch, so we can iterate on it quickly if something goes bad.

review: Approve

diff --git a/checkbox_ng/launcher/remote.py b/checkbox_ng/launcher/remote.py
index 70bdc60..945068d 100644
--- a/checkbox_ng/launcher/remote.py
+++ b/checkbox_ng/launcher/remote.py
@@ -376,50 +376,11 @@ class RemoteMaster(Command, ReportsStage, MainLoopStage):
         if any([x['user'] is not None for x in jobs_repr]):
             self.password_query()
 
-        for job in jobs_repr:
-            SimpleUI.header(
-                _('Running job {} / {}').format(
-                    job['num'], total_num,
-                    fill='-'))
-            SimpleUI.header(job['name'])
-            print(_("ID: {0}").format(job['id']))
-            print(_("Category: {0}").format(job['category_name']))
-            SimpleUI.horiz_line()
-            next_job = False
-            for interaction in self.sa.run_job(job['id']):
-                if interaction.kind == 'sudo_input':
-                    self.sa.save_password(
-                        self._sudo_provider.encrypted_password)
-                if interaction.kind == 'purpose':
-                    SimpleUI.description(_('Purpose:'), interaction.message)
-                elif interaction.kind in ['description', 'steps']:
-                    SimpleUI.description(_('Steps:'), interaction.message)
-                    if job['command'] is None:
-                        cmd = 'run'
-                    else:
-                        cmd = SimpleUI(None).wait_for_interaction_prompt(None)
-                    if cmd == 'skip':
-                        next_job = True
-                    self.sa.remember_users_response(cmd)
-                elif interaction.kind == 'verification':
-                    self.wait_for_job(dont_finish=True)
-                    if interaction.message:
-                        SimpleUI.description(
-                            _('Verification:'), interaction.message)
-                    JobAdapter = namedtuple('job_adapter', ['command'])
-                    job = JobAdapter(job['command'])
-                    cmd = SimpleUI(None)._interaction_callback(
-                        job, interaction.extra)
-                    self.sa.remember_users_response(cmd)
-                    self.finish_job(interaction.extra.get_result())
-                    next_job = True
-                elif interaction.kind == 'comment':
-                    new_comment = input(SimpleUI.C.BLUE(
-                        _('Please enter your comments:') + '\n'))
-                    self.sa.remember_users_response(new_comment + '\n')
-            if next_job:
-                continue
-            self.wait_for_job()
+        self._run_jobs(jobs_repr, total_num)
+        if self.launcher.auto_retry:
+            while True:
+                if not self._maybe_auto_retry_jobs():
+                    break
         self.finish_session()
 
     def resume_interacting(self, interaction):
@@ -473,3 +434,65 @@ class RemoteMaster(Command, ReportsStage, MainLoopStage):
         exported_stream.seek(0)
         result = transport.send(exported_stream)
         return result
+
+    def _maybe_auto_retry_jobs(self):
+        # create a list of jobs that qualify for rerunning
+        retry_candidates = self.sa.get_auto_retry_candidates()
+        # bail-out early if no job qualifies for rerunning
+        if not retry_candidates:
+            return False
+        # we wait before retrying
+        delay = self.launcher.delay_before_retry
+        _logger.info(_("Waiting {} seconds before retrying failed"
+                       " jobs...".format(delay)))
+        time.sleep(delay)
+        # include resource jobs that jobs to retry depend on
+        candidates = self.sa.prepare_auto_retry_candidates(retry_candidates)
+        self._run_jobs(self.sa.get_jobs_repr(candidates), len(candidates))
+        return True
+
+    def _run_jobs(self, jobs_repr, total_num=0):
+        for job in jobs_repr:
+            SimpleUI.header(
+                _('Running job {} / {}').format(
+                    job['num'], total_num,
+                    fill='-'))
+            SimpleUI.header(job['name'])
+            print(_("ID: {0}").format(job['id']))
+            print(_("Category: {0}").format(job['category_name']))
+            SimpleUI.horiz_line()
+            next_job = False
+            for interaction in self.sa.run_job(job['id']):
+                if interaction.kind == 'sudo_input':
+                    self.sa.save_password(
+                        self._sudo_provider.encrypted_password)
+                if interaction.kind == 'purpose':
+                    SimpleUI.description(_('Purpose:'), interaction.message)
+                elif interaction.kind in ['description', 'steps']:
+                    SimpleUI.description(_('Steps:'), interaction.message)
+                    if job['command'] is None:
+                        cmd = 'run'
+                    else:
+                        cmd = SimpleUI(None).wait_for_interaction_prompt(None)
+                    if cmd == 'skip':
+                        next_job = True
+                    self.sa.remember_users_response(cmd)
+                elif interaction.kind == 'verification':
+                    self.wait_for_job(dont_finish=True)
+                    if interaction.message:
+                        SimpleUI.description(
+                            _('Verification:'), interaction.message)
+                    JobAdapter = namedtuple('job_adapter', ['command'])
+                    job = JobAdapter(job['command'])
+                    cmd = SimpleUI(None)._interaction_callback(
+                        job, interaction.extra)
+                    self.sa.remember_users_response(cmd)
+                    self.finish_job(interaction.extra.get_result())
+                    next_job = True
+                elif interaction.kind == 'comment':
+                    new_comment = input(SimpleUI.C.BLUE(
+                        _('Please enter your comments:') + '\n'))
+                    self.sa.remember_users_response(new_comment + '\n')
+            if next_job:
+                continue
+            self.wait_for_job()
diff --git a/plainbox/impl/session/remote_assistant.py b/plainbox/impl/session/remote_assistant.py
index e1a9930..6390b9e 100644
--- a/plainbox/impl/session/remote_assistant.py
+++ b/plainbox/impl/session/remote_assistant.py
@@ -262,6 +262,10 @@ class RemoteSessionAssistant():
     def finish_bootstrap(self):
         self._sa.finish_bootstrap()
         self._state = Bootstrapped
+        if self._launcher.auto_retry:
+            for job_id in self._sa.get_static_todo_list():
+                job_state = self._sa.get_job_state(job_id)
+                job_state.attempts = self._launcher.max_attempts
         return self._sa.get_static_todo_list()
 
     def save_todo_list(self, chosen_jobs):
@@ -431,11 +435,51 @@ class RemoteSessionAssistant():
         self._sa.use_job_result(self._currently_running_job, result)
         if self._state != Bootstrapping:
             if not self._sa.get_dynamic_todo_list():
-                self._state = Idle
+                if (
+                    self._launcher.auto_retry and
+                    self.get_auto_retry_candidates()
+                ):
+                    self._state = TestsSelected
+                else:
+                    self._state = Idle
             else:
                 self._state = TestsSelected
         return result
 
+    def get_auto_retry_candidates(self):
+        """Get all the tests that might be selected for an automatic retry."""
+        def retry_predicate(job_state):
+            return job_state.result.outcome in (IJobResult.OUTCOME_FAIL,) \
+                and job_state.effective_auto_retry != 'no'
+        retry_candidates = []
+        todo_list = self._sa.get_static_todo_list()
+        job_states = {job_id: self._sa.get_job_state(job_id) for job_id
+                      in todo_list}
+        for job_id, job_state in job_states.items():
+            if retry_predicate(job_state) and job_state.attempts > 0:
+                retry_candidates.append(self._sa.get_job(job_id))
+        return retry_candidates
+
+    def prepare_auto_retry_candidates(self, retry_candidates):
+        """Include resource jobs that jobs to retry depend on."""
+        candidates = []
+        resources_to_rerun = []
+        for job in retry_candidates:
+            job_state = self._sa.get_job_state(job.id)
+            for inhibitor in job_state.readiness_inhibitor_list:
+                if inhibitor.cause == InhibitionCause.FAILED_DEP:
+                    resources_to_rerun.append(inhibitor.related_job)
+        # reset outcome of jobs that are selected for re-running
+        for job in retry_candidates + resources_to_rerun:
+            self._sa.get_job_state(job.id).result = MemoryJobResult({})
+            candidates.append(job.id)
+            _logger.info("{}: {} attempts".format(
+                job.id,
+                self._sa.get_job_state(job.id).attempts
+            ))
+        self._state = TestsSelected
+        return candidates
+
     def get_jobs_repr(self, job_ids, offset=0):
         """
         Translate jobs into a {'field': 'val'} representations.