Merge lp:~cr3/checkbox/whitelist_ordering_with_job_cache into lp:~sylvain-pineau/checkbox/whitelist_ordering_with_job_cache

[Marc Tardif]

[Sylvain Pineau (sylvain-pineau)]

As simple as a call to cmp. thanks

review: Approve

=== modified file 'checkbox/job.py'
--- checkbox/job.py	2012-02-14 20:32:39 +0000
+++ checkbox/job.py	2012-02-15 16:55:31 +0000
@@ -17,7 +17,6 @@
 # along with Checkbox.  If not, see <http://www.gnu.org/licenses/>.
 #
 import os
-import re
 import logging
 
 from gettext import gettext as _
@@ -105,58 +104,6 @@
 class JobStore(MessageStore):
     """A job store which stores its jobs in a file system hierarchy."""
 
-    msg = {}      # msg stored in a the /store filesystem
-    depends = {}  # jobs dependencies
-
-    def whitelist_ordering(self, whitelist_patterns):
-        """
-        Sort the Message file objects by whitelist patterns
-        """
-
-        logging.debug("Whitelist ordering in progress")
-        jobs = [] # ordered jobs
-
-        for filename in self._walk_messages():
-            message = self._read_message(filename, cache=True)
-            name = message.get("name")
-            os.rename(filename, filename + ".tmp")
-            self.msg[name] = filename
-            self.depends[name] = message.get("depends")
-
-        for p in whitelist_patterns:
-            job = next((n for n in self.msg.iterkeys() if p.match(n)), None)
-            if job:
-                jobs.append(job)
-
-        jobs_orig = list(jobs)
-
-        # Overrides the whitelist order and puts the requirements
-        # ahead of the dependent job
-        for name in [n for n in jobs if self.depends[n]]:
-            alldeps = self._get_dependencies([name])
-            missingdeps = set()
-            for dep in alldeps:
-                if dep not in jobs:
-                    logging.warning('"%s" dependency not found' % dep)
-                    missingdeps.add(dep)
-                    continue
-            # Dependencies must be run in whitelist order
-            for dep in sorted(alldeps-missingdeps, key=jobs_orig.index):
-                limit = jobs.index(name)
-                dep_index = jobs.index(dep)
-                if dep_index > limit:
-                    jobs.insert(limit, jobs.pop(dep_index))
-
-        for i, name in enumerate(jobs):
-            try:
-                os.rename(self.msg[name] + ".tmp",
-                          re.sub(r'\d+$', str(i), self.msg[name]))
-            except:
-                logging.warning(
-                    'Unable to process "%s", possible duplicated values' % name)
-
-        logging.debug("Whitelist ordering completed")
-
     def add(self, job):
         # TODO: Order alphabetically within suite or non-suite
 
@@ -191,21 +138,3 @@
                     break
             else:
                 yield filename
-
-    def _get_dependencies(self, jobs):
-        """
-        Get all dependencies including ancestors
-        """
-
-        dependencies = set()
-
-        for name in jobs:
-            if name in self.msg:
-                msg = self.msg[name]
-                depends = self.depends[name]
-                if depends:
-                    dependencies.update(
-                        set(depends),
-                        self._get_dependencies(depends))
-
-        return dependencies

=== modified file 'checkbox/user_interface.py'
--- checkbox/user_interface.py	2012-02-13 01:44:19 +0000
+++ checkbox/user_interface.py	2012-02-15 16:55:31 +0000
@@ -27,8 +27,6 @@
 import gettext
 from gettext import gettext as _
 
-from gi.repository import Gio
-
 from checkbox.contrib.REThread import REThread
 
 from checkbox.lib.environ import add_variable, get_variable, remove_variable
@@ -180,6 +178,8 @@
                 if os.getenv("DISPLAY") and \
                         subprocess.call(["pgrep", "-x", "-u", str(uid), "gnome-panel|gconfd-2"],
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0:
+                    from gi.repository import Gio
+
                     preferred_xml_app = Gio.app_info_get_default_for_type("application/xml",False)
                     if preferred_xml_app:
                         preferred_browser = preferred_xml_app.get_executable()

=== modified file 'plugins/jobs_info.py'
--- plugins/jobs_info.py	2012-02-14 20:32:39 +0000
+++ plugins/jobs_info.py	2012-02-15 16:55:31 +0000
@@ -54,16 +54,16 @@
 
     # Space separated list of directories where job files are stored.
     directories = List(Path(),
-        default_factory=lambda:"%(checkbox_share)s/jobs")
+        default_factory=lambda: "%(checkbox_share)s/jobs")
 
     # List of jobs to blacklist
-    blacklist = List(String(), default_factory=lambda:"")
+    blacklist = List(String(), default_factory=lambda: "")
 
     # Path to blacklist file
     blacklist_file = Path(required=False)
 
     # List of jobs to whitelist
-    whitelist = List(String(), default_factory=lambda:"")
+    whitelist = List(String(), default_factory=lambda: "")
 
     # Path to whitelist file
     whitelist_file = Path(required=False)
@@ -82,8 +82,8 @@
             try:
                 file = open(filename)
             except IOError, e:
-                error_message=(gettext.gettext("Failed to open file '%s': %s") % 
-                        (filename,e.strerror))
+                error_message=(gettext.gettext("Failed to open file '%s': %s") %
+                        (filename, e.strerror))
                 logging.critical(error_message)
                 sys.stderr.write("%s\n" % error_message)
                 sys.exit(os.EX_NOINPUT)
@@ -95,11 +95,15 @@
 
     def gather(self):
         # Register temporary handler for report-message events
+        messages = []
+
         def report_message(message):
-            self._manager.reactor.fire("report-job", message)
+            if self.whitelist_patterns:
+                name = message["name"]
+                if not [name for p in self.whitelist_patterns if p.match(name)]:
+                    return
 
-        # Send whitelist info to jobs-prompt for job ordering
-        self._manager.reactor.fire("whitelist", self.whitelist_patterns)
+            messages.append(message)
 
         # Set domain and message event handler
         old_domain = gettext.textdomain()
@@ -109,6 +113,23 @@
         for directory in self.directories:
             self._manager.reactor.fire("message-directory", directory)
 
+        # Apply whitelist ordering
+        def cmp_function(a, b):
+            a_name = a["name"]
+            b_name = b["name"]
+            for pattern in self.whitelist_patterns:
+                if pattern.match(a_name):
+                    a_index = self.whitelist_patterns.index(pattern)
+                elif pattern.match(b_name):
+                    b_index = self.whitelist_patterns.index(pattern)
+
+            return cmp(a_index, b_index)
+
+        if self.whitelist_patterns:
+            messages = sorted(messages, cmp=cmp_function)
+        for message in messages:
+            self._manager.reactor.fire("report-job", message)
+
         # Unset domain and event handler
         self._manager.reactor.cancel_call(event_id)
         gettext.textdomain(old_domain)

=== modified file 'plugins/jobs_prompt.py'
--- plugins/jobs_prompt.py	2012-02-14 20:32:39 +0000
+++ plugins/jobs_prompt.py	2012-02-15 16:55:31 +0000
@@ -61,8 +61,6 @@
              ("prompt-job", self.prompt_job),
              ("prompt-jobs", self.prompt_jobs),
              ("prompt-finish", self.prompt_finish),
-             ("jobs-ordering", self.jobs_ordering),
-             ("whitelist", self.report_whitelist),
              ("report", self.report),
              ("report-job", self.report_job)]:
             self._manager.reactor.call_on(rt, rh)
@@ -125,13 +123,6 @@
         if interface.direction == NEXT:
             self.store.delete_all_messages()
 
-    def jobs_ordering(self):
-        if self.whitelist_patterns:
-            self.store.whitelist_ordering(self.whitelist_patterns)
-
-    def report_whitelist(self, whitelist):
-        self.whitelist_patterns = whitelist
-
     def report(self):
         self.store.set_pending_offset(0)
         messages = self.store.get_pending_messages()

=== modified file 'plugins/suites_prompt.py'
--- plugins/suites_prompt.py	2012-02-14 20:32:39 +0000
+++ plugins/suites_prompt.py	2012-02-15 16:55:31 +0000
@@ -84,9 +84,6 @@
                 self._statuses[job["name"]] = job["status"]
 
     def prompt_gather(self, interface):
-        # Whitelist ordering
-        self._manager.reactor.fire("jobs-ordering")
-
         # Resolve dependencies
         resolver = Resolver()
         for key in self._jobs.iterkeys():