Merge ~kissiel/plainbox:optimizations into plainbox:master

[Maciej Kisielewski]

This branch brings a bunch of optimizations to plainbox/checkbox spoolup time.

When invoking `plainbox -h`, the elapsed time is 5-10x shorter.
As or checkbox-cli invocations it's usually more than 2x faster.

I tested it through and through with launchers and with explicit plainbox invocations.
This branch should not bring any changes in functionality

[Sylvain Pineau (sylvain-pineau)]

https://code.launchpad.net/~kissiel/plainbox/+git/plainbox/+ref/optimizations does not exist in lp. weird. Could you push again?

[Maciej Kisielewski (kissiel)]

repushed

review: Needs Resubmitting

[Sylvain Pineau (sylvain-pineau)]

Amazing work, each commit bringing its own performance fix.
Unit tests now run ~ 2X faster.

review: Approve

diff --git a/plainbox/impl/commands/cmd_session.py b/plainbox/impl/commands/cmd_session.py
index 066e399..cb41241 100644
--- a/plainbox/impl/commands/cmd_session.py
+++ b/plainbox/impl/commands/cmd_session.py
@@ -112,7 +112,7 @@ class SessionCommand(PlainBoxCommand):
         group = export_parser.add_argument_group(_("output options"))
         group.add_argument(
             '-f', '--output-format', default='text',
-            metavar=_('FORMAT'), choices=[_('?')] + get_all_exporter_names(),
+            metavar=_('FORMAT'), choices=[_('?')],
             help=_('save test results in the specified FORMAT'
                    ' (pass ? for a list of choices)'))
         group.add_argument(
diff --git a/plainbox/impl/secure/providers/v1.py b/plainbox/impl/secure/providers/v1.py
index 9f85bcb..fb9a59f 100644
--- a/plainbox/impl/secure/providers/v1.py
+++ b/plainbox/impl/secure/providers/v1.py
@@ -314,8 +314,14 @@ class UnitPlugIn(ProviderContentPlugIn):
         for unit in (unit for unit in inspect_result
                      if unit.Meta.name == 'test plan'):
             if unit.include is not None:
+                patterns = []
+                for line in unit.include.split('\n'):
+                    if '#' in line:
+                        line = line.split('#')[0]
+                    if line:
+                        patterns.append('${}^'.format(line))
                 yield WhiteList(
-                    unit.include, name=unit.partial_id, origin=unit.origin,
+                    patterns, name=unit.partial_id, origin=unit.origin,
                     implicit_namespace=unit.provider.namespace)
 
     # NOTE: this version of plugin_object() is just for legacy code support
diff --git a/plainbox/impl/secure/qualifiers.py b/plainbox/impl/secure/qualifiers.py
index 9f8e4af..773d953 100644
--- a/plainbox/impl/secure/qualifiers.py
+++ b/plainbox/impl/secure/qualifiers.py
@@ -136,13 +136,20 @@ class RegExpJobQualifier(SimpleQualifier):
     expression
     """
 
+    # RegExp qualifier is instantiated many times. To speed up the process,
+    # let's keep cache of compiled re object. (usually there are only a few of
+    # those
+    re_cache = dict()
+
     def __init__(self, pattern, origin, inclusive=True):
         """
         Initialize a new RegExpJobQualifier with the specified pattern.
         """
         super().__init__(origin, inclusive)
         try:
-            self._pattern = re.compile(pattern)
+            if pattern not in self.re_cache:
+                self.re_cache[pattern] = re.compile(pattern)
+            self._pattern = self.re_cache[pattern]
         except sre_constants.error as exc:
             assert len(exc.args) == 1
             # XXX: This is a bit crazy but this lets us have identical error
diff --git a/plainbox/impl/secure/rfc822.py b/plainbox/impl/secure/rfc822.py
index d194fd7..f144a1e 100644
--- a/plainbox/impl/secure/rfc822.py
+++ b/plainbox/impl/secure/rfc822.py
@@ -42,10 +42,13 @@ logger = logging.getLogger("plainbox.secure.rfc822")
 
 
 def normalize_rfc822_value(value):
-    # Remove the multi-line dot marker
-    value = re.sub('^(\s*)\.$', '\\1', value, flags=re.M)
-    # Remove consistent indentation
-    value = textwrap.dedent(value)
+    # multi-line markers and consistent indentation happens only on multi-line
+    # values, so let's run those operations only on multi-line values
+    if value.count('\n') > 1:
+        # Remove the multi-line dot marker
+        value = re.sub('^(\s*)\.$', '\\1', value, flags=re.M)
+        # Remove consistent indentation
+        value = textwrap.dedent(value)
     # Strip the remaining whitespace
     value = value.strip()
     return value
diff --git a/plainbox/impl/session/state.py b/plainbox/impl/session/state.py
index e92ddfa..d36b4ea 100644
--- a/plainbox/impl/session/state.py
+++ b/plainbox/impl/session/state.py
@@ -245,6 +245,7 @@ class SessionDeviceContext:
             self._provider_list = []
             self._state = SessionState(self._unit_list)
             self._unit_id_map = {}
+            self._already_added_checksums = set()
         else:
             if not isinstance(state, SessionState):
                 raise TypeError
@@ -257,6 +258,8 @@ class SessionDeviceContext:
             self._state = state
             self._unit_id_map = {unit.id: unit for unit in state.unit_list if
                                  isinstance(unit, UnitWithId)}
+            self._already_added_checksums = set(
+                [unit.checksum for unit in self.unit_list])
 
         self._test_plan_list = []
         # Connect SessionState's signals to fire our signals. This
@@ -397,9 +400,10 @@ class SessionDeviceContext:
         self.on_provider_added(provider)
         if add_units:
             for unit in provider.unit_list:
-                self.add_unit(unit)
+                self.add_unit(unit, False)
+        self.state._recompute_job_readiness()
 
-    def add_unit(self, unit):
+    def add_unit(self, unit, recompute=True):
         """
         Add a unit to the context.
 
@@ -413,10 +417,11 @@ class SessionDeviceContext:
 
         This method fires the :meth:`on_unit_added()` signal
         """
-        if unit in frozenset(self._unit_list):
+        if unit.checksum in self._already_added_checksums:
             raise ValueError(
                 _("attempting to add the same unit twice: %s" % unit.id))
-        self.state.add_unit(unit)
+        self._already_added_checksums.add(unit.checksum)
+        self.state.add_unit(unit, recompute)
         # NOTE: no need to fire the on_unit_added() signal because the state
         # object and we've connected it to will fire our version.
 
@@ -432,6 +437,7 @@ class SessionDeviceContext:
         if unit not in self._unit_list:
             raise ValueError(
                 _("attempting to remove unit not in this context"))
+        self._already_added_checksums.remove(unit.checksum)
         self.state.remove_unit(unit)
         # NOTE: no need to fire the on_unit_removed() signal because the state
         # object and we've connected it to will fire our version.
diff --git a/plainbox/impl/session/test_manager.py b/plainbox/impl/session/test_manager.py
index 1c43bfd..975a6a0 100644
--- a/plainbox/impl/session/test_manager.py
+++ b/plainbox/impl/session/test_manager.py
@@ -26,12 +26,12 @@ Test definitions for plainbox.impl.session.manager module
 
 from unittest import expectedFailure
 
-from plainbox.abc import IJobDefinition
 from plainbox.impl.session import SessionManager
 from plainbox.impl.session import SessionState
 from plainbox.impl.session import SessionStorage
 from plainbox.impl.session.state import SessionDeviceContext
 from plainbox.impl.session.suspend import SessionSuspendHelper
+from plainbox.impl.unit.job import JobDefinition
 from plainbox.vendor import mock
 from plainbox.vendor.morris import SignalTestCase
 
@@ -105,7 +105,7 @@ class SessionManagerTests(SignalTestCase):
         verify that SessionManager.load_session() correctly delegates the task
         to various other objects
         """
-        job = mock.Mock(name='job', spec_set=IJobDefinition)
+        job = mock.Mock(name='job', spec_set=JobDefinition)
         unit_list = [job]
         flags = None
         helper_name = "plainbox.impl.session.manager.SessionResumeHelper"
diff --git a/plainbox/impl/unit/unit.py b/plainbox/impl/unit/unit.py
index 0b7b4ce..33ba52c 100644
--- a/plainbox/impl/unit/unit.py
+++ b/plainbox/impl/unit/unit.py
@@ -560,7 +560,7 @@ class Unit(metaclass=UnitType):
         """
         value = self._data.get('_{}'.format(name))
         if value is None:
-            value = self._data.get('{}'.format(name), default)
+            value = self._data.get(name, default)
         if value is not None and self.is_parametric:
             value = string.Formatter().vformat(value, (), self.parameters)
         return value