Merge lp:~robru/cupstream2distro/moar-bileto into lp:cupstream2distro

[Robert Bruce Park]

Convert source_archive and source_series to Bileto.

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:1123
http://jenkins.qa.ubuntu.com/job/cu2d-choo-choo-ci/763/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/cu2d-choo-choo-ci/763/rebuild

review: Approve (continuous-integration)

[lp:~robru/cupstream2distro/moar-bileto]

1124. By Robert Bruce Park on 2015-09-06

Print silo summary early during build.

[Robert Bruce Park (robru)]

Looks good in staging.

review: Approve

=== modified file 'citrain/build.py'
--- citrain/build.py	2015-09-05 03:54:05 +0000
+++ citrain/build.py	2015-09-06 00:07:30 +0000
@@ -60,7 +60,6 @@
 from citrain.recipes.sourcesync import SourceSync
 from citrain.recipes.binarysync import BinarySync
 from cupstream2distro.silomanager import SiloState
-from cupstream2distro.launchpadmanager import lp
 from cupstream2distro.errors import BuildError, CITrainError
 from cupstream2distro.utils import (
     env,
@@ -124,9 +123,8 @@
             Source: self.names & set(self.silo_state.sources),
         }
         if self.silo_state.source_archive:
-            Sync.from_archive = lp.load(self.silo_state.source_archive)
-            Sync.from_series = lp.get_series(
-                self.silo_state.source_series, dest=Sync.from_archive)
+            Sync.from_archive = self.silo_state.source_archive
+            Sync.from_series = self.silo_state.source_series
         Merge.all_mps = self.silo_state.mps
 
     def instantiate_build_objects(self):

=== modified file 'citrain/prepare_silo.py'
--- citrain/prepare_silo.py	2015-09-05 19:41:20 +0000
+++ citrain/prepare_silo.py	2015-09-06 00:07:30 +0000
@@ -45,10 +45,8 @@
 
 from cupstream2distro.settings import (
     BILETO_GET,
-    LANDING_SCHEME,
     SILO_NAME_LIST,
     SILOS_DIR,
-    STABLE_OVERLAY_PPA,
 )
 from cupstream2distro.utils import (
     SILO_DIR,
@@ -133,32 +131,9 @@
     silo_state = SiloState.new_blank(
         siloname=env.SILONAME,
         requestid=env.REQUEST_ID)
-    if env.SYNC_REQUEST:
-        prepare_sync(silo_state)
     return silo_state
 
 
-def prepare_sync(silo_state):
-    """Process sync request."""
-    archive, comma, series = env.SYNC_REQUEST.partition(',')
-    source_series = series or env.SERIES
-    if archive.startswith('ppa:'):
-        source_archive = lp.get_ppa(archive.split(':')[1].strip('~'))
-    elif archive.isdigit():
-        siloname = '{}/landing-{:03d}'.format(env.DISTRIBUTION, int(archive))
-        source_archive = lp.get_ppa(LANDING_SCHEME.format(siloname))
-        source_state = SiloState(siloname)
-        source_series = source_state.series.name
-    elif archive == 'stable-overlay':
-        source_archive = lp.get_ppa(STABLE_OVERLAY_PPA)
-    else:
-        source_archive = lp.distributions[archive].main_archive
-    logging.info('Syncing from {} {}.'.format(
-        source_series, source_archive.web_link))
-    silo_state.source_archive = source_archive.self_link
-    silo_state.source_series = source_series
-
-
 def check_merge_status(merge):
     """Raise PrepError if merge is in wrong state."""
     try:

=== modified file 'citrain/recipes/base.py'
--- citrain/recipes/base.py	2015-09-03 18:50:16 +0000
+++ citrain/recipes/base.py	2015-09-06 00:07:30 +0000
@@ -168,6 +168,7 @@
         silo_state.set_empty()
         silo_state.set_preparing()
         silo_state.save_config()
+        logging.info(silo_state.summarize())
 
     def clean_phase(self):
         """Delete files leftover from previous builds."""

=== modified file 'cupstream2distro/silomanager.py'
--- cupstream2distro/silomanager.py	2015-09-05 20:33:19 +0000
+++ cupstream2distro/silomanager.py	2015-09-06 00:07:30 +0000
@@ -41,9 +41,9 @@
     LANDING_SCHEME,
     SILO_NAME_LIST,
     SILOS_DIR,
+    STABLE_OVERLAY_PPA,
 )
 from cupstream2distro.utils import (
-    SILO_DIR,
     env,
     memoize,
     log_value_of,
@@ -176,7 +176,7 @@
     @property
     def request_id_file(self):
         """Identify the file that contains the requestid."""
-        return SILO_DIR('request_id')
+        return join(SILOS_DIR, self.siloname, 'request_id')
 
     @staticmethod
     def find_first_available(distribution='ubuntu'):
@@ -463,7 +463,7 @@
     @property
     def step_file(self):
         """Define filename indicating silo has been published."""
-        return SILO_DIR('step')
+        return join(SILOS_DIR, self.siloname, 'step')
 
     @property
     def step(self):
@@ -537,9 +537,10 @@
             return self._series.split('+')[-1]
 
     @property
+    @memoize
     def sources(self):
         """Identify the manual source packages configured for this silo."""
-        archive = self.sync_request.split(',')[0]
+        archive = self.sync_request.partition(',')[0]
         sources = splitter(self._sources)
         if not sources and archive.isdigit():
             return self.load_source_sync_sources(int(archive))
@@ -549,7 +550,7 @@
     def load_source_sync_sources(self, silonum):
         """Check what packages are configured in a different silo."""
         self.info('Inferring source names from silo {}.'.format(silonum))
-        return SiloState(
+        return self.get_source_state(
             '{}/landing-{:03d}'.format(self._distribution, silonum)
         ).all_projects
 
@@ -574,24 +575,39 @@
         return lp.load('{}/{}'.format(self._distribution, series))
 
     @property
+    @memoize
     def source_archive(self):
-        """Identify the archive we are synching from."""
-        return self._config.get('source_archive')
-
-    @source_archive.setter
-    def source_archive(self, value):
-        """Record the archive we are syncing from."""
-        self._config['source_archive'] = value
+        """Identify the archive we are syncing from."""
+        archive = self._sync_request.partition(',')[0]
+        if archive.startswith('ppa:'):
+            return lp.get_ppa(archive.split(':')[1].strip('~'))
+        elif archive.isdigit():
+            siloname = '{}/landing-{:03d}'.format(
+                self._distribution, int(archive))
+            return lp.get_ppa(LANDING_SCHEME.format(siloname))
+        elif archive == 'stable-overlay':
+            return lp.get_ppa(STABLE_OVERLAY_PPA)
+        elif archive:
+            return lp.distributions[archive].main_archive
 
     @property
+    @memoize
     def source_series(self):
         """Identify the distro series we are syncing from."""
-        return self._config.get('source_series')
+        archive, comma, series = self._sync_request.partition(',')
+        if archive.isdigit():
+            siloname = '{}/landing-{:03d}'.format(
+                self._distribution, int(archive))
+            return self.get_source_state(siloname).series
+        elif archive:
+            return lp.load('{}/{}'.format(
+                self.source_archive.distribution.name,
+                series or self._series))
 
-    @source_series.setter
-    def source_series(self, value):
-        """Record the distro series we are syncing from."""
-        self._config['source_series'] = value
+    @memoize
+    def get_source_state(self, siloname):
+        """Fetch SiloState object representing some other silo."""
+        return SiloState(siloname)
 
     def append_mp(self, source_package, mp_link):
         """Record a new MP URL that this silo will build."""
@@ -604,7 +620,7 @@
     logging.error(message)
     if env.SILONAME:
         silo_state = SiloState(env.SILONAME)
-        silo_state.set_config_status(-1, message)
+        silo_state.set_config_status(message)
         silo_state.save_config()
 
 

=== modified file 'tests/unit/test_script_build.py'
--- tests/unit/test_script_build.py	2015-09-05 03:54:05 +0000
+++ tests/unit/test_script_build.py	2015-09-06 00:07:30 +0000
@@ -112,13 +112,9 @@
                 self.script.SourceSync: set(['b']),
             })
         self.assertEqual(self.script.Merge.all_mps, silo_state.mps)
-        self.assertEqual(self.script.lp.get_series.mock_calls, [
-            call(silo_state.source_series,
-                 dest=self.script.SourceSync.from_archive),
-        ])
         self.assertEqual(
             self.script.SourceSync.from_series,
-            self.script.lp.get_series.return_value)
+            silo_state.source_series)
 
     def test_buildmanager_instantiate_build_objects(self):
         """Create all build objects."""

=== modified file 'tests/unit/test_script_prepare_silo.py'
--- tests/unit/test_script_prepare_silo.py	2015-09-05 19:16:06 +0000
+++ tests/unit/test_script_prepare_silo.py	2015-09-06 00:07:30 +0000
@@ -23,7 +23,6 @@
 from cupstream2distro import project
 from cupstream2distro.utils import env, os_path_join_safe
 from cupstream2distro.errors import PrepError
-from cupstream2distro.settings import STABLE_OVERLAY_PPA
 
 from tests.unit import CITrainScriptTestCase
 
@@ -169,87 +168,10 @@
         env.DISTRIBUTION = 'ubuntu'
         env.SERIES = 'wily'
         self.script.lp.distributions = dict(ubuntu=Mock())
-        self.script.prepare_sync = Mock()
         self.script.create_new_state()
         self.script.SiloState.new_blank.assert_called_once_with(
             siloname='ubuntu/landing-123',
             requestid='42')
-        self.script.prepare_sync.assert_called_once_with(
-            self.script.SiloState.new_blank.return_value)
-
-    def test_prepare_sync_ppa(self):
-        """Configure syncs correctly from a PPA."""
-        silo_state = Mock()
-        env.DISTRIBUTION = 'ubuntu'
-        env.SERIES = 'wily'
-        env.SOURCES = 'foo, bar, baz'
-        env.SYNC_REQUEST = 'ppa:some/ubuntu/place,vivid'
-        self.script.prepare_sync(silo_state)
-        self.script.lp.get_ppa.assert_called_once_with('some/ubuntu/place')
-        self.assertEqual(silo_state.source_series, 'vivid')
-        self.assertEqual(
-            silo_state.source_archive,
-            self.script.lp.get_ppa.return_value.self_link)
-
-    def test_prepare_sync_silo(self):
-        """Configure syncs correctly from a silo."""
-        silo_state = Mock()
-        env.DISTRIBUTION = 'ubuntu'
-        env.SERIES = 'wily'
-        env.SOURCES = 'foo, bar, baz'
-        env.SYNC_REQUEST = '8'
-        self.script.SiloState.return_value.series.name = 'utopic'
-        self.script.prepare_sync(silo_state)
-        self.script.lp.get_ppa.assert_called_once_with(
-            'ci-train-staging-area/ubuntu/landing-008')
-        self.assertEqual(silo_state.source_series, 'utopic')
-        self.assertEqual(
-            silo_state.source_archive,
-            self.script.lp.get_ppa.return_value.self_link)
-
-    def test_prepare_sync_silo_sources(self):
-        """Configure syncs correctly from a silo."""
-        silo_state = Mock()
-        env.DISTRIBUTION = 'ubuntu'
-        env.SERIES = 'wily'
-        env.SOURCES = ''
-        env.SYNC_REQUEST = '8'
-        self.script.SiloState.return_value.series.name = 'vivid'
-        self.script.prepare_sync(silo_state)
-        self.script.lp.get_ppa.assert_called_once_with(
-            'ci-train-staging-area/ubuntu/landing-008')
-        self.assertEqual(silo_state.source_series, 'vivid')
-        self.assertEqual(
-            silo_state.source_archive,
-            self.script.lp.get_ppa.return_value.self_link)
-
-    def test_prepare_sync_overlay(self):
-        """Configure syncs correctly from the overlay."""
-        silo_state = Mock()
-        env.DISTRIBUTION = 'ubuntu'
-        env.SERIES = 'wily'
-        env.SOURCES = 'foo, bar, baz'
-        env.SYNC_REQUEST = 'stable-overlay'
-        self.script.prepare_sync(silo_state)
-        self.script.lp.get_ppa.assert_called_once_with(STABLE_OVERLAY_PPA)
-        self.assertEqual(silo_state.source_series, 'wily')
-        self.assertEqual(
-            silo_state.source_archive,
-            self.script.lp.get_ppa.return_value.self_link)
-
-    def test_prepare_sync_main(self):
-        """Configure syncs correctly from the main archive."""
-        silo_state = Mock()
-        env.DISTRIBUTION = 'ubuntu'
-        env.SERIES = 'wily'
-        env.SOURCES = 'foo, bar, baz'
-        env.SYNC_REQUEST = 'ubuntu,vivid'
-        ubuntu = Mock()
-        self.script.lp.distributions = dict(ubuntu=ubuntu)
-        self.script.prepare_sync(silo_state)
-        self.assertEqual(silo_state.source_series, 'vivid')
-        self.assertEqual(
-            silo_state.source_archive, ubuntu.main_archive.self_link)
 
     def test_check_merge_status(self):
         """Ensure that merges are in acceptable states."""

=== modified file 'tests/unit/test_silomanager.py'
--- tests/unit/test_silomanager.py	2015-09-05 20:18:00 +0000
+++ tests/unit/test_silomanager.py	2015-09-06 00:07:30 +0000
@@ -86,15 +86,74 @@
         silomanager.SILO_NAME_LIST = dict(
             ubuntu=['ubuntu/landing-00{}'.format(x) for x in range(0, 10)])
 
+    def test_silomanager_refuse_SILO_DIR(self):
+        """Prevent badness from baddening."""
+        self.assertFalse(
+            hasattr(silomanager, 'SILO_DIR'),
+            'SILO_DIR() cannot be trusted inside SiloState class.')
+
     def test_silostate_requestid(self):
         """Ensure requestid property behaves sensible."""
-        func = 'cupstream2distro.silomanager.SILO_DIR'
+        func = 'cupstream2distro.silomanager.join'
         with patch(func, lambda *ingore: self.tempdir + '/step'):
             self.state.requestid = '42'
             self.assertEqual(self.state._config['requestid'], '42')
             self.state._config['requestid'] = None
             self.assertEqual(self.state.requestid, '42')
 
+    @patch('cupstream2distro.silomanager.lp')
+    def test_silostate_source_archive_ppa(self, lp_mock):
+        """Ensure source_archive property has correct return values."""
+        lp_mock.get_ppa.return_value.distribution.name = 'ubuntu'
+        self.state._bileto.update(sync_request='ppa:team/ubuntu/ppa,vivid')
+        self.assertEqual(
+            self.state.source_archive, lp_mock.get_ppa.return_value)
+        lp_mock.get_ppa.assert_called_once_with('team/ubuntu/ppa')
+        self.assertEqual(self.state.source_series, lp_mock.load.return_value)
+        lp_mock.load.assert_called_once_with('ubuntu/vivid')
+
+    @patch('cupstream2distro.silomanager.lp')
+    @patch('cupstream2distro.silomanager.SiloState')
+    def test_silostate_source_archive_digit(self, ss_mock, lp_mock):
+        """Ensure source_archive property has correct return values."""
+        ss_mock.return_value.series = 'zip'
+        self.state._bileto.update(sync_request='12', distribution='blah')
+        self.assertEqual(
+            self.state.source_archive, lp_mock.get_ppa.return_value)
+        lp_mock.get_ppa.assert_called_once_with(
+            'ci-train-staging-area/blah/landing-012')
+        self.assertEqual(self.state.source_series, 'zip')
+
+    @patch('cupstream2distro.silomanager.lp')
+    def test_silostate_source_archive_stable(self, lp_mock):
+        """Ensure source_archive property has correct return values."""
+        lp_mock.get_ppa.return_value.distribution.name = 'ubuntu'
+        self.state._bileto.update(sync_request='stable-overlay,vivid')
+        self.assertEqual(
+            self.state.source_archive, lp_mock.get_ppa.return_value)
+        lp_mock.get_ppa.assert_called_once_with(
+            'ci-train-staging-area/ubuntu/stable-phone-overlay')
+        self.assertEqual(self.state.source_series, lp_mock.load.return_value)
+        lp_mock.load.assert_called_once_with('ubuntu/vivid')
+
+    @patch('cupstream2distro.silomanager.lp')
+    def test_silostate_source_archive_other(self, lp_mock):
+        """Ensure source_archive property has correct return values."""
+        rtm = Mock()
+        lp_mock.distributions = {'ubuntu-rtm': rtm}
+        rtm.main_archive.distribution.name = 'ubuntu-rtm'
+        self.state._bileto.update(sync_request='ubuntu-rtm,14.09')
+        self.assertEqual(self.state.source_archive, rtm.main_archive)
+        self.assertEqual(self.state.source_series, lp_mock.load.return_value)
+        lp_mock.load.assert_called_once_with('ubuntu-rtm/14.09')
+
+    @patch('cupstream2distro.silomanager.lp')
+    def test_silostate_source_archive_none(self, lp_mock):
+        """Ensure source_archive property has correct return values."""
+        self.state._bileto.update(sync_request='')
+        self.assertIsNone(self.state.source_archive)
+        self.assertIsNone(self.state.source_series)
+
     def test_mark_packages_dirty(self):
         """Create dirty package marker files."""
         names = ['bleep', 'blorp']
@@ -250,7 +309,7 @@
 
     def test_silostate_new_blank(self):
         """Create a new SiloState instance for a fresh silo assignment."""
-        methodname = 'cupstream2distro.silomanager.SILO_DIR'
+        methodname = 'cupstream2distro.silomanager.join'
         with patch(methodname, lambda *ignore: self.tempdir + '/request_id'):
             silo_state = SiloState.new_blank(
                 siloname='ubuntu/landing-789',
@@ -620,7 +679,7 @@
 
     def test_silostate_step_setters(self):
         """Set various silo steps."""
-        func = 'cupstream2distro.silomanager.SILO_DIR'
+        func = 'cupstream2distro.silomanager.join'
         with patch(func, lambda *ingore: self.tempdir + '/step'):
             self.state.set_empty()
             self.assertEqual(self.state._config['global']['step'], 0)
@@ -686,14 +745,10 @@
         """Set various state properties."""
         self.state.mps = 'flibbyjib'
         self.assertEqual(self.state._config['mps'], 'flibbyjib')
-        self.state.source_archive = 'ubuntu'
-        self.assertEqual(self.state._config['source_archive'], 'ubuntu')
-        self.state.source_series = 'vivid'
-        self.assertEqual(self.state._config['source_series'], 'vivid')
 
     def test_silostate_set_ready(self):
         """Set ready to build status."""
-        func = 'cupstream2distro.silomanager.SILO_DIR'
+        func = 'cupstream2distro.silomanager.join'
         with patch(func, lambda *ingore: self.tempdir + '/step'):
             status_mock = self.state.set_config_status = Mock()
             self.state.set_ready()
@@ -702,7 +757,7 @@
 
     def test_silostate_set_reconfigure_failed(self):
         """Set reconfigure failed status."""
-        func = 'cupstream2distro.silomanager.SILO_DIR'
+        func = 'cupstream2distro.silomanager.join'
         with patch(func, lambda *ingore: self.tempdir + '/step'):
             status_mock = self.state.set_config_status = Mock()
             self.state.set_reconfigure_failed('You did a bad thing.')
@@ -742,7 +797,7 @@
         silomanager.save_error_message('Whoa, errors galore!')
         silostate.assert_called_once_with('my/fake/silo')
         silostate().set_config_status.assert_called_once_with(
-            -1, 'Whoa, errors galore!')
+            'Whoa, errors galore!')
         silostate().save_config.assert_called_once_with()
 
     @patch('cupstream2distro.silomanager.SiloState')
@@ -778,5 +833,5 @@
         silomanager.cleanup_on_exception(None, Exception('It broke'), None)
         silostate.assert_called_once_with('ubuntu/landing-negative-one')
         silostate().set_config_status.assert_called_once_with(
-            -1, 'Uncaught exception: Exception: It broke')
+            'Uncaught exception: Exception: It broke')
         silostate().save_config.assert_called_once_with()