Merge lp:~robru/cupstream2distro/auto-discover-manual-sources into lp:cupstream2distro

[Robert Bruce Park]

Jenkins will discover manual sources and fill out the bileto field.

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:1147
http://jenkins.qa.ubuntu.com/job/cu2d-choo-choo-ci/819/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/cu2d-choo-choo-ci/819/rebuild

review: Approve (continuous-integration)

[lp:~robru/cupstream2distro/auto-discover-manual-sources]

1148. By Robert Bruce Park on 2015-10-06

Get only Pending and Published sources.

1149. By Robert Bruce Park on 2015-10-06

Tweak.

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:1149
http://jenkins.qa.ubuntu.com/job/cu2d-choo-choo-ci/820/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/cu2d-choo-choo-ci/820/rebuild

review: Approve (continuous-integration)

[Robert Bruce Park (robru)]

Looking great in staging.

review: Approve

=== modified file 'citrain/recipes/manager.py'
--- citrain/recipes/manager.py	2015-09-24 06:49:54 +0000
+++ citrain/recipes/manager.py	2015-10-06 19:30:02 +0000
@@ -71,9 +71,10 @@
         include_binaries = env.INCLUDE_BINARIES_IN_SYNC == 'true'
         Sync = BinarySync if include_binaries else SourceSync
         Source = Sync if self.silo_state.source_archive else Manual
+        merges = set(self.silo_state.mps)
         self.types = {
-            Merge: self.names & set(self.silo_state.mps),
-            Source: self.names & set(self.silo_state.sources),
+            Merge: self.names & merges,
+            Source: self.names - merges,
         }
         if self.silo_state.source_archive:
             Sync.from_archive = self.silo_state.source_archive

=== modified file 'cupstream2distro/silomanager.py'
--- cupstream2distro/silomanager.py	2015-10-05 19:25:59 +0000
+++ cupstream2distro/silomanager.py	2015-10-06 19:30:02 +0000
@@ -340,7 +340,7 @@
         for field in MANDATORY:
             if not getattr(self, '_' + field, None):
                 raise PrepError('Field "{}" is mandatory.'.format(field))
-        if not (self._merge_proposals or self._sources or self._sync_request):
+        if not (self._merge_proposals or self.sources or self._sync_request):
             raise PrepError('Need either merges, sources, or a sync.')
         if self._distribution not in SILO_NAME_LIST:
             raise PrepError('{} is unsupported.'.format(self._distribution))
@@ -392,6 +392,7 @@
             request_id=self.requestid,
             status=self.status,
             siloname=self.siloname or self._siloname,
+            sources=' '.join(sorted(self.sources)),
         )
         bileto_comment(
             author=env.BUILD_USER_ID or 'ci-train-bot',
@@ -512,10 +513,16 @@
     def sources(self):
         """Identify the manual source packages configured for this silo."""
         archive = self.sync_request.partition(',')[0]
-        sources = splitter(self._sources)
+        sources = set(splitter(self._sources))
         if not sources and archive.isdigit():
             return self.load_source_sync_sources(int(archive))
-        return sources
+        inppa = {src.source_package_name
+                 for src in self.ppa.getPublishedSources(
+                     distro_series=self.series)
+                 if src.status.startswith('P')}
+        log_value_of.sources('Bileto says')
+        log_value_of.inppa('PPA has')
+        return sources | inppa
 
     @memoize
     def load_source_sync_sources(self, silonum):

=== modified file 'tests/unit/test_silomanager.py'
--- tests/unit/test_silomanager.py	2015-10-05 19:25:59 +0000
+++ tests/unit/test_silomanager.py	2015-10-06 19:30:02 +0000
@@ -33,6 +33,7 @@
 from cupstream2distro.errors import PrepError
 
 
+SOURCES = 'bar bat foo zing'.split()
 MPS = dict(
     foo=[Mock(self_link='https://api.launchpad.net/devel/foomp',
               web_link='https://launchpad.net/foomp')],
@@ -447,6 +448,7 @@
         with self.assertRaisesRegexp(PrepError, '"landers" is mandatory'):
             self.state.validate_bileto()
 
+    @patch('cupstream2distro.silomanager.SiloState.sources', [])
     def test_silostate_validate_what_to_do(self):
         """Raise PrepError when bileto is missing at least one thing to do."""
         self.state._merge_proposals = ''
@@ -481,6 +483,7 @@
         state.load_bileto()
         self.assertFalse(vb_mock.called)
 
+    @patch('cupstream2distro.silomanager.SiloState.sources', SOURCES)
     @patch('cupstream2distro.silomanager.SiloState.requestid', '121')
     @patch('cupstream2distro.silomanager.SiloState.dest', Mock())
     @patch('cupstream2distro.silomanager.SiloState.mps', MPS)
@@ -533,6 +536,7 @@
             call().lp_save(),
         ])
 
+    @patch('cupstream2distro.silomanager.SiloState.sources', [])
     @patch('cupstream2distro.silomanager.SiloState.requestid', '121')
     @patch('cupstream2distro.silomanager.BILETO_API', 'example.com')
     @patch('cupstream2distro.silomanager.COMMENT_API', 'comment.com')
@@ -558,6 +562,7 @@
     @patch('cupstream2distro.silomanager.SiloState.requestid', '121')
     @patch('cupstream2distro.silomanager.BILETO_API', 'example.com')
     @patch('cupstream2distro.silomanager.COMMENT_API', 'comment.com')
+    @patch('cupstream2distro.silomanager.SiloState.sources', [])
     @patch('cupstream2distro.silomanager.requests')
     @patch('cupstream2distro.silomanager.logging')
     def test_silostate_push_to_bileto_no_response(self, log_mock, req_mock):
@@ -569,6 +574,7 @@
             call('Failed to contact Bileto: token=XXX'),
         ])
 
+    @patch('cupstream2distro.silomanager.SiloState.sources', [])
     @patch('cupstream2distro.silomanager.SiloState.requestid', '121')
     @patch('cupstream2distro.silomanager.BILETO_API', 'example.com')
     @patch('cupstream2distro.silomanager.COMMENT_API', 'comment.com')
@@ -611,6 +617,20 @@
         self.assertEqual(self.state.sources, ss_mock.return_value.all_projects)
         ss_mock.assert_called_once_with('ubuntu/landing-005')
 
+    @patch('cupstream2distro.silomanager.SiloState.series', Mock())
+    @patch('cupstream2distro.silomanager.SiloState.ppa')
+    def test_silostate_sources_inppa(self, ppa_mock):
+        """Confirm we can grab sources from PPA."""
+        self.state._bileto.update(sources='preserved')
+        ppa_mock.getPublishedSources.return_value = [
+            Mock(source_package_name='bleep', status='Pending'),
+            Mock(source_package_name='blorp', status='Published'),
+            Mock(source_package_name='nawp', status='Deleted'),
+        ]
+        self.assertEqual(self.state.sources, {'preserved', 'bleep', 'blorp'})
+        ppa_mock.getPublishedSources.assert_called_once_with(
+            distro_series=self.state.series)
+
     def test_silostate_dual(self):
         """Ensure that dual property reflects correct series."""
         self.state._bileto.update(series='wily+vivid')
@@ -685,10 +705,10 @@
         self.assertEqual(
             self.state.published_versions, 'bar=5.0 foo=1.0,1.0~vivid')
 
+    @patch('cupstream2distro.silomanager.SiloState.sources', ['foobar'])
     @patch('cupstream2distro.silomanager.SiloState.mps', dict(merger=[]))
     def test_silostate_all_projects(self):
         """List all projects including MPs, source uploads, and syncs."""
-        self.state._bileto['sources'] = 'foobar'
         self.assertEqual(self.state.all_projects, ['foobar', 'merger'])
 
     def test_silostate_booleans(self):