Merge lp:~robru/cupstream2distro/parallelize-migration into lp:cupstream2distro

[Robert Bruce Park]

Parallelize migration script.

[Robert Bruce Park (robru)]

https://ci-train.staging.ubuntu.com/view/0.%20Status/

I'm enjoying this in staging, just need to write tests.

[lp:~robru/cupstream2distro/parallelize-migration]

1232. By Robert Bruce Park on 2015-11-20

Run with 10 minute frequency.

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:1232
http://jenkins.qa.ubuntu.com/job/cu2d-choo-choo-ci/913/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/cu2d-choo-choo-ci/913/rebuild

review: Approve (continuous-integration)

[lp:~robru/cupstream2distro/parallelize-migration]

1233. By Robert Bruce Park on 2015-11-20

Keep more logs but fewer artifacts.

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:1233
http://jenkins.qa.ubuntu.com/job/cu2d-choo-choo-ci/914/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins.ubuntu-ci:8080/job/cu2d-choo-choo-ci/914/rebuild

review: Approve (continuous-integration)

[Robert Bruce Park (robru)]

This loks amazing in staging.

review: Approve

=== renamed file 'citrain/jenkins-templates/check-publication-migration.xml.tmpl' => 'citrain/jenkins-templates/status.xml.tmpl'
--- citrain/jenkins-templates/check-publication-migration.xml.tmpl	2015-11-18 05:51:13 +0000
+++ citrain/jenkins-templates/status.xml.tmpl	2015-11-20 11:31:16 +0000
@@ -1,25 +1,25 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <project>
   <actions/>
-  <description>Automatically check destination migration for packages.</description>
+  <description>Regularly refresh silo status.</description>
   <logRotator>
-    <daysToKeep>1</daysToKeep>
-    <numToKeep>500</numToKeep>
+    <daysToKeep>7</daysToKeep>
+    <numToKeep>-1</numToKeep>
     <artifactDaysToKeep>-1</artifactDaysToKeep>
-    <artifactNumToKeep>-1</artifactNumToKeep>
+    <artifactNumToKeep>5</artifactNumToKeep>
   </logRotator>
   <keepDependencies>false</keepDependencies>
   <properties>
     <hudson.security.AuthorizationMatrixProperty>
       <permission>hudson.model.Item.Cancel:ubuntu-core-dev</permission>
       <permission>hudson.model.Item.Cancel:canonical-ci-eng</permission>
-      <permission>hudson.model.Item.Cancel:ci-train-users</permission>
+      <permission>hudson.model.Item.Cancel:ci-train-ppa-service</permission>
       <permission>hudson.model.Item.Build:ubuntu-core-dev</permission>
       <permission>hudson.model.Item.Build:canonical-ci-eng</permission>
-      <permission>hudson.model.Item.Build:ci-train-users</permission>
+      <permission>hudson.model.Item.Build:ci-train-ppa-service</permission>
       <permission>hudson.model.Item.Read:ubuntu-core-dev</permission>
       <permission>hudson.model.Item.Read:canonical-ci-eng</permission>
-      <permission>hudson.model.Item.Read:ci-train-users</permission>
+      <permission>hudson.model.Item.Read:ci-train-ppa-service</permission>
     </hudson.security.AuthorizationMatrixProperty>
     <hudson.model.ParametersDefinitionProperty>
       <parameterDefinitions>
@@ -38,7 +38,7 @@
   <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
   <triggers>
     <hudson.triggers.TimerTrigger>
-      <spec>H/30 * * * *</spec>
+      <spec>H/10 * * * *</spec>
     </hudson.triggers.TimerTrigger>
   </triggers>
   <concurrentBuild>false</concurrentBuild>
@@ -46,15 +46,35 @@
     <hudson.tasks.Shell>
       <command>#!/bin/sh
 export LANG=en_US.UTF-8
+export WORKSPACE="$PWD"
+export SILONAME="{SILO_NAME}"
 export PYTHONPATH={LIBDIR}
 
-cd # go to home directory
-
-{BINDIR}/migration.py
+[ "$DEBUG" = "true" ] &amp;&amp; set -x
+
+[ -e {SILOS_DIR}/{SILO_NAME}/request_id_* ] || exit 0
+
+rm -rf $WORKSPACE/*
+
+cd {SILOS_DIR}/{SILO_NAME}
+
+{BINDIR}/status.py
+RETVAL=$?
+
+cp *.diff "$WORKSPACE" 2&gt;/dev/null || true
+
+exit $RETVAL
 </command>
     </hudson.tasks.Shell>
   </builders>
   <publishers>
+    <hudson.tasks.ArtifactArchiver>
+      <artifacts>*.diff</artifacts>
+      <allowEmptyArchive>true</allowEmptyArchive>
+      <onlyIfSuccessful>false</onlyIfSuccessful>
+      <fingerprint>false</fingerprint>
+      <defaultExcludes>true</defaultExcludes>
+    </hudson.tasks.ArtifactArchiver>
   </publishers>
   <buildWrappers/>
 </project>
 
=== modified file 'citrain/setup_citrain.py'
--- citrain/setup_citrain.py	2015-11-04 16:32:54 +0000
+++ citrain/setup_citrain.py	2015-11-20 11:31:16 +0000
@@ -101,7 +101,12 @@
     """Setup a silo jenkins job."""
     context = dict(SILOS_DIR=SILOS_DIR, SILO_NAME=siloname)
     siloname = siloname.replace('/', '-')
-    steps = dict(build='1', publish='2', autopkgtests='2.5', merge_clean='3')
+    steps = dict(
+        status='0',
+        build='1',
+        publish='2',
+        autopkgtests='2.5',
+        merge_clean='3')
     for job, i in sorted(steps.items()):
         job = job.replace('_', '-')
         setup_job('{}-{}-{}'.format(siloname, i, job), template(job), context)
@@ -114,7 +119,6 @@
         setup_silo(siloname)
     setup_job('cyphermox-test')
     setup_job('prepare-silo')
-    setup_job('check-publication-migration')
     setup_job('staleness-report')
     setup_job('upgrade-chroot')
     setup_job('apt-get-clean')
 
=== renamed file 'citrain/migration.py' => 'citrain/status.py'
--- citrain/migration.py	2015-11-19 23:08:21 +0000
+++ citrain/status.py	2015-11-20 11:31:16 +0000
@@ -15,9 +15,9 @@
 # this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-"""CI Train Migration Script
+"""CI Train Status Script
 
-Check if all silo packages have migrated, and if so, trigger merge & clean.
+Update silo status, also triggering Merge&Clean if silo is ready.
 
 Environment variables:
 
@@ -30,36 +30,22 @@
 
 from citrain.recipes.base import BuildBase
 from citrain.recipes.manager import Manager
-from cupstream2distro.utils import env, run_script
-from cupstream2distro.silomanager import SiloState, stock_main
+from cupstream2distro.utils import run_script
+from cupstream2distro.silomanager import stock_main
 from citrain.merge_clean import merge
 
 SUCCESS = re.compile(r'^((Release|Updates) pocket( \([^()]+\). ?)?)+$')
 
 
-def main():
-    """Execute the migration check, logging & saving any errors.
-
-    :returns: 0. Always.
-    """
-    for silo_state in SiloState.iterate():
-        env.SILONAME = silo_state
-        BuildBase.failures.clear()
-        logging.info('\n\nInspecting silo %s:', silo_state.ppa.web_link)
-        try:
-            silo_state.enforce_lock()
-            silo_state.lock_fd.close()
-        except BlockingIOError:
-            logging.info('Silo %s is in use, skipping.', env.SILONAME)
-            continue
-
-        status = silo_state.status = Manager(silo_state).get_states()
-        if SUCCESS.match(status):
-            logging.info('Looks good, proceeding with merge & clean.')
-            stock_main(merge)
-            continue
-        silo_state.save_config()
-    return 0
-
-
-run_script(__name__, __doc__, main)
+def update_status(silo_state):
+    """Set the status for this silo."""
+    silo_state.lock_fd.close()
+    BuildBase.failures.clear()
+    logging.info('Inspecting PPA %s:', silo_state.ppa.web_link)
+    status = silo_state.status = Manager(silo_state).get_states()
+    if SUCCESS.match(status):
+        logging.info('Looks good, proceeding with merge & clean.')
+        stock_main(merge)
+
+
+run_script(__name__, __doc__, lambda: stock_main(update_status))
 
=== modified file 'files/config.xml'
--- files/config.xml	2015-10-28 23:28:38 +0000
+++ files/config.xml	2015-11-20 11:31:16 +0000
@@ -202,6 +202,28 @@
     </hudson.model.AllView>
     <listView>
       <owner class="hudson" reference="../../.."/>
+      <name>0. Status</name>
+      <filterExecutors>false</filterExecutors>
+      <filterQueue>false</filterQueue>
+      <properties class="hudson.model.View$PropertyList"/>
+      <jobNames>
+        <comparator class="hudson.util.CaseInsensitiveComparator"/>
+      </jobNames>
+      <jobFilters/>
+      <columns>
+        <hudson.views.StatusColumn/>
+        <hudson.views.WeatherColumn/>
+        <hudson.views.JobColumn/>
+        <hudson.views.LastSuccessColumn/>
+        <hudson.views.LastFailureColumn/>
+        <hudson.views.LastDurationColumn/>
+        <hudson.views.BuildButtonColumn/>
+      </columns>
+      <includeRegex>.*-0-status</includeRegex>
+      <recurse>false</recurse>
+    </listView>
+    <listView>
+      <owner class="hudson" reference="../../.."/>
       <name>1. Build</name>
       <filterExecutors>false</filterExecutors>
       <filterQueue>false</filterQueue>
 
=== modified file 'tests/unit/test_script_setup_citrain.py'
--- tests/unit/test_script_setup_citrain.py	2015-11-04 16:32:54 +0000
+++ tests/unit/test_script_setup_citrain.py	2015-11-20 11:31:16 +0000
@@ -123,6 +123,10 @@
                      'publish.xml.tmpl',
                      {'SILO_NAME': 'ubuntu/landing-000',
                       'SILOS_DIR': os.path.expanduser('~/silos')}),
+                call('ubuntu-landing-000-0-status',
+                     'status.xml.tmpl',
+                     {'SILO_NAME': 'ubuntu/landing-000',
+                      'SILOS_DIR': os.path.expanduser('~/silos')}),
             ])
 
     def test_main(self):
@@ -137,7 +141,6 @@
         self.assertEqual(self.script.setup_job.mock_calls, [
             call('cyphermox-test'),
             call('prepare-silo'),
-            call('check-publication-migration'),
             call('staleness-report'),
             call('upgrade-chroot'),
             call('apt-get-clean'),
@@ -152,11 +155,11 @@
             'abandon/config.xml',
             'apt-get-clean/config.xml',
             'cyphermox-test/config.xml',
-            'check-publication-migration/config.xml',
             'pbuilder-clean/config.xml',
             'prepare-silo/config.xml',
             'revert/config.xml',
             'staleness-report/config.xml',
+            'ubuntu-landing-{:03d}-0-status/config.xml',
             'ubuntu-landing-{:03d}-1-build/config.xml',
             'ubuntu-landing-{:03d}-2-publish/config.xml',
             'ubuntu-landing-{:03d}-2.5-autopkgtests/config.xml',
 
=== renamed file 'tests/unit/test_script_migration.py' => 'tests/unit/test_script_status.py'
--- tests/unit/test_script_migration.py	2015-11-19 23:08:21 +0000
+++ tests/unit/test_script_status.py	2015-11-20 11:31:16 +0000
@@ -14,31 +14,25 @@
 # this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-"""Tests for CI Train Migration script."""
+"""Tests for CI Train Status script."""
 
-from os.path import join
 from mock import Mock, call
 
 from tests.unit import CITrainScriptTestCase
 
-from cupstream2distro.settings import SILOS_DIR
-
 # Unused import necessary for code coverage reporting
-from citrain import migration
-COVERAGE = [migration]
-
-
-class MigrationTestCase(CITrainScriptTestCase):
-    """Tests for CI Train Migration script."""
-    scriptname = 'migration.py'
+from citrain import status
+COVERAGE = [status]
+
+
+class StatusTestCase(CITrainScriptTestCase):
+    """Tests for CI Train Status script."""
+    scriptname = 'status.py'
 
     def setUp(self):
         super().setUp()
         self.script.Manager = Mock()
         self.script.stock_main = Mock()
-        self.script.glob.return_value = [
-            join(SILOS_DIR, 'ubuntu', 'landing-00{}'.format(x))
-            for x in range(3)]
 
     def test_success_regex(self):
         """Ensure we can properly recognize silos ready to auto-merge."""
@@ -60,89 +54,34 @@
         results = [bool(self.script.SUCCESS.match(stat)) for stat in bad]
         self.assertEqual(results, [False] * len(bad))
 
-    def test_main(self):
+    def test_status(self):
         """Trigger merges after successful migrations."""
         tokenize = lambda: Mock(return_value={'hi'})
-        states = self.script.SiloState.iterate.return_value = [Mock(
-            tokenize=tokenize(),
-            is_published=True,
-            is_autopkgtesting=False) for rid in range(3)]
+        silo_state = Mock(tokenize=tokenize())
         mgr = self.script.Manager.return_value
         mgr.get_states.return_value = 'Release pocket (foo).'
-        self.assertEqual(self.script.main(), 0)
-        for state in states:
-            self.assertEqual(state.mock_calls, [
-                call.enforce_lock(),
-                call.lock_fd.close(),
-            ])
+        self.assertIsNone(self.script.update_status(silo_state))
+        self.assertEqual(silo_state.mock_calls, [
+            call.lock_fd.close(),
+        ])
         self.assertEqual(self.script.Manager.mock_calls, [
-            call(states[0]),
-            call().get_states(),
-            call(states[1]),
-            call().get_states(),
-            call(states[2]),
+            call(silo_state),
             call().get_states(),
         ])
         self.assertEqual(self.script.stock_main.mock_calls, [
             call(self.script.merge)
-        ] * 3)
-
-    def test_main_skip_empty_silos(self):
-        """Don't check silos that are empty."""
-        self.script.SiloState.iterate.return_value = []
-        self.assertEqual(self.script.main(), 0)
-        self.assertEqual(self.script.Manager.mock_calls, [])
-        self.assertEqual(self.script.stock_main.mock_calls, [])
-
-    def test_main_skip_locked_silos(self):
-        """Skip over silos that are currently used by other processes."""
-        states = self.script.SiloState.iterate.return_value = [Mock()]
-        states[0].enforce_lock.side_effect = BlockingIOError
-        self.assertEqual(self.script.main(), 0)
-        self.assertEqual(self.script.Manager.mock_calls, [])
-        self.assertEqual(self.script.stock_main.mock_calls, [])
-        self.assertEqual(states[0].mock_calls, [call.enforce_lock()])
-
-    def test_main_skip_unpublished_silos(self):
+        ])
+
+    def test_status_skip_unpublished_silos(self):
         """Don't merge silos that are unpublished."""
         tokenize = Mock(return_value={'hi'})
-        states = self.script.SiloState.iterate.return_value = [
-            Mock(tokenize=tokenize, is_published=False) for rid in range(3)]
-        mgr = self.script.Manager.return_value
-        mgr.get_states.return_value = 'Proposed pocket (foo).'
-        self.assertEqual(self.script.main(), 0)
-        self.assertEqual(self.script.Manager.mock_calls, [
-            call(states[0]),
-            call().get_states(),
-            call(states[1]),
-            call().get_states(),
-            call(states[2]),
-            call().get_states(),
-        ])
-        self.assertEqual(self.script.stock_main.mock_calls, [])
-        for state in states:
-            self.assertEqual(state.set_migrating.mock_calls, [])
-            self.assertEqual(state.save_config.mock_calls, [call()])
-
-    def test_main_dont_merge_migrating_silos(self):
-        """Don't merge silos that are still migrating."""
-        tokenize = Mock(return_value={'hi'})
-        states = self.script.SiloState.iterate.return_value = [Mock(
-            tokenize=tokenize,
-            is_published=True,
-            requestid=str(1),
-        )]
-        mgr = self.script.Manager.return_value
-        mgr.get_states.return_value = 'Proposed pocket (foo).'
-        self.assertEqual(self.script.main(), 0)
-        self.assertEqual(states[0].status, mgr.get_states.return_value)
-        self.assertEqual(states[0].mock_calls, [
-            call.enforce_lock(),
-            call.lock_fd.close(),
-            call.save_config(),
-        ])
-        self.assertEqual(self.script.Manager.mock_calls, [
-            call(states[0]),
-            call().get_states(),
-        ])
-        self.assertEqual(self.script.stock_main.mock_calls, [])
+        silo_state = Mock(tokenize=tokenize)
+        mgr = self.script.Manager.return_value
+        mgr.get_states.return_value = 'Proposed pocket (foo).'
+        self.assertIsNone(self.script.update_status(silo_state))
+        self.assertEqual(self.script.Manager.mock_calls, [
+            call(silo_state),
+            call().get_states(),
+        ])
+        self.assertEqual(self.script.stock_main.mock_calls, [])
+        self.assertEqual(silo_state.save_config.mock_calls, [])