Merge lp:~cjwatson/launchpad/init-branch-livefses into lp:launchpad

[Colin Watson]

Copy live filesystems from the previous series on initialisation.

Copy live filesystems from the previous series on initialisation.

To avoid zombie livefses hanging around forever, this only copies those that have had builds created for them in the last 30 days, so at worst they should exist for one series longer than their true lifetime.

[Colin Watson (cjwatson)]

We talked about this in the LP team meeting a couple of weeks ago. We might want to copy "official" livefses (mostly those owned by ~ubuntu-cdimage) but not other ones, because that isn't the kind of change LP would normally apply to user-owned artifacts. The 30-day limit is too much of a hack.

If somebody wants to tackle this in future, the place to start would be by introducing a database-backed notion of whether livefses are official. For now it's not too bad to keep this client-side.

review: Disapprove

Unmerged revisions

17814. By Colin Watson on 2015-10-13

Copy live filesystems from the previous series on initialisation.

=== modified file 'lib/lp/soyuz/scripts/initialize_distroseries.py'
--- lib/lp/soyuz/scripts/initialize_distroseries.py	2015-05-13 12:40:44 +0000
+++ lib/lp/soyuz/scripts/initialize_distroseries.py	2015-10-13 15:40:46 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2014 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2015 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Initialize a distroseries from its parent distroseries."""
@@ -21,6 +21,7 @@
 from lp.registry.interfaces.pocket import PackagePublishingPocket
 from lp.registry.model.distroseries import DistroSeries
 from lp.services.database import bulk
+from lp.services.database.constants import THIRTY_DAYS_AGO
 from lp.services.database.interfaces import IMasterStore
 from lp.services.database.sqlbase import sqlvalues
 from lp.services.helpers import ensure_unicode
@@ -38,6 +39,7 @@
 from lp.soyuz.interfaces.distributionjob import (
     IDistroSeriesDifferenceJobSource,
     )
+from lp.soyuz.interfaces.livefs import ILiveFSSet
 from lp.soyuz.interfaces.packagecloner import IPackageCloner
 from lp.soyuz.interfaces.packageset import (
     IPackagesetSet,
@@ -45,6 +47,8 @@
     )
 from lp.soyuz.interfaces.queue import IPackageUploadSet
 from lp.soyuz.model.binarypackagebuild import COPY_ARCHIVE_SCORE_PENALTY
+from lp.soyuz.model.livefs import LiveFS
+from lp.soyuz.model.livefsbuild import LiveFSBuild
 from lp.soyuz.model.packageset import Packageset
 from lp.soyuz.scripts.packagecopier import do_copy
 
@@ -289,6 +293,7 @@
         self._copy_pocket_permissions()
         self._create_dsds()
         self._set_initialized()
+        self._copy_livefses()
         transaction.commit()
 
     def _set_parents(self):
@@ -737,3 +742,28 @@
                     """ % sqlvalues(
                         self.distroseries.main_archive, self.distroseries.id,
                         parent.id))
+
+    def _copy_livefses(self):
+        """Copy live filesystem configurations from the parent series."""
+        # We could probably be more efficient here, but the number of
+        # livefses is likely to be small.
+        livefs_set = getUtility(ILiveFSSet)
+        for parent in self.derivation_parents:
+            livefses = self._store.find(
+                LiveFS,
+                LiveFS.distro_series == parent,
+                LiveFSBuild.livefs == LiveFS.id,
+                LiveFSBuild.date_created >= THIRTY_DAYS_AGO)
+            for livefs in livefses:
+                if not livefs_set.exists(
+                        livefs.owner, self.distroseries, livefs.name):
+                    self._store.execute("""
+                        INSERT INTO LiveFS (
+                            registrant, owner, distro_series, name, json_data,
+                            require_virtualized)
+                        SELECT
+                            registrant, owner, %s, name, json_data,
+                            require_virtualized
+                        FROM LiveFS AS parent_livefs
+                        WHERE parent_livefs.id = %s
+                        """ % sqlvalues(self.distroseries.id, livefs.id))

=== modified file 'lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py'
--- lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py	2015-04-20 15:59:52 +0000
+++ lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py	2015-10-13 15:40:46 +0000
@@ -5,6 +5,13 @@
 
 __metaclass__ = type
 
+from datetime import (
+    datetime,
+    timedelta,
+    )
+
+import pytz
+from testtools.matchers import MatchesStructure
 import transaction
 from zope.component import getUtility
 
@@ -19,7 +26,9 @@
     )
 from lp.registry.interfaces.distroseriesparent import IDistroSeriesParentSet
 from lp.registry.interfaces.pocket import PackagePublishingPocket
+from lp.services.database.constants import ONE_DAY_AGO
 from lp.services.database.interfaces import IStore
+from lp.services.features.testing import FeatureFixture
 from lp.soyuz.enums import (
     ArchivePurpose,
     PackageUploadStatus,
@@ -28,6 +37,10 @@
 from lp.soyuz.interfaces.archivepermission import IArchivePermissionSet
 from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
 from lp.soyuz.interfaces.component import IComponentSet
+from lp.soyuz.interfaces.livefs import (
+    ILiveFSSet,
+    LIVEFS_FEATURE_FLAG,
+    )
 from lp.soyuz.interfaces.packageset import (
     IPackagesetSet,
     NoSuchPackageSet,
@@ -1632,3 +1645,86 @@
             ("The selected architecture independent architecture tag is not "
              "among the selected architectures."),
             ids.check)
+
+    def test_copying_livefses(self):
+        # If a parent series has live filesystems, we should copy them.
+        self.parent, self.parent_das = self.setupParent()
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            livefs1 = self.factory.makeLiveFS(distroseries=self.parent)
+            self.factory.makeLiveFSBuild(
+                livefs=livefs1, date_created=ONE_DAY_AGO)
+            livefs2 = self.factory.makeLiveFS(
+                distroseries=self.parent,
+                metadata={'project': 'ubuntu-core', 'image_format': 'plain'},
+                require_virtualized=False)
+            self.factory.makeLiveFSBuild(
+                livefs=livefs2, date_created=ONE_DAY_AGO)
+        child = self._fullInitialize([self.parent])
+        child_livefs1 = getUtility(ILiveFSSet).getByName(
+            livefs1.owner, child, livefs1.name)
+        self.assertThat(child_livefs1, MatchesStructure.byEquality(
+            registrant=livefs1.registrant, owner=livefs1.owner,
+            distro_series=child, name=livefs1.name, metadata=livefs1.metadata,
+            require_virtualized=livefs1.require_virtualized))
+        child_livefs2 = getUtility(ILiveFSSet).getByName(
+            livefs2.owner, child, livefs2.name)
+        self.assertThat(child_livefs2, MatchesStructure.byEquality(
+            registrant=livefs2.registrant, owner=livefs2.owner,
+            distro_series=child, name=livefs2.name, metadata=livefs2.metadata,
+            require_virtualized=livefs2.require_virtualized))
+
+    def test_copying_livefses_no_duplication(self):
+        # Copying live filesystems only copies them from the most recent
+        # series, rather than merging them from all series.
+        previous_parent, _ = self.setupParent()
+        parent = self._fullInitialize([previous_parent])
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            parent_livefs1 = self.factory.makeLiveFS(
+                distroseries=parent, metadata={'marker': 'parent'})
+            self.factory.makeLiveFSBuild(
+                livefs=parent_livefs1, date_created=ONE_DAY_AGO)
+            parent_livefs2 = self.factory.makeLiveFS(distroseries=parent)
+            self.factory.makeLiveFSBuild(
+                livefs=parent_livefs2, date_created=ONE_DAY_AGO)
+        child = self._fullInitialize(
+            [previous_parent], previous_series=parent,
+            distribution=parent.distribution)
+        child_livefs1 = getUtility(ILiveFSSet).getByName(
+            parent_livefs1.owner, child, parent_livefs1.name)
+        child_livefs1.metadata = {'marker': 'child'}
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            self.factory.makeLiveFSBuild(
+                livefs=child_livefs1, date_created=ONE_DAY_AGO)
+        child_livefs2 = getUtility(ILiveFSSet).getByName(
+            parent_livefs2.owner, child, parent_livefs2.name)
+        child_livefs2.destroySelf()
+        grandchild = self._fullInitialize(
+            [previous_parent], previous_series=child,
+            distribution=parent.distribution)
+        grandchild_livefs1 = getUtility(ILiveFSSet).getByName(
+            child_livefs1.owner, grandchild, child_livefs1.name)
+        self.assertEqual({'marker': 'child'}, grandchild_livefs1.metadata)
+        self.assertFalse(
+            getUtility(ILiveFSSet).exists(
+                parent_livefs2.owner, grandchild, parent_livefs2.name))
+
+    def test_copying_livefses_skips_dormant(self):
+        # Live filesystems that have not been built in the last 30 days are
+        # not copied.
+        self.parent, self.parent_das = self.setupParent()
+        with FeatureFixture({LIVEFS_FEATURE_FLAG: u'on'}):
+            livefs1 = self.factory.makeLiveFS(distroseries=self.parent)
+            for days in 31, 30, 29:
+                self.factory.makeLiveFSBuild(
+                    livefs=livefs1,
+                    date_created=datetime.now(pytz.UTC) - timedelta(days=days))
+            livefs2 = self.factory.makeLiveFS(distroseries=self.parent)
+            for days in 33, 32, 31:
+                self.factory.makeLiveFSBuild(
+                    livefs=livefs2,
+                    date_created=datetime.now(pytz.UTC) - timedelta(days=days))
+        child = self._fullInitialize([self.parent])
+        self.assertTrue(
+            getUtility(ILiveFSSet).exists(livefs1.owner, child, livefs1.name))
+        self.assertFalse(
+            getUtility(ILiveFSSet).exists(livefs2.owner, child, livefs2.name))