Merge lp:~cjwatson/launchpad/ds-publish-by-hash into lp:launchpad

[Colin Watson]

Add and export DistroSeries.publish_by_hash and DistroSeries.advertise_by_hash.

Add and export DistroSeries.publish_by_hash and DistroSeries.advertise_by_hash. This won't do anything useful until later branches that implement it in archivepublisher, but I'm separating it out to make the branches a more manageable size.

[William Grant (wgrant)]

For testing, do we want a way to force by-hash to be published but leave the Release flag unset? AIUI one can force by-hash in sources.list.

review: Approve (code)

[Colin Watson (cjwatson)]

Makes sense. I've added advertise_by_hash now as well.

=== modified file 'lib/lp/registry/configure.zcml'
--- lib/lp/registry/configure.zcml	2016-02-28 19:13:17 +0000
+++ lib/lp/registry/configure.zcml	2016-03-18 15:41:25 +0000
@@ -318,6 +318,8 @@
                 backports_not_automatic
                 include_long_descriptions
                 index_compressors
+                publish_by_hash
+                advertise_by_hash
                 inherit_overrides_from_parents"/>
 
         <!-- NB: check with SABDFL before modifying these, there is potential to

=== modified file 'lib/lp/registry/interfaces/distroseries.py'
--- lib/lp/registry/interfaces/distroseries.py	2016-02-05 20:28:29 +0000
+++ lib/lp/registry/interfaces/distroseries.py	2016-03-18 15:41:25 +0000
@@ -387,6 +387,20 @@
             A list of compression types to use for published index files
             (Packages, Sources, etc.).""")))
 
+    publish_by_hash = exported(Bool(
+        title=_("Publish by-hash directories"), required=True,
+        description=_("""
+            Publish archive index files in by-hash directories so that apt
+            can retrieve them based on their hash, avoiding race conditions
+            between InRelease and other files during mirror updates.""")))
+
+    advertise_by_hash = exported(Bool(
+        title=_("Advertise by-hash directories"), required=True,
+        description=_("""
+            Advertise by-hash directories with a flag in the Release file so
+            that apt uses them by default.  Only effective if
+            publish_by_hash is also set.""")))
+
     inherit_overrides_from_parents = Bool(
         title=_("Inherit overrides from parents"),
         readonly=False, required=True)

=== modified file 'lib/lp/registry/model/distroseries.py'
--- lib/lp/registry/model/distroseries.py	2016-03-04 14:18:23 +0000
+++ lib/lp/registry/model/distroseries.py	2016-03-18 15:41:25 +0000
@@ -276,6 +276,8 @@
                 "index_compressors": [
                     compressor.title
                     for compressor in DEFAULT_INDEX_COMPRESSORS],
+                "publish_by_hash": False,
+                "advertise_by_hash": False,
                 }
         super(DistroSeries, self).__init__(*args, **kwargs)
 
@@ -841,6 +843,24 @@
         self.publishing_options["index_compressors"] = [
             compressor.title for compressor in value]
 
+    @property
+    def publish_by_hash(self):
+        return self.publishing_options.get("publish_by_hash", False)
+
+    @publish_by_hash.setter
+    def publish_by_hash(self, value):
+        assert isinstance(value, bool)
+        self.publishing_options["publish_by_hash"] = value
+
+    @property
+    def advertise_by_hash(self):
+        return self.publishing_options.get("advertise_by_hash", False)
+
+    @advertise_by_hash.setter
+    def advertise_by_hash(self, value):
+        assert isinstance(value, bool)
+        self.publishing_options["advertise_by_hash"] = value
+
     def _customizeSearchParams(self, search_params):
         """Customize `search_params` for this distribution series."""
         search_params.setDistroSeries(self)

=== modified file 'lib/lp/registry/stories/webservice/xx-distroseries.txt'
--- lib/lp/registry/stories/webservice/xx-distroseries.txt	2016-02-08 15:09:00 +0000
+++ lib/lp/registry/stories/webservice/xx-distroseries.txt	2016-03-18 15:41:25 +0000
@@ -60,6 +60,7 @@
     active: True
     active_milestones_collection_link:
         u'http://.../ubuntu/hoary/active_milestones'
+    advertise_by_hash: False
     all_milestones_collection_link: u'http://.../ubuntu/hoary/all_milestones'
     architectures_collection_link: u'http://.../ubuntu/hoary/architectures'
     bug_reported_acknowledgement: None
@@ -83,6 +84,7 @@
     official_bug_tags: []
     owner_link: u'http://.../~ubuntu-team'
     parent_series_link: u'http://.../ubuntu/warty'
+    publish_by_hash: False
     registrant_link: u'http://.../~mark'
     resource_type_link: ...
     self_link: u'http://.../ubuntu/hoary'

=== modified file 'lib/lp/registry/tests/test_distroseries.py'
--- lib/lp/registry/tests/test_distroseries.py	2016-02-05 20:28:29 +0000
+++ lib/lp/registry/tests/test_distroseries.py	2016-03-18 15:41:25 +0000
@@ -383,6 +383,26 @@
         self.assertEqual(
             ["xz"], naked_distroseries.publishing_options["index_compressors"])
 
+    def test_publish_by_hash(self):
+        distroseries = self.factory.makeDistroSeries()
+        self.assertFalse(distroseries.publish_by_hash)
+        with admin_logged_in():
+            distroseries.publish_by_hash = True
+        self.assertTrue(distroseries.publish_by_hash)
+        naked_distroseries = removeSecurityProxy(distroseries)
+        self.assertTrue(
+            naked_distroseries.publishing_options["publish_by_hash"])
+
+    def test_advertise_by_hash(self):
+        distroseries = self.factory.makeDistroSeries()
+        self.assertFalse(distroseries.advertise_by_hash)
+        with admin_logged_in():
+            distroseries.advertise_by_hash = True
+        self.assertTrue(distroseries.advertise_by_hash)
+        naked_distroseries = removeSecurityProxy(distroseries)
+        self.assertTrue(
+            naked_distroseries.publishing_options["advertise_by_hash"])
+
 
 class TestDistroSeriesPackaging(TestCaseWithFactory):
 

=== modified file 'lib/lp/soyuz/scripts/initialize_distroseries.py'
--- lib/lp/soyuz/scripts/initialize_distroseries.py	2015-05-13 12:40:44 +0000
+++ lib/lp/soyuz/scripts/initialize_distroseries.py	2016-03-18 15:41:25 +0000
@@ -375,6 +375,12 @@
         self.distroseries.include_long_descriptions = any(
             parent.include_long_descriptions
                 for parent in self.derivation_parents)
+        self.distroseries.publish_by_hash = any(
+            parent.publish_by_hash
+                for parent in self.derivation_parents)
+        self.distroseries.advertise_by_hash = any(
+            parent.advertise_by_hash
+                for parent in self.derivation_parents)
 
     def _copy_architectures(self):
         das_filter = ' AND distroseries IN %s ' % (

=== modified file 'lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py'
--- lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py	2015-04-20 15:59:52 +0000
+++ lib/lp/soyuz/scripts/tests/test_initialize_distroseries.py	2016-03-18 15:41:25 +0000
@@ -83,6 +83,8 @@
             spfss_utility.add(parent, format_selection)
         parent.backports_not_automatic = True
         parent.include_long_descriptions = False
+        parent.publish_by_hash = True
+        parent.advertise_by_hash = True
         self._populate_parent(parent, parent_das, packages, pocket)
         return parent, parent_das
 
@@ -610,6 +612,8 @@
         # Other configuration bits are copied too.
         self.assertTrue(child.backports_not_automatic)
         self.assertFalse(child.include_long_descriptions)
+        self.assertTrue(child.publish_by_hash)
+        self.assertTrue(child.advertise_by_hash)
 
     def test_initialize(self):
         # Test a full initialize with no errors.