Merge lp:~laney/ubuntu-archive-publishing/updates into lp:ubuntu-archive-publishing

[Iain Lane]

Run for -updates. I'm not really sure of all the consequences here, maybe consider this as a basis for discussion initially.

[Dimitri John Ledkov (xnox)]

I support this change. How to best test this?

Unmerged revisions

116. By Iain Lane on 2021-05-13

generate_extra_overrides: Germinate for stable releases too

Currently we only generate for the release pocket of -devel releases.
This means that Task headers don't get updated for SRUs meaning that,
for example, it's not easily possible to add new packages to the live
seed (other seeds are OK because they get pulled in by Depends of the
metapackage).

Germinate for -updates, and for stable releases. When generating
overrides for stable releases, write overrides to -updates.

This is not a full solution. The immutable nature of release pockets
means that some things can't work properly, for example

  - dependencies of the new package which aren't in -updates won't get
    Task entries
  - *dropping* things from seeds can't work properly, at least not
    without care from consuming code to have -updates dominate release;
    a simple grep-dctrl will not do

=== modified file 'lib/scripts/generate_extra_overrides.py'
--- lib/scripts/generate_extra_overrides.py	2021-05-13 12:49:01 +0000
+++ lib/scripts/generate_extra_overrides.py	2021-05-13 12:49:01 +0000
@@ -82,7 +82,12 @@
     return [
         series for series in distribution.series
         if series.status in (
-            "Experimental", "Active Development", "Pre-release Freeze")]
+            "Current Stable Release",
+            "Supported",
+            "Experimental",
+            "Active Development",
+            "Pre-release Freeze"
+        )]
 
 
 class GenerateExtraOverrides(ScriptBase):
@@ -346,8 +351,8 @@
         if "build-essential" in structure.names and primary_flavour:
             write_overrides("build-essential", "Build-Essential", "yes")
 
-    def germinateArch(self, series_name, components, arch, flavours,
-                      structures):
+    def germinateArch(self, series_name, series_is_stable, components, arch,
+                      flavours, structures):
         """Germinate seeds on all flavours for a single architecture."""
         # Buffer log output for each architecture so that it appears
         # sequential.
@@ -358,8 +363,11 @@
         germinator = Germinator(arch)
 
         # Read archive metadata.
+        series = [series_name]
+        if series_is_stable:
+            series += [series_name + "-updates"]
         archive = TagFile(
-            series_name, components, arch, "file://%s" % self.archiveroot,
+            series, components, arch, "file://%s" % self.archiveroot,
             cleanup=True)
         germinator.parse_archive(archive)
 
@@ -405,8 +413,12 @@
             if os.path.basename(output) not in seed_outputs:
                 os.remove(output)
 
-    def generateExtraOverrides(self, series_name, components, architectures,
+    def generateExtraOverrides(self, series, components, architectures,
                                flavours, seed_bases=None):
+        series_name = series.name
+        series_is_stable = series.status in ("Current Stable Release",
+                                             "Supported")
+        suffix = series_name + "-updates" if series_is_stable else series_name
         structures = self.makeSeedStructures(
             series_name, flavours, seed_bases=seed_bases)
 
@@ -420,7 +432,8 @@
                 if pid == 0:  # child
                     os._exit(self.germinateArchChild(
                         close_in_child, wfd,
-                        series_name, components, arch, flavours, structures))
+                        series_name, series_is_stable, components, arch,
+                        flavours, structures))
                 else:  # parent
                     os.close(wfd)
                     reader = os.fdopen(rfd, "rb")
@@ -429,7 +442,7 @@
             seed_outputs = set()
             override_path = os.path.join(
                 self.options.misc_root,
-                "more-extra.override.%s.main" % series_name)
+                "more-extra.override.%s.main" % suffix)
             with AtomicFile(override_path) as override_file:
                 for pid, reader in procs:
                     log_records, overrides, arch_seed_outputs = pickle.load(
@@ -447,7 +460,6 @@
         self.setUp()
 
         for series in self.series:
-            series_name = series.name
             components = self.getComponents(series)
             if not self.options.architectures:
                 architectures = sorted(
@@ -456,7 +468,7 @@
                 architectures = self.options.architectures
 
             self.generateExtraOverrides(
-                series_name, components, architectures, self.args,
+                series, components, architectures, self.args,
                 seed_bases=seed_bases)
 
         # Add a marker line so that other code can tell from the log file

=== modified file 'tests/test_generate_extra_overrides.py'
--- tests/test_generate_extra_overrides.py	2021-05-13 12:49:01 +0000
+++ tests/test_generate_extra_overrides.py	2021-05-13 12:49:01 +0000
@@ -311,23 +311,39 @@
             sources_path = os.path.join(
                 script.archiveroot, "dists", distroseries.name, component,
                 "source", "Sources.gz")
+            sources_path_updates = os.path.join(
+                script.archiveroot, "dists",
+                distroseries.name + "-updates", component,
+                "source", "Sources.gz")
             ensure_directory_exists(os.path.dirname(sources_path))
+            ensure_directory_exists(os.path.dirname(sources_path_updates))
             sources_index = gzip.GzipFile(sources_path, "wb")
             for spp in distroseries.test_sources[component]:
                 stanza = spp.getIndexStanza().encode("utf-8") + "\n\n"
                 sources_index.write(stanza)
             sources_index.close()
+            sources_index_updates = gzip.GzipFile(sources_path_updates,
+                                                  "wb")
+            sources_index_updates.close()
 
             for arch in distroseries.architectures:
                 packages_path = os.path.join(
                     script.archiveroot, "dists", distroseries.name, component,
                     "binary-%s" % arch.architecture_tag, "Packages.gz")
+                packages_path_updates = os.path.join(
+                    script.archiveroot, "dists",
+                    distroseries.name + "-updates", component,
+                    "binary-%s" % arch.architecture_tag, "Packages.gz")
                 ensure_directory_exists(os.path.dirname(packages_path))
+                ensure_directory_exists(os.path.dirname(packages_path_updates))
                 packages_index = gzip.GzipFile(packages_path, "wb")
                 for bpp in arch.test_binaries[component]:
                     stanza = bpp.getIndexStanza().encode("utf-8") + "\n\n"
                     packages_index.write(stanza)
                 packages_index.close()
+                packages_index_updates = gzip.GzipFile(packages_path_updates,
+                                                       "wb")
+                packages_index_updates.close()
 
     def composeSeedPath(self, flavour, series_name, seed_name):
         return os.path.join(
@@ -405,24 +421,17 @@
         self.setUpDistroAndScript()
         self.assertEqual(self.distro, self.script.distribution)
 
-    def test_prefers_development_distro_series(self):
-        # The script prefers a DEVELOPMENT series for the named
-        # distribution over CURRENT and SUPPORTED series.
+    def test_uses_all_series(self):
+        # The script operates on DEVELOPMENT and CURRENT series.
         self.setUpDistroAndScript(
             ["Supported", "Current Stable Release", "Active Development"])
-        self.assertEqual([self.distroseries[2]], self.script.series)
+        self.assertEqual(self.distroseries, self.script.series)
 
     def test_permits_frozen_distro_series(self):
-        # If there is no DEVELOPMENT series, a FROZEN one will do.
+        # FROZEN series are included
         self.setUpDistroAndScript(
             ["Supported", "Current Stable Release", "Pre-release Freeze"])
-        self.assertEqual([self.distroseries[2]], self.script.series)
-
-    def test_requires_development_frozen_distro_series(self):
-        # If there is no DEVELOPMENT or FROZEN series, the script fails.
-        self.setUpDistroAndScript(
-            ["Supported", "Current Stable Release"], run_setup=False)
-        self.assertRaises(ScriptFailure, self.script.processOptions)
+        self.assertEqual(self.distroseries, self.script.series)
 
     def test_multiple_development_frozen_distro_series(self):
         # If there are multiple DEVELOPMENT or FROZEN series, they are all
@@ -457,6 +466,24 @@
                 self.distroseries[0].name, arch),
             output)
 
+    def test_stable_compose_output_path_in_miscroot(self):
+        # Output files are written to the correct locations under
+        # misc_root when running for a stable release.
+        flavour = factory.getUniqueString()
+        self.setUpDistroAndScript(
+            ["Current Stable Release"], extra_args=[flavour])
+        self.setUpComponent()
+        self.distroseries[0].makeDistroArchSeries()
+        self.makeIndexFiles(self.script, self.distroseries[0])
+        seed = factory.getUniqueString()
+        self.makeSeedStructure(flavour, self.distroseries[0].name, [seed])
+        self.makeSeed(flavour, self.distroseries[0].name, seed, [])
+
+        self.script.process(seed_bases=["file://%s" % self.seeddir])
+        self.assertTrue(os.path.exists(os.path.join(
+            self.script.options.misc_root,
+            "more-extra.override.%s-updates.main" % self.distroseries[0].name)))
+
     def test_make_seed_structures_missing_seeds(self):
         # makeSeedStructures ignores missing seeds.
         self.setUpDistroAndScript()
@@ -497,9 +524,12 @@
             distroseries.name, flavours,
             seed_bases=["file://%s" % self.seeddir])
 
+        series_is_stable = distroseries.status in ("Current Stable Release",
+                                                   "Supported")
+
         _, overrides, _ = script.germinateArch(
-            distroseries.name, script.getComponents(distroseries), arch,
-            flavours, structures)
+            distroseries.name, series_is_stable,
+            script.getComponents(distroseries), arch, flavours, structures)
         return overrides.splitlines()
 
     def test_germinate_output(self):