Merge lp:~laney/ubuntu-archive-publishing/more-options into lp:ubuntu-archive-publishing

[Iain Lane]

Add some options to make it easier to run generate-extra-overrides locally

[Colin Watson (cjwatson)]

review: Approve

=== modified file 'lib/scripts/generate_extra_overrides.py'
--- lib/scripts/generate_extra_overrides.py	2020-02-06 12:21:14 +0000
+++ lib/scripts/generate_extra_overrides.py	2021-05-13 12:48:40 +0000
@@ -103,6 +103,15 @@
         self.parser.add_option(
             "-s", "--series", dest="series", action="append",
             help="Series to operate on (may be given multiple times).")
+        self.parser.add_option(
+            "-a", "--arch", dest="architectures", action="append",
+            help="Arches to operate on (may be given multiple times).")
+        self.parser.add_option(
+            "-g", "--germinate-root", dest="germinate_root",
+            help="Location to write germinate output to.")
+        self.parser.add_option(
+            "-m", "--misc-root", dest="misc_root",
+            help="Location to write miscellaneous output to.")
 
     @property
     def name(self):
@@ -140,18 +149,20 @@
         # the distribution's archives in the ARCHIVEROOTS environment
         # variable, joined by spaces.  We only care about the first archive.
         self.archiveroot = os.environ["ARCHIVEROOTS"].split()[0]
-        self.germinateroot = self.archiveroot + "-germinate"
-        self.miscroot = self.archiveroot + "-misc"
+        if not self.options.germinate_root:
+            self.options.germinate_root = self.archiveroot + "-germinate"
+        if not self.options.misc_root:
+            self.options.misc_root = self.archiveroot + "-misc"
 
     def setUpDirs(self):
         """Create output directories if they did not already exist."""
-        if not file_exists(self.germinateroot):
+        if not file_exists(self.options.germinate_root):
             self.logger.debug(
-                "Creating germinate root %s.", self.germinateroot)
-            os.makedirs(self.germinateroot)
-        if not file_exists(self.miscroot):
-            self.logger.debug("Creating misc root %s.", self.miscroot)
-            os.makedirs(self.miscroot)
+                "Creating germinate root %s.", self.options.germinate_root)
+            os.makedirs(self.options.germinate_root)
+        if not file_exists(self.options.misc_root):
+            self.logger.debug("Creating misc root %s.", self.options.misc_root)
+            os.makedirs(self.options.misc_root)
 
     def addLogHandler(self):
         """Send germinate's log output to a separate file."""
@@ -160,7 +171,8 @@
 
         self.germinate_logger = logging.getLogger("germinate")
         self.germinate_logger.setLevel(logging.INFO)
-        self.log_file = os.path.join(self.germinateroot, "germinate.output")
+        self.log_file = os.path.join(self.options.germinate_root,
+                                     "germinate.output")
         self.log_handler = logging.FileHandler(self.log_file, mode="w")
         self.log_handler.setFormatter(GerminateFormatter())
         self.germinate_logger.addHandler(self.log_handler)
@@ -212,7 +224,7 @@
 
     def composeOutputPath(self, flavour, series_name, arch, base):
         return os.path.join(
-            self.germinateroot,
+            self.options.germinate_root,
             "%s_%s_%s_%s" % (base, flavour, series_name, arch))
 
     def recordOutput(self, path, seed_outputs):
@@ -387,7 +399,8 @@
         Any per-seed outputs not in seed_outputs are considered stale.
         """
         all_outputs = glob.glob(
-            os.path.join(self.germinateroot, "*_*_%s_*" % series_name))
+            os.path.join(self.options.germinate_root,
+                         "*_*_%s_*" % series_name))
         for output in all_outputs:
             if os.path.basename(output) not in seed_outputs:
                 os.remove(output)
@@ -415,7 +428,8 @@
 
             seed_outputs = set()
             override_path = os.path.join(
-                self.miscroot, "more-extra.override.%s.main" % series_name)
+                self.options.misc_root,
+                "more-extra.override.%s.main" % series_name)
             with AtomicFile(override_path) as override_file:
                 for pid, reader in procs:
                     log_records, overrides, arch_seed_outputs = pickle.load(
@@ -435,8 +449,11 @@
         for series in self.series:
             series_name = series.name
             components = self.getComponents(series)
-            architectures = sorted(
-                arch.architecture_tag for arch in series.architectures)
+            if not self.options.architectures:
+                architectures = sorted(
+                    arch.architecture_tag for arch in series.architectures)
+            else:
+                architectures = self.options.architectures
 
             self.generateExtraOverrides(
                 series_name, components, architectures, self.args,

=== modified file 'tests/test_generate_extra_overrides.py'
--- tests/test_generate_extra_overrides.py	2014-06-30 01:11:46 +0000
+++ tests/test_generate_extra_overrides.py	2021-05-13 12:48:40 +0000
@@ -453,7 +453,7 @@
             flavour, self.distroseries[0].name, arch, base)
         self.assertEqual(
             "%s/%s_%s_%s_%s" % (
-                self.script.germinateroot, base, flavour,
+                self.script.options.germinate_root, base, flavour,
                 self.distroseries[0].name, arch),
             output)
 
@@ -693,7 +693,7 @@
         seed_old_file = "old_flavour_%s_i386" % series_name
         seed_new_file = "new_flavour_%s_i386" % series_name
         other_file = "other-file"
-        output = partial(os.path.join, self.script.germinateroot)
+        output = partial(os.path.join, self.script.options.germinate_root)
         for base in (seed_old_file, seed_new_file, other_file):
             write_file(output(base), "")
         self.script.removeStaleOutputs(series_name, set([seed_new_file]))
@@ -716,10 +716,10 @@
 
         self.script.process(seed_bases=["file://%s" % self.seeddir])
         self.assertFalse(os.path.exists(os.path.join(
-            self.script.miscroot,
+            self.script.options.misc_root,
             "more-extra.override.%s.main" % self.distroseries[0].name)))
         self.assertTrue(os.path.exists(os.path.join(
-            self.script.miscroot,
+            self.script.options.misc_root,
             "more-extra.override.%s.main" % self.distroseries[1].name)))
 
     def test_process_removes_only_stale_files(self):
@@ -784,7 +784,8 @@
 
         self.script.process(seed_bases=["file://%s" % self.seeddir])
         override_path = os.path.join(
-            self.script.miscroot, "more-extra.override.%s.main" % series_name)
+            self.script.options.misc_root,
+            "more-extra.override.%s.main" % series_name)
         expected_overrides = [
             "%s/%s  Task  %s" % (package, das_one.architecture_tag, seed),
             "%s/%s  Task  %s" % (package, das_two.architecture_tag, seed),
@@ -792,7 +793,8 @@
         self.assertCountEqual(
             expected_overrides, file_contents(override_path).splitlines())
 
-        log_file = os.path.join(self.script.germinateroot, "germinate.output")
+        log_file = os.path.join(self.script.options.germinate_root,
+                                "germinate.output")
         self.assertIn("Downloading file://", file_contents(log_file))
 
     def test_run_script_help(self):