Merge ~mwhudson/ubuntu-cdimage:run-linters-in-ci into ubuntu-cdimage:main

[Michael Hudson-Doyle]

run linting tests in CI and fix failures

[Steve Langasek (vorlon)]

review: Approve

diff --git a/.launchpad.yaml b/.launchpad.yaml
index f01d515..e023b8a 100644
--- a/.launchpad.yaml
+++ b/.launchpad.yaml
@@ -3,9 +3,9 @@ pipeline:
 
 jobs:
     run-tests:
-        series: focal
+        series: jammy
         architectures: amd64
-        packages: [dctrl-tools, python3, python3, python3-mock, python3-simplestreams, procmail, squashfs-tools, file]
+        packages: [dctrl-tools, python3, python3, python3-mock, python3-simplestreams, procmail, squashfs-tools, file, pep8, pyflakes3]
         run: ./run-tests >output
         output:
             paths: [output]
diff --git a/bin/regenerate-streams b/bin/regenerate-streams
index 2a27878..34d2fad 100755
--- a/bin/regenerate-streams
+++ b/bin/regenerate-streams
@@ -26,7 +26,6 @@ sys.path.insert(0, os.path.join(sys.path[0], os.pardir, "lib"))
 
 def main():
     from cdimage.config import Config
-    from cdimage.tree import Publisher, Tree
     from cdimage.simplestreams import SimpleStreams
 
     parser = OptionParser(
diff --git a/lib/cdimage/build.py b/lib/cdimage/build.py
index ea9c507..fd3c79d 100644
--- a/lib/cdimage/build.py
+++ b/lib/cdimage/build.py
@@ -18,7 +18,6 @@
 from __future__ import print_function
 
 import contextlib
-import gzip
 import os
 import shutil
 import signal
@@ -41,7 +40,7 @@ from cdimage.livefs import (
 )
 from cdimage.log import logger, reset_logging
 from cdimage.mail import get_notify_addresses, send_mail
-from cdimage.mirror import find_mirror, trigger_mirrors
+from cdimage.mirror import trigger_mirrors
 from cdimage.tracker import tracker_set_rebuild_status
 from cdimage.tree import Publisher, Tree
 from cdimage.config import Touch
diff --git a/lib/cdimage/livefs.py b/lib/cdimage/livefs.py
index 355d8ed..c912dc0 100644
--- a/lib/cdimage/livefs.py
+++ b/lib/cdimage/livefs.py
@@ -368,7 +368,8 @@ def run_live_builds(config):
                 cpuarch, subarch = split_arch(config, arch)
                 for build in lp_livefs.builds:
                     try:
-                        metadata_subarch = build.metadata_override.get("subarch", "")
+                        metadata_subarch = build.metadata_override.get(
+                            "subarch", "")
                     except AttributeError:
                         metadata_subarch = ""
                     if (build.distro_arch_series.architecture_tag == cpuarch
diff --git a/lib/cdimage/simplestreams.py b/lib/cdimage/simplestreams.py
index 92eea0b..9487a8b 100644
--- a/lib/cdimage/simplestreams.py
+++ b/lib/cdimage/simplestreams.py
@@ -19,7 +19,6 @@
 
 import os
 import re
-import json
 import time
 import hashlib
 import simplestreams.generate_simplestreams as generate_simplestreams
diff --git a/lib/cdimage/tests/test_build.py b/lib/cdimage/tests/test_build.py
index 8d4d164..18409d8 100644
--- a/lib/cdimage/tests/test_build.py
+++ b/lib/cdimage/tests/test_build.py
@@ -20,7 +20,6 @@
 from __future__ import print_function
 
 from functools import partial
-import gzip
 import optparse
 import os
 import signal
diff --git a/lib/cdimage/tests/test_config.py b/lib/cdimage/tests/test_config.py
index 020f920..a45bfca 100644
--- a/lib/cdimage/tests/test_config.py
+++ b/lib/cdimage/tests/test_config.py
@@ -66,11 +66,6 @@ class TestSeries(TestCase):
             _core_series="22")
         mantic = Series("mantic", "23.10", "Mantic Minotaur")
         mantic._override_is_latest = True
-        all_series = [
-            impish,
-            jammy,
-            mantic
-        ]
         self.assertEqual("22", jammy.core_series)
         self.assertEqual("24", mantic.core_series)
         self.assertEqual(None, impish.core_series)
diff --git a/lib/cdimage/tests/test_germinate.py b/lib/cdimage/tests/test_germinate.py
index 5c68812..4f1255d 100644
--- a/lib/cdimage/tests/test_germinate.py
+++ b/lib/cdimage/tests/test_germinate.py
@@ -20,7 +20,6 @@
 from __future__ import print_function
 
 from functools import partial
-import gzip
 import os
 import subprocess
 from textwrap import dedent
@@ -104,7 +103,8 @@ class TestGermination(TestCase):
             ("ubuntu-mate", "xenial", ["ubuntu-mate-dev", "ubuntu-core-dev"]),
             ("ubuntukylin", "xenial",
              ["ubuntukylin-members", "ubuntu-core-dev"]),
-            ("ubuntu-budgie", "bionic", ["ubuntubudgie-dev", "ubuntu-core-dev"]),
+            ("ubuntu-budgie", "bionic",
+             ["ubuntubudgie-dev", "ubuntu-core-dev"]),
             ("ubuntustudio", "trusty",
              ["ubuntustudio-dev", "ubuntu-core-dev"]),
         ):
diff --git a/lib/cdimage/tests/test_mirror.py b/lib/cdimage/tests/test_mirror.py
index 6dc437a..d1c584c 100644
--- a/lib/cdimage/tests/test_mirror.py
+++ b/lib/cdimage/tests/test_mirror.py
@@ -52,35 +52,43 @@ class TestChecksumFile(TestCase):
 
     def test_amd64(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "amd64", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "amd64", series)
 
     def test_armel(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "armel", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "armel", series)
 
     def test_hppa(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "hppa", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "hppa", series)
 
     def test_i386(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "i386", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "i386", series)
 
     def test_powerpc(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "powerpc", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "powerpc", series)
 
     def test_ppc64el(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "ppc64el", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "ppc64el", series)
 
     def test_s390x(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "s390x", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "s390x", series)
 
     def test_sparc(self):
         for series in all_series:
-            self.assertMirrorEqual("http://ftpmaster.internal/ubuntu/", "sparc", series)
+            self.assertMirrorEqual(
+                "http://ftpmaster.internal/ubuntu/", "sparc", series)
 
 
 class TestTriggerMirrors(TestCase):
diff --git a/lib/cdimage/tests/test_simplestreams.py b/lib/cdimage/tests/test_simplestreams.py
index a54734f..1825480 100644
--- a/lib/cdimage/tests/test_simplestreams.py
+++ b/lib/cdimage/tests/test_simplestreams.py
@@ -32,7 +32,7 @@ from cdimage.simplestreams import (
     SimpleReleaseSimpleStreams)
 from cdimage.config import Config, Series
 from cdimage.tree import (
-    Tree, Publisher, DailyTreePublisher, FullReleasePublisher,
+    Tree, DailyTreePublisher, FullReleasePublisher,
     FullReleaseTree, SimpleReleasePublisher, SimpleReleaseTree)
 from cdimage.tests.helpers import TestCase
 
@@ -235,7 +235,7 @@ class TestSimpleStreams(TestCase):
                     self.config, publisher)
                 self.assertIsInstance(streams, cls_streams)
             else:
-                with self.assertRaises(Exception) as e:
+                with self.assertRaises(Exception):
                     SimpleStreams.get_simplestreams(
                         self.config, publisher)
 
@@ -255,7 +255,7 @@ class TestSimpleStreams(TestCase):
                     self.config, name)
                 self.assertIsInstance(streams, cls_streams)
             else:
-                with self.assertRaises(Exception) as e:
+                with self.assertRaises(Exception):
                     SimpleStreams.get_simplestreams_by_name(
                         self.config, name)
 
diff --git a/lib/cdimage/tests/test_static.py b/lib/cdimage/tests/test_static.py
index c6a909d..cc3ad1a 100644
--- a/lib/cdimage/tests/test_static.py
+++ b/lib/cdimage/tests/test_static.py
@@ -63,11 +63,11 @@ class TestStatic(TestCase):
             return
         subp = subprocess.Popen(
             ["pep8"] + self.all_paths(),
-            stdout=subprocess.PIPE, universal_newlines=True)
-        output = subp.communicate()[0].splitlines()
-        for line in output:
-            print(line)
-        self.assertEqual(0, len(output))
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+            universal_newlines=True)
+        output = subp.communicate()[0]
+        if output:
+            self.fail("pep8 produced output:\n\n" + output)
 
     def test_pyflakes_clean(self):
         if sys.version < "3":
@@ -90,19 +90,20 @@ class TestStatic(TestCase):
                     if not line.startswith("#"):
                         exclusions.add(line.rstrip())
 
-        error = False
         subp = subprocess.Popen(
             [pyflakes] + self.all_paths(),
             stdout=subprocess.PIPE, universal_newlines=True)
         output = subp.communicate()[0].splitlines()
+        not_excluded = []
         for line in output:
             if line.startswith("#"):
                 continue
             line = line.rstrip()
-            canon_line = re.sub(r":[0-9]+:(?:[0-9]+)?", ":*:", line, 1)
+            canon_line = re.sub(r":[0-9]+:(?:[0-9]+)?:?", ":*:", line, 1)
             canon_line = re.sub(r"line [0-9]+", "line *", canon_line)
             if canon_line not in exclusions:
-                print(line)
-                error = True
+                not_excluded.append(line)
 
-        self.assertFalse(error)
+        if not_excluded:
+            self.fail(
+                "pyflakes produced output:\n\n" + "\n".join(not_excluded))
diff --git a/lib/cdimage/tree.py b/lib/cdimage/tree.py
index d4a4d20..038d0fe 100644
--- a/lib/cdimage/tree.py
+++ b/lib/cdimage/tree.py
@@ -646,7 +646,7 @@ class Publisher:
                 "The netboot tarball contains files needed to boot the %s "
                 "installer over the network." % (capproject,))
         elif publish_type == "mini-iso":
-            sentances.append(
+            sentences.append(
                 "The mini ISO image is a small ISO image that can be used "
                 "to choose which other Ubuntu image to download and install.")
         elif publish_type == "alternate":
@@ -1212,8 +1212,6 @@ class Publisher:
             if entry.endswith(".%s" % extension)])
 
     def make_web_indices(self, directory, base_prefix, status="release"):
-        series = self.config["DIST"]
-
         prefixes = [base_prefix]
         if base_prefix.count(".") >= 2:
             # point release - need the base version too