Merge ~jugmac00/launchpad:allow_limiting_uct_imports into launchpad:master

[Jürgen Gmach]

Enable filtering for UCTImports

[Jürgen Gmach (jugmac00)]

Enable filtering for UCTImports

[Colin Watson (cjwatson)]

review: Approve

[Colin Watson (cjwatson)]

Could you fix the commit message of this MP, since that will be used by the merge bot when landing this?

[Jürgen Gmach (jugmac00)]

Thanks for the review!

diff --git a/lib/lp/bugs/scripts/tests/sampledata/CVE-2007-0255 b/lib/lp/bugs/scripts/tests/sampledata/CVE-2007-0255
new file mode 100644
index 0000000..db2403d
--- /dev/null
+++ b/lib/lp/bugs/scripts/tests/sampledata/CVE-2007-0255
@@ -0,0 +1,61 @@
+PublicDate: 2007-01-16 23:28:00 UTC
+Candidate: CVE-2007-0255
+References: 
+ https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-0255
+ http://xine.sourceforge.net/security
+Description:
+ XINE 0.99.4 allows user-assisted remote attackers to cause a denial of
+ service (application crash) and possibly execute arbitrary code via a
+ certain M3U file that contains a long #EXTINF line and contains format
+ string specifiers in an invalid udp:// URI, possibly a variant of
+ CVE-2007-0017.
+Ubuntu-Description: 
+Notes: 
+ sbeattie> issue is unlisted on xine upstream website
+Priority: medium
+Bugs: 
+Discovered-by:
+Assigned-to:
+CVSS:
+
+Patches_xine-ui:
+upstream_xine-ui: needs-triage
+dapper_xine-ui: ignored (reached end-of-life)
+edgy_xine-ui: needed (reached end-of-life)
+feisty_xine-ui: needed (reached end-of-life)
+gutsy_xine-ui: needed (reached end-of-life)
+hardy_xine-ui: ignored (reached end-of-life)
+intrepid_xine-ui: needed (reached end-of-life)
+jaunty_xine-ui: ignored (reached end-of-life)
+karmic_xine-ui: ignored (reached end-of-life)
+lucid_xine-ui: ignored (reached end-of-life)
+maverick_xine-ui: ignored (reached end-of-life)
+natty_xine-ui: ignored (reached end-of-life)
+oneiric_xine-ui: ignored (reached end-of-life)
+precise_xine-ui: ignored (reached end-of-life)
+precise/esm_xine-ui: DNE (precise was needed)
+quantal_xine-ui: ignored (reached end-of-life)
+raring_xine-ui: ignored (reached end-of-life)
+saucy_xine-ui: ignored (reached end-of-life)
+trusty_xine-ui: ignored (reached end-of-life)
+trusty/esm_xine-ui: DNE (trusty was needed)
+utopic_xine-ui: ignored (reached end-of-life)
+vivid_xine-ui: ignored (reached end-of-life)
+vivid/stable-phone-overlay_xine-ui: DNE
+vivid/ubuntu-core_xine-ui: DNE
+wily_xine-ui: ignored (reached end-of-life)
+xenial_xine-ui: ignored (end of standard support, was needed)
+yakkety_xine-ui: ignored (reached end-of-life)
+zesty_xine-ui: ignored (reached end-of-life)
+artful_xine-ui: ignored (reached end-of-life)
+bionic_xine-ui: needed
+cosmic_xine-ui: ignored (reached end-of-life)
+disco_xine-ui: ignored (reached end-of-life)
+eoan_xine-ui: ignored (reached end-of-life)
+focal_xine-ui: needed
+groovy_xine-ui: ignored (reached end-of-life)
+hirsute_xine-ui: ignored (reached end-of-life)
+impish_xine-ui: ignored (reached end-of-life)
+jammy_xine-ui: needed
+kinetic_xine-ui: needed
+devel_xine-ui: needed
\ No newline at end of file
diff --git a/lib/lp/bugs/scripts/tests/sampledata/CVE-2022-3219 b/lib/lp/bugs/scripts/tests/sampledata/CVE-2022-3219
new file mode 100644
index 0000000..14aaa73
--- /dev/null
+++ b/lib/lp/bugs/scripts/tests/sampledata/CVE-2022-3219
@@ -0,0 +1,43 @@
+Candidate: CVE-2022-3219
+PublicDate: 2022-09-28
+References:
+ https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-3219
+ https://access.redhat.com/security/cve/CVE-2022-3219
+ https://marc.info/?l=oss-security&m=165696590211434&w=4
+Description:
+ gnupg: denial of service issue (resource consumption) using compressed
+ packets
+Ubuntu-Description:
+Notes:
+ mdeslaur> per the upstream gnupg bug, the change will not be applied
+ mdeslaur> as of 2022-09-28, proposed patch has not been accepted by
+ mdeslaur> upstream developers
+Mitigation:
+Bugs:
+ https://dev.gnupg.org/T5993
+Priority: low
+Discovered-by:
+Assigned-to:
+CVSS:
+
+Patches_gnupg:
+upstream_gnupg: needs-triage
+esm-infra/xenial_gnupg: deferred (2022-09-28)
+trusty_gnupg: ignored (out of standard support)
+xenial_gnupg: ignored (out of standard support)
+bionic_gnupg: DNE
+focal_gnupg: DNE
+jammy_gnupg: DNE
+trusty/esm_gnupg: deferred (2022-09-28)
+
+Patches_gnupg2:
+ other: https://dev.gnupg.org/D556
+upstream_gnupg2: needs-triage
+esm-infra/xenial_gnupg2: deferred (2022-09-28)
+trusty_gnupg2: ignored (out of standard support)
+xenial_gnupg2: ignored (end of standard support)
+bionic_gnupg2: deferred (2022-09-28)
+focal_gnupg2: deferred (2022-09-28)
+jammy_gnupg2: deferred (2022-09-28)
+kinetic_gnupg2: deferred (2022-09-28)
+devel_gnupg2: deferred (2022-09-28)
\ No newline at end of file
diff --git a/lib/lp/bugs/scripts/tests/test_uctimport.py b/lib/lp/bugs/scripts/tests/test_uctimport.py
new file mode 100644
index 0000000..2e36bb5
--- /dev/null
+++ b/lib/lp/bugs/scripts/tests/test_uctimport.py
@@ -0,0 +1,88 @@
+from pathlib import Path
+
+from lp.testing import TestCase
+from lp.testing.layers import LaunchpadZopelessLayer
+from lp.testing.script import run_script
+
+
+class TestUCTImportScript(TestCase):
+    """Test the TestUCTImportScript class."""
+
+    layer = LaunchpadZopelessLayer
+
+    def test_no_path_given(self):
+        """TestUCTImportScript errors when no path given"""
+        exit_code, out, err = run_script(
+            script="scripts/uct-import.py",
+            args=[],
+        )
+        self.assertEqual(2, exit_code)
+        self.assertEqual("", out)
+        self.assertEqual(
+            "Usage: uct-import.py [options] PATH\n\nuct-import.py: "
+            "error: Please specify a path to import\n",
+            err,
+        )
+
+    def test_load_from_file(self):
+        load_from = Path(__file__).parent / "sampledata" / "CVE-2022-23222"
+        exit_code, out, err = run_script(
+            script="scripts/uct-import.py",
+            args=[str(load_from)],
+        )
+        self.assertEqual(0, exit_code)
+        self.assertEqual("", out)
+        self.assertIn("CVE-2022-23222 was imported successfully", err)
+
+    def test_load_from_directory(self):
+        load_from = Path(__file__).parent / "sampledata"
+        exit_code, out, err = run_script(
+            script="scripts/uct-import.py",
+            args=[str(load_from)],
+        )
+        self.assertEqual(0, exit_code)
+        self.assertEqual("", out)
+        self.assertIn("CVE-2007-0255 was imported successfully", err)
+        self.assertIn("CVE-2022-3219 was imported successfully", err)
+        self.assertIn("CVE-2022-23222 was imported successfully", err)
+
+    def test_dry_run_does_not_crash(self):
+        load_from = Path(__file__).parent / "sampledata" / "CVE-2022-23222"
+        exit_code, out, err = run_script(
+            script="scripts/uct-import.py",
+            args=[str(load_from), "--dry-run"],
+        )
+        self.assertEqual(0, exit_code)
+        self.assertEqual("", out)
+        self.assertRegex(err, r"^INFO    Importing.*CVE-2022-23222.*")
+
+    def test_filter_cve(self):
+        load_from = Path(__file__).parent / "sampledata"
+        exit_code, out, err = run_script(
+            script="scripts/uct-import.py",
+            args=[str(load_from), "--filter", "2007*"],
+        )
+        self.assertEqual(0, exit_code)
+        self.assertEqual("", out)
+        self.assertNotIn("CVE-2022-23222 was imported successfully", err)
+        self.assertIn("CVE-2007-0255 was imported successfully", err)
+
+        exit_code, out, err = run_script(
+            script="scripts/uct-import.py",
+            args=[str(load_from), "--filter", "2022*"],
+        )
+        self.assertEqual(0, exit_code)
+        self.assertEqual("", out)
+        self.assertIn("CVE-2022-23222 was imported successfully", err)
+        self.assertIn("CVE-2022-3219 was imported successfully", err)
+        self.assertNotIn("CVE-2007-0255 was imported successfully", err)
+
+        exit_code, out, err = run_script(
+            script="scripts/uct-import.py",
+            args=[str(load_from), "--filter", "20[02][27]*"],
+        )
+        self.assertEqual(0, exit_code)
+        self.assertEqual("", out)
+        self.assertIn("CVE-2022-23222 was imported successfully", err)
+        self.assertIn("CVE-2022-3219 was imported successfully", err)
+        self.assertIn("CVE-2007-0255 was imported successfully", err)
diff --git a/lib/lp/bugs/scripts/uctimport.py b/lib/lp/bugs/scripts/uctimport.py
new file mode 100644
index 0000000..aed11c3
--- /dev/null
+++ b/lib/lp/bugs/scripts/uctimport.py
@@ -0,0 +1,63 @@
+import logging
+from pathlib import Path
+
+from lp.app.validators.cve import CVEREF_PATTERN
+from lp.bugs.scripts.uct import UCTImporter
+from lp.services.scripts.base import LaunchpadScript
+
+logger = logging.getLogger(__name__)
+
+
+class UCTImportScript(LaunchpadScript):
+    """CLI for UCTImport
+
+    Command line options:
+    The filter option takes a glob-style pattern.
+    Example: `2007*` filters all CVEs from the year 2007.
+    """
+
+    usage = "usage: %prog [options] PATH"
+    description = (
+        "Import bugs into Launchpad from CVE entries in ubuntu-cve-tracker. "
+        "PATH is either path to a CVE file, or path to a directory "
+        "containing the CVE files."
+    )
+    loglevel = logging.INFO
+
+    def add_my_options(self):
+        self.parser.add_option(
+            "--dry-run",
+            action="store_true",
+            dest="dry_run",
+            default=False,
+            help="Don't commit changes to the DB.",
+        )
+        self.parser.add_option(
+            "--filter",
+            action="store",
+            dest="filter",
+            default="*",
+            help="Apply given glob-style pattern to filter CVEs.",
+        )
+
+    def main(self):
+        if len(self.args) != 1:
+            self.parser.error("Please specify a path to import")
+        path = Path(self.args[0])
+        if path.is_dir():
+            logger.info(
+                "Importing CVE files from directory: %s", path.resolve()
+            )
+            cve_paths = sorted(
+                p
+                for p in path.rglob("CVE-%s" % self.options.filter)
+                if p.is_file() and CVEREF_PATTERN.match(p.name)
+            )
+            if not cve_paths:
+                logger.warning("Could not find CVE files in %s", path)
+                return
+        else:
+            cve_paths = [path]
+        importer = UCTImporter(dry_run=self.options.dry_run)
+        for cve_path in cve_paths:
+            importer.import_cve_from_file(cve_path)
diff --git a/scripts/uct-import.py b/scripts/uct-import.py
index 489d6ea..9ade412 100755
--- a/scripts/uct-import.py
+++ b/scripts/uct-import.py
@@ -4,59 +4,7 @@
 # GNU Affero General Public License version 3 (see the file LICENSE).
 import _pythonpath  # noqa: F401
 
-import logging
-from pathlib import Path
-
-from lp.app.validators.cve import CVEREF_PATTERN
-from lp.bugs.scripts.uct import UCTImporter
-from lp.services.scripts.base import LaunchpadScript
-
-logger = logging.getLogger(__name__)
-
-
-class UCTImportScript(LaunchpadScript):
-
-    usage = "usage: %prog [options] PATH"
-    description = (
-        "Import bugs into Launchpad from CVE entries in ubuntu-cve-tracker. "
-        "PATH is either path to a CVE file, or path to a directory "
-        "containing the CVE files"
-    )
-    loglevel = logging.INFO
-
-    def add_my_options(self):
-        self.parser.add_option(
-            "--dry-run",
-            action="store_true",
-            dest="dry_run",
-            default=False,
-            help="Don't commit changes to the DB.",
-        )
-
-    def main(self):
-        if len(self.args) != 1:
-            self.parser.error("Please specify a path to import")
-
-        path = Path(self.args[0])
-        if path.is_dir():
-            logger.info(
-                "Importing CVE files from directory: %s", path.resolve()
-            )
-            cve_paths = sorted(
-                p
-                for p in path.rglob("CVE-*")
-                if p.is_file() and CVEREF_PATTERN.match(p.name)
-            )
-            if not cve_paths:
-                logger.warning("Could not find CVE files in %s", path)
-                return
-        else:
-            cve_paths = [path]
-
-        importer = UCTImporter(dry_run=self.options.dry_run)
-        for cve_path in cve_paths:
-            importer.import_cve_from_file(cve_path)
-
+from lp.bugs.scripts.uctimport import UCTImportScript
 
 if __name__ == "__main__":
     script = UCTImportScript("lp.services.scripts.uctimport")