Merge ~litios/ubuntu-cve-tracker:aliases-customer-ppa into ubuntu-cve-tracker:master

[David Fernandez Gonzalez]

This is the first approach to handle the aliases for projects where the name of the package is specific to it. For example, in project test package foo is actually named test-foo.

I'm relying on the structures that already exist for handling subprojects and adding a new field for taking care of the aliases for the packages.

We discussed naming them boilerplates but I wasn't sure about that because of the already existing boilerplates and because it's a different unrelated thing.

The solution detects if an aliases.yaml exists in the project and performs extra operations according to the aliases specified. Only two extra checks are performed: are all the aliases in the CVE and does the original package name appear (if not specified as an alias)?

The yaml file would be a list of packages and an array of aliases like:

```
foo: ["test-foo", "test2-foo"]
bar: ["test-bar", "test2-bar", "bar"]
```

As I said in the beginning, this is more of a first idea and I would love to get input from people with more experience with UCT.

[Eduardo Barretto (ebarretto)]

in boilerplates we have the "release" as part of the structure. Here we have package to package mapping. Could this create some kind of issue? I can't think of one right now, so just trying to brainstorm.

[Eduardo Barretto (ebarretto)]

thanks for clarifying that this will be per subproject/release
Let's integrate this, and later we could remove the hardcoded part as we discussed

review: Approve

diff --git a/scripts/check-syntax b/scripts/check-syntax
index cacdedf..a9a12a2 100755
--- a/scripts/check-syntax
+++ b/scripts/check-syntax
@@ -421,9 +421,44 @@ for cve in args:
         # Verify have required releases for each package
         listed_releases = set(sorted(data["pkgs"][pkg].keys()))
         all_required_releases = (set(cve_lib.all_releases + ["devel"]) - set([cve_lib.devel_release])) - set(cve_lib.eol_releases)
-        missing_releases = all_required_releases - listed_releases
         # get the name of a release which is listed in the CVE so we can
         # place the generated error message on this release's line etc
+        nearby_rel = list(listed_releases)[0]
+        aliases_releases = set()
+
+        # Check aliases
+        for rel in all_required_releases:
+            if rel in source and pkg in source[rel] and \
+                'aliases' in source[rel][pkg]:
+                aliases_releases.add(rel)
+
+                # This package-release uses aliases,
+                # it shouldn't be listed
+                if rel in listed_releases and pkg \
+                    not in source[rel][pkg]['aliases']:
+                    filename = srcmap["pkgs"][pkg][nearby_rel][0]
+                    linenum = srcmap["pkgs"][pkg][nearby_rel][1]
+                    print(
+                        "%s: %d: package '%s' not in '%s'"
+                        % (filename, linenum, pkg, rel),
+                        file=sys.stderr,
+                    )
+
+                failed = 0
+                for alias in source[rel][pkg]['aliases']:
+                    if alias not in data["pkgs"].keys():
+                        filename = srcmap["pkgs"][pkg][nearby_rel][0]
+                        linenum = srcmap["pkgs"][pkg][nearby_rel][1]
+                        print(
+                            "%s: %d: %s missing release '%s'"
+                            # put the error on a line near where this entry should go
+                            % (filename, linenum, alias, rel),
+                            file=sys.stderr,
+                        )
+                        cve_okay = False
+                        failed += 1
+
+        missing_releases = all_required_releases - listed_releases - aliases_releases
         nearby_rel = list(listed_releases - missing_releases)[0]
         for rel in missing_releases:
             # only warn on active CVEs
diff --git a/scripts/check-syntax-fixup b/scripts/check-syntax-fixup
index e6b77f9..c7c77c2 100755
--- a/scripts/check-syntax-fixup
+++ b/scripts/check-syntax-fixup
@@ -166,6 +166,14 @@ for line in args.infile:
     elif "unknown package" in msg or "not in" in msg or "unknown release" in msg:
         pkg, rel = get_pkg_rel_from_msg(msg)
 
+        # remove this hard-coded hack one-day...
+        if rel in cve_lib.external_releases or rel == "trusty/esm":
+            cve = os.path.join(
+                cve_lib.get_external_subproject_cve_dir(rel), os.path.basename(cve)
+            )
+            # linenum is only relevant to the original cve file
+            linenum = 1
+
         # delete this line since
         delete_from_file(cve, linenum, args.dry_run, args.verbose)
 
diff --git a/scripts/cve_lib.py b/scripts/cve_lib.py
index d8d6bf6..82ff330 100755
--- a/scripts/cve_lib.py
+++ b/scripts/cve_lib.py
@@ -804,6 +804,12 @@ def load_external_subprojects():
                                      "eol": False})
         # an external subproject can append to an internal one
         subprojects[rel]["packages"].append(supported_txt)
+
+        # check if aliases for packages exist
+        if os.path.isfile(f'{supported_txt[:-len("supported.txt")]}aliases.yaml'):
+            subprojects[rel].setdefault("aliases",
+                f'{supported_txt[:-len("supported.txt")]}aliases.yaml')
+
         try:
             # use config to populate other parts of the
             # subproject settings
diff --git a/scripts/source_map.py b/scripts/source_map.py
index 741058b..d4044f3 100755
--- a/scripts/source_map.py
+++ b/scripts/source_map.py
@@ -17,6 +17,7 @@ import re
 import subprocess
 import sys
 import cve_lib
+import yaml
 
 apt_pkg.init_system()
 
@@ -465,6 +466,19 @@ def load_subprojects_lists(releases=None):
                     map[rel][pkg] = dict()
                     map[rel][pkg]['pocket'] = ''
                     map[rel][pkg]['section'] = 'main'
+
+            if 'aliases' in details:
+                with open(details['aliases'], 'r') as file:
+                    aliases = yaml.safe_load(file)
+
+                    for pkg in aliases:
+                        if map[rel][pkg]:
+                            map[rel][pkg]['aliases'] = aliases[pkg]
+                        else:
+                            print("WARN: pkg %s found in aliases but not in supported. Skipping" % pkg)
+            else:
+                pass
+
     return map