Merge ~alexmurray/ubuntu-cve-tracker/+git/ubuntu-cve-tracker:binary-versions into ubuntu-cve-tracker:master

[Alex Murray]

Ensure binary package versions listed in USNs are correct.

This resolves https://bugs.launchpad.net/ubuntu-cve-tracker/+bug/1841848 by having sis-changes export the actual binary package version from launchpad to a json file, and then having sis-generate-usn import this json file to override the source package version with this binary version when available.

A quick test doing a fake USN for libreoffice from the Ubuntu archive:

$ SRCPKG=libreoffice; $UCT/scripts/sis-changes --ppa ubuntu --download /tmp/pending $SRCPKG;
$ cd /tmp/pending
$ USN=100000-1; $UCT/scripts/sis-generate-usn $USN *.changes > ~/new-usn.sh.orig

$ cd $UCT
$ git checkout binary-versions # is this branch
$ cd /tmp/pending
$ USN=100000-1; $UCT/scripts/sis-generate-usn $USN *.changes > ~/new-usn.sh

$ diff ~/new-usn.sh.orig ~/new-usn.sh
< usn.py $DB $USN --release bionic --package fonts-opensymbol --binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package fonts-opensymbol --binary-version 2:102.10+LibO6.0.7-0ubuntu0.18.04.9
183,184c183,184
< usn.py $DB $USN --release bionic --package libreoffice-mysql-connector --binary-version 1:6.0.7-0ubuntu0.18.04.9
< usn.py $DB $USN --release bionic --package libreoffice-nlpsolver --binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package libreoffice-mysql-connector --binary-version 1.0.2+LibO6.0.7-0ubuntu0.18.04.9
> usn.py $DB $USN --release bionic --package libreoffice-nlpsolver --binary-version 0.9+LibO6.0.7-0ubuntu0.18.04.9
206c206
< usn.py $DB $USN --release bionic --package libreoffice-wiki-publisher --binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package libreoffice-wiki-publisher --binary-version 1.2.0+LibO6.0.7-0ubuntu0.18.04.9
211,212c211,212
< usn.py $DB $USN --release bionic --package uno-libs3 --binary-version 1:6.0.7-0ubuntu0.18.04.9
< usn.py $DB $USN --release bionic --package ure --binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package uno-libs3 --binary-version 6.0.7-0ubuntu0.18.04.9
> usn.py $DB $USN --release bionic --package ure --binary-version 6.0.7-0ubuntu0.18.04.9
215c215
< usn.py $DB $USN --release disco --package fonts-opensymbol --binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package fonts-opensymbol --binary-version 2:102.10+LibO6.2.6-0ubuntu0.19.04.1
244c244
< usn.py $DB $USN --release disco --package libreoffice-nlpsolver --binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package libreoffice-nlpsolver --binary-version 0.9+LibO6.2.6-0ubuntu0.19.04.1
271c271
< usn.py $DB $USN --release disco --package libreoffice-wiki-publisher --binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package libreoffice-wiki-publisher --binary-version 1.2.0+LibO6.2.6-0ubuntu0.19.04.1
276,277c276,277
< usn.py $DB $USN --release disco --package uno-libs3 --binary-version 1:6.2.6-0ubuntu0.19.04.1
< usn.py $DB $USN --release disco --package ure --binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package uno-libs3 --binary-version 6.2.6-0ubuntu0.19.04.1
> usn.py $DB $USN --release disco --package ure --binary-version 6.2.6-0ubuntu0.19.04.1
280c280
< usn.py $DB $USN --release xenial --package fonts-opensymbol --binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package fonts-opensymbol --binary-version 2:102.7+LibO5.1.6~rc2-0ubuntu1~xenial9
304c304
< usn.py $DB $USN --release xenial --package libreoffice-mysql-connector --binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package libreoffice-mysql-connector --binary-version 1.0.2+LibO5.1.6~rc2-0ubuntu1~xenial9
325c325
< usn.py $DB $USN --release xenial --package libreoffice-wiki-publisher --binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package libreoffice-wiki-publisher --binary-version 1.2.0+LibO5.1.6~rc2-0ubuntu1~xenial9
329,330c329,330
< usn.py $DB $USN --release xenial --package uno-libs3 --binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
< usn.py $DB $USN --release xenial --package ure --binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package uno-libs3 --binary-version 5.1.6~rc2-0ubuntu1~xenial9
> usn.py $DB $USN --release xenial --package ure --binary-version 5.1.6~rc2-0ubuntu1~xenial9
1045c1045
< usn.py $DB $USN --release bionic --package fonts-opensymbol --all-binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package fonts-opensymbol --all-binary-version 2:102.10+LibO6.0.7-0ubuntu0.18.04.9
1074,1075c1074,1075
< usn.py $DB $USN --release bionic --package libreoffice-mysql-connector --all-binary-version 1:6.0.7-0ubuntu0.18.04.9
< usn.py $DB $USN --release bionic --package libreoffice-nlpsolver --all-binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package libreoffice-mysql-connector --all-binary-version 1.0.2+LibO6.0.7-0ubuntu0.18.04.9
> usn.py $DB $USN --release bionic --package libreoffice-nlpsolver --all-binary-version 0.9+LibO6.0.7-0ubuntu0.18.04.9
1097c1097
< usn.py $DB $USN --release bionic --package libreoffice-wiki-publisher --all-binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package libreoffice-wiki-publisher --all-binary-version 1.2.0+LibO6.0.7-0ubuntu0.18.04.9
1102,1103c1102,1103
< usn.py $DB $USN --release bionic --package uno-libs3 --all-binary-version 1:6.0.7-0ubuntu0.18.04.9
< usn.py $DB $USN --release bionic --package ure --all-binary-version 1:6.0.7-0ubuntu0.18.04.9
---
> usn.py $DB $USN --release bionic --package uno-libs3 --all-binary-version 6.0.7-0ubuntu0.18.04.9
> usn.py $DB $USN --release bionic --package ure --all-binary-version 6.0.7-0ubuntu0.18.04.9
1106c1106
< usn.py $DB $USN --release disco --package fonts-opensymbol --all-binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package fonts-opensymbol --all-binary-version 2:102.10+LibO6.2.6-0ubuntu0.19.04.1
1135c1135
< usn.py $DB $USN --release disco --package libreoffice-nlpsolver --all-binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package libreoffice-nlpsolver --all-binary-version 0.9+LibO6.2.6-0ubuntu0.19.04.1
1162c1162
< usn.py $DB $USN --release disco --package libreoffice-wiki-publisher --all-binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package libreoffice-wiki-publisher --all-binary-version 1.2.0+LibO6.2.6-0ubuntu0.19.04.1
1167,1168c1167,1168
< usn.py $DB $USN --release disco --package uno-libs3 --all-binary-version 1:6.2.6-0ubuntu0.19.04.1
< usn.py $DB $USN --release disco --package ure --all-binary-version 1:6.2.6-0ubuntu0.19.04.1
---
> usn.py $DB $USN --release disco --package uno-libs3 --all-binary-version 6.2.6-0ubuntu0.19.04.1
> usn.py $DB $USN --release disco --package ure --all-binary-version 6.2.6-0ubuntu0.19.04.1
1171c1171
< usn.py $DB $USN --release xenial --package fonts-opensymbol --all-binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package fonts-opensymbol --all-binary-version 2:102.7+LibO5.1.6~rc2-0ubuntu1~xenial9
1195c1195
< usn.py $DB $USN --release xenial --package libreoffice-mysql-connector --all-binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package libreoffice-mysql-connector --all-binary-version 1.0.2+LibO5.1.6~rc2-0ubuntu1~xenial9
1216c1216
< usn.py $DB $USN --release xenial --package libreoffice-wiki-publisher --all-binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package libreoffice-wiki-publisher --all-binary-version 1.2.0+LibO5.1.6~rc2-0ubuntu1~xenial9
1220,1221c1220,1221
< usn.py $DB $USN --release xenial --package uno-libs3 --all-binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
< usn.py $DB $USN --release xenial --package ure --all-binary-version 1:5.1.6~rc2-0ubuntu1~xenial9
---
> usn.py $DB $USN --release xenial --package uno-libs3 --all-binary-version 5.1.6~rc2-0ubuntu1~xenial9
> usn.py $DB $USN --release xenial --package ure --all-binary-version 5.1.6~rc2-0ubuntu1~xenial9

[Jamie Strandboge (jdstrand)]

Thanks for this patch! A few comments inline.

review: Needs Fixing

[Seth Arnold (seth-arnold)]

Looks good to me; is the binaries.json stored in the /tmp/pending/ directory if following the wiki instructions for sis-changes and sis-generate-usn?

Thanks

de6e9f0... by Alex Murray on 2019-09-03

sis-generate-usn: Clarify comments and catch more specific exception

93d06bc... by Alex Murray on 2019-09-03

sis-changes: refactor binary version mapping for efficiency

Gather binary version info for all commands but only output on the changes
action so we don't have to query launchpad twice for this info.

[Alex Murray (alexmurray)]

@jdstrand - see additional commits which should address your comments.

[Jamie Strandboge (jdstrand)]

Thanks! LGTM

review: Approve

[Alex Murray (alexmurray)]

Thanks - merged as 7f96b9f7cb db5e98e15a and 55a0652ff5

diff --git a/scripts/sis-changes b/scripts/sis-changes
index e9bd8f5..be9e793 100755
--- a/scripts/sis-changes
+++ b/scripts/sis-changes
@@ -21,6 +21,7 @@ import sys
 import tempfile
 import urllib2
 import cve_lib
+import json
 from source_map import version_compare
 
 try:
@@ -207,7 +208,7 @@ def get_arch_from_dsc(dsc):
 
 
 # pkg -> { release, release -> { version } }
-def load_pkg_details_from_lp(pkgs, pkg, item):
+def load_pkg_details_from_lp(pkgs, binaries, pkg, item):
 
     rel = item.distro_series.name
     if opt.debug:
@@ -346,11 +347,20 @@ def load_pkg_details_from_lp(pkgs, pkg, item):
         else:
             raise ValueError("Unknown downloadable source file from %s %s '%s'" % (pkg, version, file_url))
 
-    # Check that all built binaries have actually published into the PPA
-    if opt.skip_build_check is True:
-        print("WARNING: skipping binary publication check. Please check manually.", file=sys.stderr)
-    else:
-        for binary in item.getPublishedBinaries():
+    binaries.setdefault(pkg, dict())
+    binaries[pkg].setdefault(rel, dict())
+    # map binaries as pkg -> rel -> binary -> version - this is used by
+    # sis-generate-usn to know what the actual binary package version is
+    # since this may be different than the version of the source
+    # package. Also check that all built binaries have actually published
+    # into the PPA
+    for binary in item.getPublishedBinaries():
+        # collect binary versions
+        binaries[pkg][rel].setdefault(binary.binary_package_name,
+                                      binary.binary_package_version)
+        if opt.skip_build_check is True:
+            print("WARNING: skipping binary publication check. Please check manually.", file=sys.stderr)
+        else:
             if binary.status != 'Published':
                 if opt.debug:
                     print("BinaryPublication(%s,%s,%s) state: %s" % (rel, binary.distro_arch_series.architecture_tag, binary.binary_package_name, binary.status), file=sys.stderr)
@@ -380,6 +390,7 @@ else:
 archive, group, ppa = lpl_common.get_archive(opt.ppa, lp, opt.debug, distribution=distribution)
 
 pkgs = dict()
+binaries = dict()
 if opt.superseded_version:
     status = "Superseded"
 else:
@@ -392,7 +403,7 @@ for pkg_name in args:
         params['pocket'] = opt.pocket
 
     for item in archive.getPublishedSources(**params):
-        load_pkg_details_from_lp(pkgs, pkg_name, item)
+        load_pkg_details_from_lp(pkgs, binaries, pkg_name, item)
 
 if opt.action == 'changes':
     for pkg in args:
@@ -439,6 +450,10 @@ if opt.action == 'changes':
                     continue
                 download_url(pkgs[pkg][rel][arch]['changes'])
 
+    # write out binaries as json so can be consumed by sis-generate-usn
+    with open(os.path.join(tmpdir, "binaries.json"), 'w') as fp:
+        json.dump(binaries, fp)
+
 elif opt.action == 'binaries':
     for pkg in args:
         if pkg not in pkgs:
diff --git a/scripts/sis-generate-usn b/scripts/sis-generate-usn
index e1ecd9f..15aa258 100755
--- a/scripts/sis-generate-usn
+++ b/scripts/sis-generate-usn
@@ -12,6 +12,7 @@
 from __future__ import print_function
 
 import apt_pkg
+import json
 import gzip
 import optparse
 import os
@@ -33,6 +34,7 @@ opter.add_option("--add-cves", metavar="CVES", help="Comma separated list of CVE
 opter.add_option("--ignore-cves", metavar="CVES", help="Comma separated list of CVEs to ignore when doing CVE autodetection.", default=None)
 opter.add_option("--embargoed", help="Include embargoed directory when looking for CVE descriptions", action='store_true')
 opter.add_option("--include-eol", help="Include EoL releases", action='store_true')
+opter.add_option("--binaries-json", help="Path to JSON mapping of binary packages to versions (default: binaries.json)", default="binaries.json")
 (opt, args) = opter.parse_args()
 
 if len(args) < 2:
@@ -457,6 +459,11 @@ usn = args[0]
 for changes in args[1:]:
     parse_changes(changes, info, CVEs, cvelines)
 
+# don't try and catch this exception if binaries.json is not available for
+# now since we want this to be noisy
+with open(opt.binaries_json) as fp:
+    binaries = json.load(fp)
+
 if opt.ignore_cves:
     CVEs.difference_update(set(opt.ignore_cves.split(',')))
 
@@ -747,6 +754,13 @@ for release in releases:
     for source in sorted(info[release].keys()):
         version = info[release][source]['version']
         for deb in sorted(info[release][source]['binaries']):
+            # binary packages can have different versions than their source
+            # package, so override the source package version with the
+            # binary package version if we have this available
+            try:
+                version = binaries[source][release][deb]
+            except KeyError as e:
+                pass
             if not filter_dbg.search(deb) and (not opt.filter_bins or opt.filter_bins.search(deb)):
                 print('  usn.py $DB $USN --release %s --package %s --binary-version %s' % (release,deb,version))
         print()
@@ -777,6 +791,13 @@ for release in releases:
     for source in sorted(info[release].keys()):
         version = info[release][source]['version']
         for deb in sorted(info[release][source]['binaries']):
+            # binary packages can have different versions than their source
+            # package, so override the source package version with the
+            # binary package version if we have this available
+            try:
+                version = binaries[source][release][deb]
+            except KeyError as e:
+                pass
             if not filter_dbg.search(deb) and (not opt.filter_bins or opt.filter_bins.search(deb)):
                 print('  usn.py $DB $USN --release %s --package %s --all-binary-version %s' % (release,deb,version))
         print()