Merge ~jslarraz/ubuntu-security-tools:jslarraz/umt_changelog into ubuntu-security-tools:master

[Jorge Sancho Larraz]

1) Adding support to umt changelog for ~0.XX.YY.1 and ~XX.YY.1 ending version names. 2) Enable changelog to suggest correct new version name when several releases are using the same original package version

The increment_version.py has been modified to consider that the original version may end in ~0.XX.YY.1 and ~XX.YY.1.
changelog has been modified to use rmadison to check whether other ubuntu releases are using the same package version to include the XX.YY in the proposed version when necessary.

[Alex Murray (alexmurray)]

Thanks for this @jslarraz - LGTM!

review: Approve

[Alex Murray (alexmurray)]

I like the idea of having umt changelog try and be more helpful but if possible can we add a test for this? (or could you include as a comment here what the new output is compared to the old behaviour so it is easier to review?)

[Jorge Sancho Larraz (jslarraz)]

I have added some test cases to verify (hopefully) the expected behavior. Please check not only the code looks good but also that test cases are correct.

[Eduardo Barretto (ebarretto)]

just some small comments in a specific case

review: Needs Fixing

[Eduardo Barretto (ebarretto)]

lgtm, thanks!

review: Approve

diff --git a/build-tools/umt b/build-tools/umt
index 2417175..3cdeee4 100755
--- a/build-tools/umt
+++ b/build-tools/umt
@@ -416,7 +416,8 @@ def cmd_changelog():
         if not note:
             note = "No-change re-build upload."
     else:
-        new_ver = increment_version_security(details['version'], opt.esm)
+        pkg_list = search_packages(details['package'], use_rmadison=True)   # Avoid dependencies on local installation
+        new_ver = increment_version_security(details['version'], opt.esm, details=details, pkg_list=pkg_list)
 
         if opt.repeat:
             repeat_release = opt.release
diff --git a/package-tools/increment_version.py b/package-tools/increment_version.py
index 6da79f6..7c48ca3 100644
--- a/package-tools/increment_version.py
+++ b/package-tools/increment_version.py
@@ -1,3 +1,11 @@
+# add location of source_map.py so this can be found since it is in UCT
+import os
+import sys
+uct_path = os.getenv("UCT")
+if uct_path is not None:
+    sys.path.append(os.path.join(uct_path, "scripts"))
+import cve_lib
+
 import re
 
 def _validate_version(old_version):
@@ -10,6 +18,68 @@ def _validate_version(old_version):
 
     return old_version
 
+def get_suffix(devel, details, pkg_list):
+    if devel:
+        suffix = "ubuntu1"
+    else:
+        suffix = "ubuntu0.1"
+        if (details is not None) and (pkg_list is not None):
+            release_id = get_release_id(details, pkg_list)
+            if release_id is not None:
+                suffix = "ubuntu0." + release_id + ".1"
+    return suffix
+
+def get_release_id(details, pkg_list):
+    release_id = None
+    if (details is None) or (pkg_list is None) or \
+            (False in [x in details for x in ['package', 'release']]):
+        return None
+
+    target_release = details['release']
+    old_version = pkg_list[target_release][0]
+
+    if (not 'ubuntu' in old_version) and (not 'esm' in old_version):
+        base_version = get_base_version(old_version)
+        for release in pkg_list:
+            # The same package version is used in several releases
+            if (
+                (cve_lib.get_orig_rel_name(release) in cve_lib.releases)
+                and (cve_lib.get_orig_rel_name(release) != target_release)
+                and (base_version == get_base_version(pkg_list[release][0]))
+                and (
+                    cve_lib.is_active_release(release)
+                    or (
+                        cve_lib.is_active_esm_release(release)
+                        and (
+                            is_srcpkg_supported(details['package'], cve_lib.get_esm_name(release, pkg_list[release][2]))
+                            or ('ubuntu' in pkg_list[release][0])
+                        )
+                    )
+                )
+            ):
+                release_name = cve_lib.subprojects['ubuntu/' + target_release]['name']
+                release_id = release_name.split(' ')[1]
+    return release_id
+
+def get_base_version(version):
+    if 'build' in version:
+        return version.split('build')[0]
+    elif 'ubuntu' in version:
+        return version.split('ubuntu')[0]
+    elif re.search("~([0-9]+\\.)?([0-9]{2}\\.[0-9]{2})(\\.[0-9]+)$", version):
+        return '~'.join(version.split('~')[:-1])
+    else:
+        return version
+
+def is_srcpkg_supported(srcpkg, esm_release):
+    file = esm_release.replace('/', '-') + '-supported.txt'
+    f = open(uct_path + '/' + file, 'r')
+    supported_packages = f.readlines()
+    f.close()
+    for supported_package in supported_packages:
+        if (srcpkg + '\n' == supported_package):
+            return True
+    return False
 
 def increment_version_build(old_version):
     """Returns an incremented version number per no-change build rules"""
@@ -31,14 +101,13 @@ def increment_version_build(old_version):
         new_version = old_version + "build1"
     return new_version
 
-
-def increment_version_security(old_version, esm=False, devel=False):
+def increment_version_security(old_version, esm=False, devel=False, details=None, pkg_list=None):
     """Returns an incremented version number per security update rules"""
     old_version = _validate_version(old_version)
 
     if "build" in old_version:
         # eg 2.0-2build1
-        suffix = "ubuntu0.1" if not devel else "ubuntu1"
+        suffix = get_suffix(devel, details, pkg_list)
         new_version = re.sub("build.*", "", old_version) + suffix
         if esm:
             new_version = new_version + "~esm1"
@@ -71,18 +140,27 @@ def increment_version_security(old_version, esm=False, devel=False):
             )
     elif not "-" in old_version or re.search("-[+a-z.0-9]+$", old_version):
         # eg 2.0-2 or 2.0cvs.20060707-2 or 1.5.0-3+cvs20071006
-        suffix = "ubuntu0.1" if not devel else "ubuntu1"
+        suffix = get_suffix(devel, details, pkg_list)
         new_version = old_version + suffix
         if esm:
             new_version = new_version + "~esm1"
-    elif re.search(r"~\d{2}\.\d{2}$", old_version):
-        # eg ending in ~YY.MM
+    elif re.search("~([0-9]+\\.)?([0-9]{2}\\.[0-9]{2})(\\.[0-9]+)$", old_version):
+        # eg. ~XX.YY ~XX.YY.1, ~0.XX.YY.1
         if devel:
             new_version = increment_version_security(old_version.split("~")[0], esm=esm, devel=devel)
         else:
-            new_version = old_version + ".1"
-            if esm:
-                new_version = new_version + "~esm1"
+            split = old_version.split(".")
+            if len(split) == 2:
+                new_version = old_version + ".1"
+                if esm:
+                    new_version = new_version + "~esm1"
+            else:
+                if esm:
+                    new_version = old_version + "+esm1"
+                else:
+                    new_version = re.sub(
+                        "\\.[0-9]+$", "." + str(int(split[len(split) - 1]) + 1), old_version
+                    )
     else:
         raise ValueError("Could not find version for %s" % old_version)
 
diff --git a/package-tools/tests/test_increment_version.py b/package-tools/tests/test_increment_version.py
 old mode 100644
 new mode 100755
index 5c49aa7..49dfa86
--- a/package-tools/tests/test_increment_version.py
+++ b/package-tools/tests/test_increment_version.py
@@ -5,11 +5,13 @@ from increment_version import increment_version_security
 class TestIncrementVersion(unittest.TestCase):
     def test_increment_version_security(self):
         class TC:
-            def __init__(self, version, expected, esm=False, devel=False):
+            def __init__(self, version, expected, esm=False, devel=False, details=None, pkg_list=None):
                 self.version = version
                 self.expected = expected
                 self.esm = esm
                 self.devel = devel
+                self.details = details
+                self.pkg_list = pkg_list
 
         test_cases = [
             TC("2.0-2", "2.0-2ubuntu0.1"),
@@ -38,10 +40,65 @@ class TestIncrementVersion(unittest.TestCase):
             TC("2:2.1-3ubuntu0.7.04.1", "2:2.1-3ubuntu1", devel=True),
             TC("2.0cvs.20060707-2", "2.0cvs.20060707-2ubuntu1", devel=True),
             TC("2.0-2ubuntu0.1~esm9", "2.0-2ubuntu0.1~esm10", esm=True),
+            TC("2.1.0-3build2~16.04.1", "2.1.0-3ubuntu0.1~esm1", esm=True),
+            # The exact same version is not used in other releases (debian build changes)
+            TC("4.3-1", "4.3-1ubuntu0.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'jammy': ['4.3-2', '', '']}),
+            # The exact same version is used in a debian release
+            TC("4.3-1", "4.3-1ubuntu0.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'stretch': ['4.3-1', '', '']}),
+            # The exact same version is used in other active release
+            TC("4.3-1", "4.3-1ubuntu0.20.04.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'jammy': ['4.3-1', '', '']}),
+            # The exact same version is used in other active release which was rebuilt
+            TC("4.3-1", "4.3-1ubuntu0.20.04.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'jammy': ['4.3-1build1', '', '']}),
+            # The exact same version is used in other active release which is an interim one
+            TC("4.3-1", "4.3-1ubuntu0.20.04.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'lunar': ['4.3-1', '', '']}),
+            # The exact same version is used in other release which is not active anymore
+            TC("4.3-1", "4.3-1ubuntu0.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'kinetic': ['4.3-1', '', '']}),
+            # The exact same version is used also in the development release
+            TC("4.3-1", "4.3-1ubuntu0.20.04.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'mantic': ['4.3-1', '', '']}),
+            # The exact same version is used in a LTS release which is out of standard support but the package is
+            # supported by esm
+            TC("4.3-1", "4.3-1ubuntu0.20.04.1", details={'package': 'busybox', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'trusty': ['4.3-1', '', '']}),
+            # The exact same version is used in a LTS release which is out of standard support, the package is not
+            # supported by esm and the package was not updated previously for this release
+            TC("4.3-1", "4.3-1ubuntu0.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'trusty': ['4.3-1', '', '']}),
+            # The exact same version is used in a LTS release which is out of standard support, the package is not
+            # supported by esm  but an update already exists for this release (may happen if target release
+            # is an interim one)
+            TC("4.3-1", "4.3-1ubuntu0.20.04.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'trusty/esm': ['4.3-1ubuntu0.16.04.1~esm1', '', '']}),
+            # The exact same version is used in a LTS release which is out of standard support, the package is not
+            # supported by esm but an update already exists for this release (this case should never happen
+            # as focal should also be patched already)
+            TC("4.3-1", "4.3-1ubuntu0.20.04.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'trusty': ['4.3-1ubuntu0.14.04.1', '', '']}),
+            # The exact same version is used only in other release which is the esm variant of the target one
+            TC("4.3-1", "4.3-1ubuntu0.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'esm-apps/focal': ['4.3-1ubuntu0.1~esm1', '', '']}),
+            # The exact some version is used in other release, both versions were backported
+            TC("4.3-1~20.04.1", "4.3-1~20.04.2", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1~20.04.1', '', ''], 'jammy': ['4.3-1~22.04.1', '', '']}),
+            # The exact some version is used in other release, the package was rebuild and then backported
+            TC("4.3-1build2~20.04.1", "4.3-1ubuntu0.20.04.1", details={'package': 'plib', 'release': 'focal'},
+               pkg_list={'focal': ['4.3-1build2~20.04.1', '', ''], 'jammy': ['4.3-1~22.04.1', '', '']}),
+            # Target release is an interim one, the exact same version is used in other active release
+            TC("4.3-1", "4.3-1ubuntu0.23.04.1", details={'package': 'plib', 'release': 'lunar'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'lunar': ['4.3-1', '', '']}),
+            # Target release is the development one, the exact same version is used in other active release
+            TC("4.3-1", "4.3-1ubuntu1", devel=True, details={'package': 'plib', 'release': 'mantic'},
+               pkg_list={'focal': ['4.3-1', '', ''], 'mantic': ['4.3-1', '', '']}),
         ]
         for tc in test_cases:
-            with self.subTest(version=tc.version, esm=tc.esm, devel=tc.devel):
+            with self.subTest(version=tc.version, esm=tc.esm, devel=tc.devel, details=tc.details, pkg_list=tc.pkg_list):
                 self.assertEqual(
-                    increment_version_security(tc.version, esm=tc.esm, devel=tc.devel),
+                    increment_version_security(tc.version, esm=tc.esm, devel=tc.devel, details=tc.details, pkg_list=tc.pkg_list),
                     tc.expected,
                 )