Merge ~paride/utah:forward-file-lists into utah:master

[Paride Legovini]

Make release-specific file lists apply to future releases

Static validation file lists specific for a given release are allowed
in the form of files named with the scheme:

  file_list.<release>

This change makes these lists apply for <release> and for all the
releases that follow <release>, unless a newer release-specific file
exists, in which case that's the one that applies.

[Michael Hudson-Doyle (mwhudson)]

The idea seems fine but I think there's a logic bug hiding.

review: Needs Fixing

[Paride Legovini (paride)]

review: Needs Fixing

[Paride Legovini (paride)]

MP updated with fixed logic and helper function.

[Michael Hudson-Doyle (mwhudson)]

LGTM but i'm mostly asleep :)

review: Approve

diff --git a/utah/isotest/data/file_list_subiquity.groovy b/utah/isotest/data/file_list_subiquity.groovy
new file mode 100644
index 0000000..43f9991
--- /dev/null
+++ b/utah/isotest/data/file_list_subiquity.groovy
@@ -0,0 +1,21 @@
+.disk
+.disk/info
+EFI
+EFI/boot
+EFI/boot/bootx64.efi
+EFI/boot/grubx64.efi
+README.diskdefines
+boot
+boot/grub
+casper
+casper/filesystem.manifest
+casper/filesystem.squashfs
+casper/filesystem.squashfs.gpg
+casper/installer.squashfs
+casper/vmlinuz
+dists
+install
+md5sum.txt
+pics
+pool
+ubuntu
diff --git a/utah/isotest/data/file_list_ubiquity b/utah/isotest/data/file_list_ubiquity
index f7c6306..64c12ea 100644
--- a/utah/isotest/data/file_list_ubiquity
+++ b/utah/isotest/data/file_list_ubiquity
@@ -3,6 +3,7 @@ casper
 .disk
 dists
 install
+isolinux
 md5sum.txt
 pics
 pool
diff --git a/utah/isotest/data/file_list_ubiquity.groovy b/utah/isotest/data/file_list_ubiquity.groovy
new file mode 100644
index 0000000..f7c6306
--- /dev/null
+++ b/utah/isotest/data/file_list_ubiquity.groovy
@@ -0,0 +1,18 @@
+boot
+casper
+.disk
+dists
+install
+md5sum.txt
+pics
+pool
+preseed
+README.diskdefines
+ubuntu
+casper
+casper/filesystem.squashfs
+casper/vmlinuz
+casper/initrd
+casper/filesystem.manifest
+casper/filesystem.manifest-remove
+casper/filesystem.size
diff --git a/utah/isotest/data/file_list_uefi.groovy b/utah/isotest/data/file_list_uefi.groovy
new file mode 100644
index 0000000..d46843d
--- /dev/null
+++ b/utah/isotest/data/file_list_uefi.groovy
@@ -0,0 +1,3 @@
+EFI/boot/bootx64.efi
+EFI/boot/grubx64.efi
+boot/grub/x86_64-efi
diff --git a/utah/isotest/iso_static_validation.py b/utah/isotest/iso_static_validation.py
index 5c101c2..4480d33 100755
--- a/utah/isotest/iso_static_validation.py
+++ b/utah/isotest/iso_static_validation.py
@@ -58,6 +58,7 @@ import re
 import tempfile
 import shutil
 import time
+import glob
 import platform
 import distro_info
 from traceback import format_exception
@@ -211,6 +212,23 @@ class TestValidateISO(unittest.TestCase):
         else:
             self.fail("Image name be in the form of release-variant-arch.iso")
 
+    def get_latest_file_list(self, filelist):
+        file_list_path = os.path.join(DATA_PATH, filelist)
+        relspec_flists = glob.glob(file_list_path + ".*")
+
+        if not relspec_flists:
+            # No release-specific file lists available
+            return file_list_path
+
+        relindex = distro_info.UbuntuDistroInfo().all.index(self.st_release)
+        releases = distro_info.UbuntuDistroInfo().all[:relindex+1]
+        for release in reversed(releases):
+            new_file_list_path = file_list_path + "." + release
+            if new_file_list_path in relspec_flists:
+                return new_file_list_path
+
+        return file_list_path
+
     def test_sha256_checksum(self):
         """Test if the ISO SHA checksum matches what is given in the server."""
         current_iso_digest = {}
@@ -347,13 +365,11 @@ class TestValidateISO(unittest.TestCase):
         (stdout, stderr) = self.iso.listfiles()
         logging.debug('Check for error in extracting file list from the iso')
         self.assertEqual(stderr, '')
-        files_list_path = os.path.join(DATA_PATH, 'file_list_ubiquity')
-        if os.path.exists(files_list_path + '.' + self.st_release):
-            files_list_path = files_list_path + '.' + self.st_release
-        files_list = open(os.path.join(files_list_path))
+        file_list_path = self.get_latest_file_list("file_list_ubiquity")
+        file_list = open(file_list_path)
         exclude_files = []
         logging.debug('Check if relevant files are present in the iso')
-        for list_server in files_list:
+        for list_server in file_list:
             path = list_server.rstrip()
             if path in exclude_files:
                 logging.debug('%s excluded based on release', path)
@@ -398,23 +414,18 @@ class TestValidateISO(unittest.TestCase):
         logging.debug('Check for error in extracting file list from the iso')
         self.assertEqual(stderr, '')
 
+        file_list_fname = "file_list_subiquity"
         if self.st_arch == 'ppc64el':
-            files_list = open(os.path.join(DATA_PATH,
-                              'file_list_subiquity_powerpc'))
+            file_list_fname = file_list_fname + '_powerpc'
         elif self.st_arch == 's390x':
-            files_list = open(os.path.join(DATA_PATH,
-                              'file_list_subiquity_s390x'))
+            file_list_fname = file_list_fname + '_s390x'
         elif self.st_arch == 'arm64':
-            files_list = open(os.path.join(DATA_PATH,
-                              'file_list_subiquity_arm64'))
-        else:
-            files_list_path = os.path.join(DATA_PATH, 'file_list_subiquity')
-            if os.path.exists(files_list_path + '.' + self.st_release):
-                files_list_path = files_list_path + '.' + self.st_release
-            files_list = open(os.path.join(files_list_path))
+            file_list_fname = file_list_fname + '_arm64'
 
-        for list_server in files_list:
-            logging.debug('check if important d-i files are present in iso')
+        file_list_path = self.get_latest_file_list(file_list_fname)
+        file_list = open(file_list_path)
+        for list_server in file_list:
+            logging.debug('check if important live-server files are present in iso')
             path = list_server.rstrip()
             self.assertIn(path, stdout)
 
@@ -495,13 +506,13 @@ class TestValidateISO(unittest.TestCase):
 
     @unittest.skipUnless(uefi_image,
                          "Test EFI files on ubuntu amd64 images")
-    @unittest.skipIf(st_release == 'groovy', "no EFI/BOOT LP1895131")
     def test_efi_files_present(self):
         """Test if the relevant files are present in the ISO for UEFI."""
         (stdout, stderr) = self.iso.listfiles()
         logging.debug('Check for error in extracting file list from the iso')
         self.assertEqual(stderr, '')
-        files_list = open(os.path.join(DATA_PATH, 'file_list_uefi'))
+        file_list_path = self.get_latest_file_list("file_list_uefi")
+        files_list = open(file_list_path)
         logging.debug('Check if relevant files are present in the iso')
         for list_server in files_list:
             path = list_server.rstrip()
@@ -509,7 +520,6 @@ class TestValidateISO(unittest.TestCase):
 
     @unittest.skipUnless(uefi_image,
                          "Test Secure Boot UEFI signatures")
-    @unittest.skipIf(st_release == 'groovy', "no EFI/BOOT LP1895131")
     def test_efi_secure_boot_signatures(self):
         logging.debug('Generating verification certificates')
         repo_dir = os.path.expanduser('~/repo_dir')
@@ -539,8 +549,18 @@ class TestValidateISO(unittest.TestCase):
             keys, 'canonical-master-signing-public-chain.pem')
 
         logging.debug('Extracting UEFI boot and kernel images')
-        shim = self.iso.extract('./EFI/BOOT/BOOTx64.EFI', self.temp_dir)
-        grub = self.iso.extract('./EFI/BOOT/grubx64.efi', self.temp_dir)
+
+        udi_all = distro_info.UbuntuDistroInfo().all
+
+        shimfile = './EFI/boot/bootx64.efi'
+        grubfile = './EFI/boot/grubx64.efi'
+        if udi_all.index(self.st_release) < udi_all.index("groovy"):
+            # Pre-groovy releases had uppercase EFI files
+            shimfile = './EFI/BOOT/BOOTx64.EFI'
+            grubfile = './EFI/BOOT/grubx64.efi'
+
+        shim = self.iso.extract(shimfile, self.temp_dir)
+        grub = self.iso.extract(grubfile, self.temp_dir)
         kernel = self.iso.extract(iso.kernelpath(), self.temp_dir)
 
         logging.debug('Verifying UEFI shim')