Merge ~ubuntu-release/britney/+git/britney2-ubuntu:sil2100/source-and-binaries into ~ubuntu-release/britney/+git/britney2-ubuntu:sil2100/security-britney

[Łukasz Zemczak]

Britney operates on source packages, but cloud-test-framework on binary packages. Support listing which binaries we want to test per-source.

Britney operates on source packages, but cloud-test-framework on binary packages. Support listing which binaries we want to test per-source.

This is still WIP, but submitting this for discussion. The cloud_package_set file would need to be updated.

But question is: does cloud-test-framework support calls with multiple --install-ppa-package lines?

[John Chittum (jchittum)]

review: Approve

[Aleksa Svitlica (aleksa-svitlica)]

Will take a look at the rest of the MP soon but yes cloud-test-framework does support multiple --install-ppa-package lines

Unmerged commits

4a4227a... by Łukasz Zemczak on 2023-03-03

Britney operates on source packages, but cloud-test-framework on binary packages. Support listing which binaries we want to test per-source.

diff --git a/britney2/policies/cloud.py b/britney2/policies/cloud.py
index 56fa755..36ed4ba 100644
--- a/britney2/policies/cloud.py
+++ b/britney2/policies/cloud.py
@@ -87,6 +87,7 @@ class CloudPolicy(BasePolicy):
         self.failures = {}
         self.errors = {}
         self.email_needed = False
+        self.package_set = {}
 
     def initialise(self, britney):
         super().initialise(britney)
@@ -96,7 +97,7 @@ class CloudPolicy(BasePolicy):
 
     def apply_src_policy_impl(self, policy_info, item, source_data_tdist, source_data_srcdist, excuse):
         self.logger.info("Cloud Policy: Looking at {}".format(item.package))
-        if item.package not in self.package_set:
+        if item.package not in self.package_set or not self.package_set[item.package]:
             return PolicyVerdict.PASS
 
         if self.dry_run:
@@ -215,7 +216,9 @@ class CloudPolicy(BasePolicy):
 
         with open(self.PACKAGE_SET_FILE) as file:
             for line in file:
-                package_set.add(line.strip())
+                source, binaries = line.strip().lsplit(': ')
+                binaries = binaries.split(' ')
+                package_set[source] = binaries
 
         return package_set
 
@@ -275,7 +278,10 @@ class CloudPolicy(BasePolicy):
             "/snap/bin/cloud-test-framework",
             "--instance-prefix", "britney-{}-{}".format(package, series)
         ]
-        params.extend(self._format_install_flags(package, sources, source_type))
+        # Britney operates on source packages, but we actually want to test all the relevant binaries
+        self.logger.info("Cloud Policy: The following binary packages will be tested: {}".format(self.package_set[package]))
+        for binary in self.package_set[package]:
+            params.extend(self._format_install_flags(binary, sources, source_type))
         params.extend(
             [
                 "azure_gen2",
diff --git a/tests/test_cloud.py b/tests/test_cloud.py
index 8129cef..679f6d0 100644
--- a/tests/test_cloud.py
+++ b/tests/test_cloud.py
@@ -44,6 +44,19 @@ class T(unittest.TestCase):
     def tearDown(self):
         self.policy._cleanup_work_directory()
 
+    @patch("britney2.policies.cloud.CloudPolicy._format_install_flags")
+    @patch("britney2.policies.cloud.CloudPolicy._store_extra_test_result_info")
+    @patch("britney2.policies.cloud.CloudPolicy._parse_xunit_test_results")
+    @patch("subprocess.run")
+    def test_run_cloud_test_multiple_binaries(self, mock_run, mock_xunit, mock_extra, mock_flags):
+        """Cloud tests should run for multiple binaries in a package."""
+        self.policy.failures = {}
+        self.policy.errors = {}
+        self.policy.state = {}
+        self.policy.package_set = {"hello": ["hello", "world"]}
+        self.policy._run_cloud_tests("hello", "2.10", "zazzy", ["proposed"], "archive")
+        self.assertEqual(mock_flags.call_count, 2)
+
     @patch("britney2.policies.cloud.CloudPolicy._store_extra_test_result_info")
     @patch("britney2.policies.cloud.CloudPolicy._parse_xunit_test_results")
     @patch("subprocess.run")
@@ -70,6 +83,7 @@ class T(unittest.TestCase):
         # Package already tested, no tests should run
         self.policy.failures = {}
         self.policy.errors = {}
+        self.policy.package_set = {"hello": ["hello"], "chromium-browser": ["chromium-browser"]}
         self.policy._run_cloud_tests("chromium-browser", "55.0", "zazzy", ["proposed"], "archive")
         self.assertDictEqual(expected_state, self.policy.state)
         mock_run.assert_not_called()
@@ -130,6 +144,7 @@ class T(unittest.TestCase):
         with open(self.policy.options.cloud_state_file, "w") as file:
             json.dump(start_state, file)
         self.policy._load_state()
+        self.policy.package_set = {"hello": ["hello"], "chromium-browser": ["chromium-browser"]}
 
         # Package already tested, but only had errors - rerun
         self.policy._run_cloud_tests("chromium-browser", "55.0", "zazzy", ["proposed"], "archive")
@@ -139,7 +154,7 @@ class T(unittest.TestCase):
     def test_run_cloud_tests_called_for_package_in_manifest(self, mock_run):
         """Cloud tests should run for a package in the cloud package set.
         """
-        self.policy.package_set = set(["chromium-browser"])
+        self.policy.package_set = {"chromium-browser": ["chromium-browser"]}
         self.policy.options.series = "jammy"
 
         self.policy.apply_src_policy_impl(
@@ -154,7 +169,7 @@ class T(unittest.TestCase):
     def test_run_cloud_tests_not_called_for_package_not_in_manifest(self, mock_run):
         """Cloud tests should not run for packages not in the cloud package set"""
 
-        self.policy.package_set = set(["vim"])
+        self.policy.package_set = {"vim": ["vim"]}
         self.policy.options.series = "jammy"
 
         self.policy.apply_src_policy_impl(
@@ -167,7 +182,7 @@ class T(unittest.TestCase):
     @patch("britney2.policies.cloud.CloudPolicy._run_cloud_tests")
     def test_no_tests_run_during_dry_run(self, mock_run, smtp):
         self.policy = CloudPolicy(self.fake_options, {}, dry_run=True)
-        self.policy.package_set = set(["chromium-browser"])
+        self.policy.package_set = {"chromium-browser": ["chromium-browser"]}
         self.policy.options.series = "jammy"
         self.policy.source = "jammy-proposed"