Merge ppa-dev-tools:add_dependent_packages_function into ppa-dev-tools:main

[Bryce Harrington]

This branch finally implements the core mechanics of this feature, the mapping of provided binary packages of some source packages to the reverse dependencies of others. The dependent_packages() API is introduced to provide a way to look up for a given source packages, the other packages that should be tested along with it.

I mulled over a few different ideas for the API name before setting on "dependent_packages()" but am still unsure that's the best name. Alternative ideas would be welcome, and I reserve the option to rename this later if inspiration strikes. :-)

This API will be used to generate all the various trigger URLs. I wanted to include a smoketest to showcase this, but this is already plenty to review.

This branch just introduces the core data structure and basic lookups. It doesn't handle recursive lookups and some corner cases like packages with no build dependencies, but those enhancements should be straightforward modifications, and will be tackled in a future MP.

[Athos Ribeiro (athos-ribeiro)]

LGTM. Thanks, Bryce!

review: Approve

[Bryce Harrington (bryce)]

Thanks, landed:

To git+ssh://git.launchpad.net/ppa-dev-tools
   100b629..272c487 main -> main

diff --git a/ppa/suite.py b/ppa/suite.py
index 72eb8a6..4812f5d 100644
--- a/ppa/suite.py
+++ b/ppa/suite.py
@@ -43,6 +43,8 @@ class Suite:
 
         self._suite_name = suite_name
         self._cache_dir = cache_dir
+        self._provides_table = None
+        self._rdepends_table = None
 
     def __repr__(self) -> str:
         """Machine-parsable unique representation of object.
@@ -64,6 +66,23 @@ class Suite:
         """
         return f'{self._suite_name}'
 
+    def _rebuild_lookup_tables(self) -> bool:
+        """Regenerates the provides and rdepends lookup tables.
+
+        :rtype: bool
+        :returns: True if tables were rebuilt, False otherwise"""
+        self._provides_table = {}
+        self._rdepends_table = {}
+        for source_name, source in self.sources.items():
+            print(source_name, source)
+            for build_dep_binary_name in source.build_dependencies.keys():
+                self._rdepends_table.setdefault(build_dep_binary_name, [])
+                self._rdepends_table[build_dep_binary_name].append(source)
+
+            for provided_binary_name in source.provides_binaries.keys():
+                self._provides_table[provided_binary_name] = source
+        return self._provides_table and self._rdepends_table
+
     @property
     @lru_cache
     def info(self) -> dict[str, str]:
@@ -184,6 +203,42 @@ class Suite:
             raise ValueError(f'Could not load {binary_packages_dir}/Packages.xz')
         return binaries
 
+    def dependent_packages(self, source_package: SourcePackage) -> dict[str, SourcePackage]:
+        """Relevant packages to run autotests against for a given source package.
+
+        Calculates the collection of build and reverse dependencies for
+        a given source package, that would be appropriate to re-run autopkgtests
+        on, using the given @param source_package's name as a trigger.
+
+        :param str source_package_name: The archive name of the source package.
+        :rtype: dict[str, SourcePackage]
+        :returns: Collection of source packages, keyed by name.
+        """
+        # Build the lookup table for provides and rdepends
+        if not self._provides_table or not self._rdepends_table:
+            if not self._rebuild_lookup_tables():
+                raise RuntimeError("Could not regenerate provides/rdepends lookup tables")
+
+        dependencies = {}
+
+        # Get source packages that depend on things we supply
+        for binary_package_name in source_package.provides_binaries.keys():
+            rdeps = self._rdepends_table.get(binary_package_name)
+            if rdeps:
+                for rdep_source in rdeps:
+                    dependencies[rdep_source.name] = rdep_source
+
+        # Get source packages that provide our build dependencies
+        for build_dependency_name in source_package.build_dependencies.keys():
+            bdep_source = self._provides_table.get(build_dependency_name)
+            if not bdep_source:
+                raise RuntimeError(f'Could not get source object for bdep {build_dependency_name}')
+            dependencies[bdep_source.name] = bdep_source
+
+        if not dependencies:
+            raise RuntimeError(f'Could not calculate dependencies for {source_package.name}')
+        return dependencies
+
 
 if __name__ == '__main__':
     # pylint: disable=invalid-name
diff --git a/tests/test_suite.py b/tests/test_suite.py
index 0203ecc..0dd430d 100644
--- a/tests/test_suite.py
+++ b/tests/test_suite.py
@@ -8,6 +8,8 @@
 # Released under GNU GPLv2 or later, read the file 'LICENSE.GPLv2+' for
 # more information.
 
+# pylint: disable=protected-access,line-too-long
+
 """Tests the Suite class as an interface to Apt suite records."""
 
 import os
@@ -367,3 +369,207 @@ def test_binaries(tmp_path, architectures, packages_contents, expected_binaries)
     # Create the suite to wrapper our path and access Packages.xz
     suite = Suite(suite_dir.name, suite_dir)
     assert sorted(suite.binaries.keys()) == sorted(expected_binaries.keys())
+
+
[e-mail]('sources, expected_rdepends, expected_provides', [
+    ({}, [], []),
+    (
+        {'a': SourcePackage({'Package': 'a', 'Version': 'x', 'Build-Depends': 'a', 'Binary': 'x'})},
+        ['a'],
+        ['x']
+    ),
+    (
+        {'a': SourcePackage({'Package': 'a', 'Version': 'x', 'Build-Depends': 'a, b, c', 'Binary': 'x'})},
+        ['a', 'b', 'c'],
+        ['x']
+    ),
+    (
+        {
+            'a': SourcePackage({'Package': 'a', 'Version': 'x', 'Binary': 'a1'}),
+            'b': SourcePackage({'Package': 'b', 'Version': 'x', 'Build-Depends': 'a1', 'Binary': 'b1'}),
+            'c': SourcePackage({'Package': 'c', 'Version': 'x', 'Build-Depends': 'a1, b1', 'Binary': 'c1'}),
+        },
+        ['a1', 'b1'],
+        ['a1', 'b1', 'c1']
+    ),
+    (
+        {'a': SourcePackage({'Package': 'a', 'Version': 'x', 'Binary': 'a'})},
+        [],
+        ['a']),
+    (
+        {'a': SourcePackage({'Package': 'a', 'Version': 'x', 'Binary': 'a, b, c'})},
+        [],
+        ['a', 'b', 'c']
+    ),
+    (
+        {
+            'dovecot': SourcePackage(
+                {
+                    'Package': 'dovecot',
+                    'Version': '1:2.3.19.1+dfsg1-2ubuntu4',
+                    'Binary': 'dovecot-core, dovecot-dev, dovecot-imapd, dovecot-pop3d, dovecot-lmtpd, dovecot-managesieved, dovecot-pgsql, dovecot-mysql, dovecot-sqlite, dovecot-ldap, dovecot-gssapi, dovecot-sieve, dovecot-solr, dovecot-lucene, dovecot-submissiond, dovecot-auth-lua',  # noqa: E501
+                    'Architecture': 'any',
+                    'Build-Depends': 'debhelper-compat (= 13), default-libmysqlclient-dev, krb5-multidev, libapparmor-dev [linux-any], libbz2-dev, libcap-dev [linux-any], libclucene-dev, libdb-dev, libexpat-dev, libexttextcat-dev, libicu-dev, libldap2-dev, liblua5.3-dev, liblz4-dev, liblzma-dev, libpam0g-dev, libpq-dev, libsasl2-dev, libsodium-dev, libsqlite3-dev, libssl-dev, libstemmer-dev, libsystemd-dev [linux-any], libunwind-dev [amd64 arm64 armel armhf hppa i386 ia64 mips mips64 mips64el mipsel powerpc powerpcspe ppc64 ppc64el sh4], libwrap0-dev, libzstd-dev, lsb-release, pkg-config, zlib1g-dev',  # noqa: E501
+                    'Testsuite-Triggers': 'lsb-release, python3, systemd-sysv',
+                }
+            )
+        },
+        [
+            'debhelper-compat',
+            'default-libmysqlclient-dev',
+            'krb5-multidev',
+            'libapparmor-dev',
+            'libbz2-dev',
+            'libcap-dev',
+            'libclucene-dev',
+            'libdb-dev',
+            'libexpat-dev',
+            'libexttextcat-dev',
+            'libicu-dev',
+            'libldap2-dev',
+            'liblua5.3-dev',
+            'liblz4-dev',
+            'liblzma-dev',
+            'libpam0g-dev',
+            'libpq-dev',
+            'libsasl2-dev',
+            'libsodium-dev',
+            'libsqlite3-dev',
+            'libssl-dev',
+            'libstemmer-dev',
+            'libsystemd-dev',
+            'libunwind-dev',
+            'libwrap0-dev',
+            'libzstd-dev',
+            'lsb-release',
+            'pkg-config',
+            'zlib1g-dev',
+        ],
+        [
+            'dovecot-core',
+            'dovecot-dev',
+            'dovecot-imapd',
+            'dovecot-pop3d',
+            'dovecot-lmtpd',
+            'dovecot-managesieved',
+            'dovecot-pgsql',
+            'dovecot-mysql',
+            'dovecot-sqlite',
+            'dovecot-ldap',
+            'dovecot-gssapi',
+            'dovecot-sieve',
+            'dovecot-solr',
+            'dovecot-lucene',
+            'dovecot-submissiond',
+            'dovecot-auth-lua',
+        ]
+    ),
+])
+def test_rebuild_tables(monkeypatch, sources, expected_rdepends, expected_provides):
+    """Checks generation of the internal lookup tables for provides and rdepends."""
+    monkeypatch.setattr(Suite, "sources", sources)
+    # Verify provides and rdepends table are as expected
+    suite = Suite('x', '/tmp')
+    suite._rebuild_lookup_tables()
+
+    assert sorted(suite._rdepends_table.keys()) == sorted(expected_rdepends)
+    assert sorted(suite._provides_table.keys()) == sorted(expected_provides)
+
+
+def test_rebuild_tables_mapping(monkeypatch):
+    """Checks the mapping of rdepends to provides in the generated tables.
+
+    The two lookup tables are essential to the rdepends test functionality
+    since they define the mappings between various source package provides
+    and depends.  This test builds a synthetic collection of source packages,
+    generates the tables, and then verifies the tables can be used to lookup
+    the appropriate related packages.
+
+    For purposes of this test, we assume each source package provides
+    binaries of the same name appended with either '1' or '2'.
+
+    Also, note that the packages are set up with a circular dependency
+    (a depends on c, but c depends on a).  This is an unhealthy
+    situation for an archive to be in, but it certainly does happen in
+    the wild.  We're just setting it up that way for convenience since
+    we can then assume all provided binaries will be required by
+    something in the archive.
+    """
+    sources = {
+        'a': SourcePackage({'Package': 'a', 'Version': 'x', 'Build-Depends': 'c1', 'Binary': 'a1'}),
+        'b': SourcePackage({'Package': 'b', 'Version': 'x', 'Build-Depends': 'a1, c2', 'Binary': 'b1'}),
+        'c': SourcePackage({'Package': 'c', 'Version': 'x', 'Build-Depends': 'a1, b1', 'Binary': 'c1, c2'}),
+    }
+    monkeypatch.setattr(Suite, "sources", sources)
+    suite = Suite('x', '/tmp')
+    suite._rebuild_lookup_tables()
+
+    # Check the integrity of the lookup tables for the sources we gave it
+    for source in sources.values():
+        # Verify our dependency is satisfied by a SourcePackage in the collection
+        for dependency in source.build_dependencies:
+            assert dependency in suite._provides_table
+            package = suite._provides_table[dependency]
+            assert isinstance(package, SourcePackage)
+            assert dependency in [f"{package.name}1", f"{package.name}2"]
+
+        # Verify SourcePackages that depend on us can be located
+        for binary in source.provides_binaries:
+            assert binary in suite._rdepends_table
+            for package in suite._rdepends_table[binary]:
+                assert isinstance(package, SourcePackage)
+                assert binary in package.build_dependencies
+
+
[e-mail]('sources_dict, source_package_name, expected_packages', [
+    pytest.param(
+        [{'Package': 'a', 'Version': 'x', 'Binary': 'a1'}], 'a', [],
+        marks=pytest.mark.xfail(reason='Does not handle undefined build-depends'),
+    ),
+    pytest.param(
+        [{'Package': 'a', 'Version': 'x', 'Build-Depends': None, 'Binary': 'a1'}], 'a', [],
+        marks=pytest.mark.xfail(reason='Does not handle undefined build-depends'),
+    ),
+    pytest.param(
+        [{'Package': 'a', 'Version': 'x', 'Build-Depends': '', 'Binary': 'a1'}], 'a', [],
+        marks=pytest.mark.xfail(reason='Does not handle undefined build-depends'),
+    ),
+
+    (
+        [
+            {'Package': 'a', 'Version': 'x', 'Build-Depends': 'b1', 'Binary': 'a1'},
+            {'Package': 'b', 'Version': 'x', 'Build-Depends': 'a1', 'Binary': 'b1'},
+            {'Package': 'c', 'Version': 'x', 'Build-Depends': 'b1', 'Binary': 'c1'},
+        ],
+        'a',
+        ['b'],
+    ),
+
+    (
+        [
+            {'Package': 'a', 'Version': 'x', 'Build-Depends': 'd1', 'Binary': 'a1'},
+            {'Package': 'b', 'Version': 'x', 'Build-Depends': 'a1', 'Binary': 'b1'},
+            {'Package': 'c', 'Version': 'x', 'Build-Depends': 'b1', 'Binary': 'c1, c2'},
+            {'Package': 'd', 'Version': 'x', 'Build-Depends': 'a1, b1, c2', 'Binary': 'd1'},
+        ],
+        'd',
+        ['a', 'b', 'c'],
+    ),
+])
+def test_dependent_packages(monkeypatch, sources_dict, source_package_name, expected_packages):
+    '''Checks that dependent_packages() returns the right packages to test.
+
+    This member function is the main API for looking up what packages
+    should have autopkgtests run, triggered against our desired package.
+    '''
+    sources = {pkg['Package']: SourcePackage(pkg) for pkg in sources_dict}
+
+    monkeypatch.setattr(
+        Suite,
+        "sources",
+        sources
+    )
+    suite = Suite('x', '/tmp')
+    source_package = sources[source_package_name]
+
+    assert sorted(suite.dependent_packages(source_package)) == expected_packages