Merge ~cjwatson/launchpad:charmcraft-parser into launchpad:master

[Colin Watson]

Add a charmcraft.yaml parser

Parse the data from charmcraft.yaml and use it to determine which distroarchseries to dispatch builds to. This is somewhat similar to our snapcraft.yaml parser, but charmcraft.yaml specifies series (although it calls them "channels") as well as architectures, so we need to handle those too.

a24958d... by Colin Watson on 2021-06-03

Add a charmcraft.yaml parser

Parse the data from charmcraft.yaml and use it to determine which
distroarchseries to dispatch builds to. This is somewhat similar to our
snapcraft.yaml parser, but charmcraft.yaml specifies series (although it
calls them "channels") as well as architectures, so we need to handle
those too.

[Cristian Gonzalez (cristiangsp)]

review: Approve

[Ioana Lasc (ilasc)]

review: Approve

diff --git a/lib/lp/charms/adapters/__init__.py b/lib/lp/charms/adapters/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/charms/adapters/__init__.py
diff --git a/lib/lp/charms/adapters/buildarch.py b/lib/lp/charms/adapters/buildarch.py
new file mode 100644
index 0000000..5c76fa9
--- /dev/null
+++ b/lib/lp/charms/adapters/buildarch.py
@@ -0,0 +1,180 @@
+# Copyright 2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    "determine_instances_to_build",
+    ]
+
+from collections import (
+    Counter,
+    OrderedDict,
+    )
+import json
+
+import six
+
+from lp.services.helpers import english_list
+
+
+class CharmBasesParserError(Exception):
+    """Base class for all exceptions in this module."""
+
+
+class MissingPropertyError(CharmBasesParserError):
+    """Error for when an expected property is not present in the YAML."""
+
+    def __init__(self, prop):
+        super(MissingPropertyError, self).__init__(
+            "Base specification is missing the {!r} property".format(prop))
+        self.property = prop
+
+
+class BadPropertyError(CharmBasesParserError):
+    """Error for when a YAML property is malformed in some way."""
+
+
+class DuplicateRunOnError(CharmBasesParserError):
+    """Error for when multiple `run-on`s include the same architecture."""
+
+    def __init__(self, duplicates):
+        super(DuplicateRunOnError, self).__init__(
+            "{} {} present in the 'run-on' of multiple items".format(
+                english_list([str(d) for d in duplicates]),
+                "is" if len(duplicates) == 1 else "are"))
+
+
[e-mail]_2_unicode_compatible
+class CharmBase:
+    """A single base in charmcraft.yaml."""
+
+    def __init__(self, name, channel, architectures=None):
+        self.name = name
+        if not isinstance(channel, six.string_types):
+            raise BadPropertyError(
+                "Channel {!r} is not a string (missing quotes?)".format(
+                    channel))
+        self.channel = channel
+        self.architectures = architectures
+
+    @classmethod
+    def from_dict(cls, properties):
+        """Create a new base from a dict."""
+        try:
+            name = properties["name"]
+        except KeyError:
+            raise MissingPropertyError("name")
+        try:
+            channel = properties["channel"]
+        except KeyError:
+            raise MissingPropertyError("channel")
+        return cls(
+            name=name, channel=channel,
+            architectures=properties.get("architectures"))
+
+    def __eq__(self, other):
+        return (
+            self.name == other.name and
+            self.channel == other.channel and
+            self.architectures == other.architectures)
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        return hash((self.name, self.channel, tuple(self.architectures)))
+
+    def __str__(self):
+        return "{} {} {}".format(
+            self.name, self.channel, json.dumps(self.architectures))
+
+
+class CharmBaseConfiguration:
+    """A base configuration entry in charmcraft.yaml."""
+
+    def __init__(self, build_on, run_on=None):
+        self.build_on = build_on
+        self.run_on = list(build_on) if run_on is None else run_on
+
+    @classmethod
+    def from_dict(cls, properties):
+        """Create a new base configuration from a dict."""
+        # Expand short-form configuration into long-form.  Account for
+        # common typos in case the user intends to use long-form but did so
+        # incorrectly (for better error message handling).
+        if not any(
+                item in properties
+                for item in ("run-on", "run_on", "build-on", "build_on")):
+            base = CharmBase.from_dict(properties)
+            return cls([base], run_on=[base])
+
+        try:
+            build_on = properties["build-on"]
+        except KeyError:
+            raise MissingPropertyError("build-on")
+        build_on = [CharmBase.from_dict(item) for item in build_on]
+        run_on = properties.get("run-on")
+        if run_on is not None:
+            run_on = [CharmBase.from_dict(item) for item in run_on]
+        return cls(build_on, run_on=run_on)
+
+
+def determine_instances_to_build(charmcraft_data, supported_arches,
+                                 default_distro_series):
+    """Return a list of instances to build based on charmcraft.yaml.
+
+    :param charmcraft_data: A parsed charmcraft.yaml.
+    :param supported_arches: An ordered list of all `DistroArchSeries` that
+        we can create builds for.  Note that these may span multiple
+        `DistroSeries`.
+    :param default_distro_series: The default `DistroSeries` to use if
+        charmcraft.yaml does not explicitly declare any bases.
+    :return: A list of `DistroArchSeries`.
+    """
+    bases_list = charmcraft_data.get("bases")
+
+    if bases_list:
+        configs = [
+            CharmBaseConfiguration.from_dict(item) for item in bases_list]
+    else:
+        # If no bases are specified, build one for each supported
+        # architecture for the default series.
+        configs = [
+            CharmBaseConfiguration([
+                CharmBase(
+                    default_distro_series.distribution.name,
+                    default_distro_series.version, das.architecturetag),
+                ])
+            for das in supported_arches
+            if das.distroseries == default_distro_series]
+
+    # Ensure that multiple `run-on` items don't overlap; this is ambiguous
+    # and forbidden by charmcraft.
+    run_ons = Counter()
+    for config in configs:
+        run_ons.update(config.run_on)
+    duplicates = {config for config, count in run_ons.items() if count > 1}
+    if duplicates:
+        raise DuplicateRunOnError(duplicates)
+
+    instances = OrderedDict()
+    for config in configs:
+        # Charms are allowed to declare that they build on architectures
+        # that Launchpad doesn't currently support (perhaps they're
+        # upcoming, or perhaps they used to be supported).  We just ignore
+        # those.
+        for build_on in config.build_on:
+            for das in supported_arches:
+                if (das.distroseries.distribution.name == build_on.name and
+                        build_on.channel in (
+                            das.distroseries.name,
+                            das.distroseries.version) and
+                        das.architecturetag in build_on.architectures):
+                    instances[das] = None
+                    break
+            else:
+                continue
+            break
+    return list(instances)
diff --git a/lib/lp/charms/adapters/tests/__init__.py b/lib/lp/charms/adapters/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/lib/lp/charms/adapters/tests/__init__.py
diff --git a/lib/lp/charms/adapters/tests/test_buildarch.py b/lib/lp/charms/adapters/tests/test_buildarch.py
new file mode 100644
index 0000000..7511f67
--- /dev/null
+++ b/lib/lp/charms/adapters/tests/test_buildarch.py
@@ -0,0 +1,312 @@
+# Copyright 2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+from functools import partial
+
+from testscenarios import (
+    load_tests_apply_scenarios,
+    WithScenarios,
+    )
+from testtools.matchers import (
+    Equals,
+    MatchesException,
+    MatchesListwise,
+    MatchesStructure,
+    Raises,
+    )
+from zope.component import getUtility
+
+from lp.app.interfaces.launchpad import ILaunchpadCelebrities
+from lp.buildmaster.interfaces.processor import (
+    IProcessorSet,
+    ProcessorNotFound,
+    )
+from lp.charms.adapters.buildarch import (
+    CharmBase,
+    CharmBaseConfiguration,
+    determine_instances_to_build,
+    DuplicateRunOnError,
+    )
+from lp.testing import (
+    TestCase,
+    TestCaseWithFactory,
+    )
+from lp.testing.layers import LaunchpadZopelessLayer
+
+
+class TestCharmBaseConfiguration(WithScenarios, TestCase):
+
+    scenarios = [
+        ("expanded", {
+            "base": {
+                "build-on": [{
+                    "name": "ubuntu",
+                    "channel": "18.04",
+                    "architectures": ["amd64"],
+                    }],
+                "run-on": [
+                    {
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64", "arm64"],
+                        },
+                    {
+                        "name": "ubuntu",
+                        "channel": "18.04",
+                        "architectures": ["amd64"],
+                        },
+                    ],
+                },
+            "expected_build_on": [
+                CharmBase(
+                    name="ubuntu", channel="18.04", architectures=["amd64"]),
+                ],
+            "expected_run_on": [
+                CharmBase(
+                    name="ubuntu", channel="20.04",
+                    architectures=["amd64", "arm64"]),
+                CharmBase(
+                    name="ubuntu", channel="18.04", architectures=["amd64"]),
+                ],
+            }),
+        ("short form", {
+            "base": {
+                "name": "ubuntu",
+                "channel": "20.04",
+                },
+            "expected_build_on": [CharmBase(name="ubuntu", channel="20.04")],
+            "expected_run_on": [CharmBase(name="ubuntu", channel="20.04")],
+            }),
+        ("no run-on", {
+            "base": {
+                "build-on": [{
+                    "name": "ubuntu",
+                    "channel": "20.04",
+                    "architectures": ["amd64"],
+                    }],
+                },
+            "expected_build_on": [
+                CharmBase(
+                    name="ubuntu", channel="20.04", architectures=["amd64"]),
+                ],
+            "expected_run_on": [
+                CharmBase(
+                    name="ubuntu", channel="20.04", architectures=["amd64"]),
+                ],
+            }),
+        ]
+
+    def test_base(self):
+        config = CharmBaseConfiguration.from_dict(self.base)
+        self.assertEqual(self.expected_build_on, config.build_on)
+        self.assertEqual(self.expected_run_on, config.run_on)
+
+
+class TestDetermineInstancesToBuild(WithScenarios, TestCaseWithFactory):
+
+    layer = LaunchpadZopelessLayer
+
+    # Scenarios taken from the charmcraft build providers specification:
+    # https://docs.google.com/document/d/1Tix0V2J21hfXj-dukgArbTEN8rg31b7za4VM_KylDkc
+    scenarios = [
+        ("single entry, single arch", {
+            "bases": [{
+                "build-on": [{
+                    "name": "ubuntu",
+                    "channel": "18.04",
+                    "architectures": ["amd64"],
+                    }],
+                "run-on": [{
+                    "name": "ubuntu",
+                    "channel": "18.04",
+                    "architectures": ["amd64"],
+                    }],
+                }],
+            "expected": [("18.04", "amd64")],
+            }),
+        ("multiple entries, single arch", {
+            "bases": [
+                {
+                    "build-on": [{
+                        "name": "ubuntu",
+                        "channel": "18.04",
+                        "architectures": ["amd64"],
+                        }],
+                    "run-on": [{
+                        "name": "ubuntu",
+                        "channel": "18.04",
+                        "architectures": ["amd64"],
+                        }],
+                    },
+                {
+                    "build-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    "run-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    },
+                {
+                    "build-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["riscv64"],
+                        }],
+                    "run-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["riscv64"],
+                        }],
+                    },
+                ],
+            "expected": [
+                ("18.04", "amd64"), ("20.04", "amd64"), ("20.04", "riscv64")],
+            }),
+        ("single entry, multiple arches", {
+            "bases": [{
+                "build-on": [{
+                    "name": "ubuntu",
+                    "channel": "20.04",
+                    "architectures": ["amd64"],
+                    }],
+                "run-on": [{
+                    "name": "ubuntu",
+                    "channel": "20.04",
+                    "architectures": ["amd64", "riscv64"],
+                    }],
+                }],
+            "expected": [("20.04", "amd64")],
+            }),
+        ("multiple entries, with cross-arch", {
+            "bases": [
+                {
+                    "build-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    "run-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["riscv64"],
+                        }],
+                    },
+                {
+                    "build-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    "run-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    },
+                ],
+            "expected": [("20.04", "amd64")],
+            }),
+        ("multiple run-on entries", {
+            "bases": [{
+                "build-on": [{
+                    "name": "ubuntu",
+                    "channel": "20.04",
+                    "architectures": ["amd64"],
+                    }],
+                "run-on": [
+                    {
+                        "name": "ubuntu",
+                        "channel": "18.04",
+                        "architectures": ["amd64"],
+                        },
+                    {
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64", "riscv64"],
+                        },
+                    ],
+                }],
+            "expected": [("20.04", "amd64")],
+            }),
+        ("redundant outputs", {
+            "bases": [
+                {
+                    "build-on": [{
+                        "name": "ubuntu",
+                        "channel": "18.04",
+                        "architectures": ["amd64"],
+                        }],
+                    "run-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    },
+                {
+                    "build-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    "run-on": [{
+                        "name": "ubuntu",
+                        "channel": "20.04",
+                        "architectures": ["amd64"],
+                        }],
+                    },
+                ],
+            "expected_exception": MatchesException(
+                DuplicateRunOnError,
+                r"ubuntu 20\.04 \[\"amd64\"\] is present in the 'run-on' of "
+                r"multiple items"),
+            }),
+        ("no bases specified", {
+            "bases": None,
+            "expected": [
+                ("20.04", "amd64"), ("20.04", "arm64"), ("20.04", "riscv64")],
+            }),
+        ]
+
+    def test_parser(self):
+        distro_serieses = [
+            self.factory.makeDistroSeries(
+                distribution=getUtility(ILaunchpadCelebrities).ubuntu,
+                version=version)
+            for version in ("20.04", "18.04")]
+        dases = []
+        for arch_tag in ("amd64", "arm64", "riscv64"):
+            try:
+                processor = getUtility(IProcessorSet).getByName(arch_tag)
+            except ProcessorNotFound:
+                processor = self.factory.makeProcessor(
+                    name=arch_tag, supports_virtualized=True)
+            for distro_series in distro_serieses:
+                dases.append(self.factory.makeDistroArchSeries(
+                    distroseries=distro_series, architecturetag=arch_tag,
+                    processor=processor))
+        charmcraft_data = {}
+        if self.bases is not None:
+            charmcraft_data["bases"] = self.bases
+        build_instances_factory = partial(
+            determine_instances_to_build,
+            charmcraft_data, dases, distro_serieses[0])
+        if hasattr(self, "expected_exception"):
+            self.assertThat(
+                build_instances_factory, Raises(self.expected_exception))
+        else:
+            self.assertThat(build_instances_factory(), MatchesListwise([
+                MatchesStructure(
+                    distroseries=MatchesStructure.byEquality(version=version),
+                    architecturetag=Equals(arch_tag))
+                for version, arch_tag in self.expected]))
+
+
+load_tests = load_tests_apply_scenarios