Merge ~artivis/lpci:feature/yaml-include into lpci:main

[jeremie]

[Do Not Merge Yet]

Note: This is based on top of another branch (https://git.launchpad.net/~artivis/lpcraft/tree/?h=feature/yaml-alias-filter) which may replace MP https://code.launchpad.net/~artivis/lpcraft/+git/lpcraft/+merge/435370

Add support for yaml inclusion.

This allows for including other yaml files to a configuration file using the 'include' tag located at the root. Included file paths can be relative or absolute. Since yaml anchors are lost during the inclusion, this MP also introduce the 'extends' tag that essentially does the work of mapping.

Hereafter is an example of what that looks like,

# .included.launchpad.yaml
.test:
  series: bionic
  architectures: [amd64]

# .launchpad.yaml
pipeline:
  - test

include:
  - .included.launchpad.yaml

jobs:
  test-a:
      series: focal
      architectures: [amd64]
  test-b:
      extends: .test # maps '.test' nested entries
      packages: [file] # can be further extended with new entries

[Do Not Merge Yet]

[Jürgen Gmach (jugmac00)]

Similar to the notes on YAML aliases, let's discuss whether an external tool / script could be a viable solution.
https://chat.canonical.com/canonical/pl/r71cgk3ejtr1zmbfnfitbi187c

Unmerged commits

1bee04c... by jeremie on 2023-01-10

test yaml inclusion

Signed-off-by: artivis <email address hidden>

	test:0 (build)
	build:0 (build)

1 → 2 of 2 results

First • Previous • Next • Last

e1c0554... by jeremie on 2023-01-10

add support for yaml inclusion

Signed-off-by: artivis <email address hidden>

5e3aa4f... by jeremie on 2023-01-09

couple tests yaml anchor/alias

Signed-off-by: artivis <email address hidden>

	test:0 (build)
	build:0 (build)

1 → 2 of 2 results

First • Previous • Next • Last

31e792a... by jeremie on 2023-01-09

pydantic filter out extra fields with prefix

Signed-off-by: artivis <email address hidden>

diff --git a/lpcraft/config.py b/lpcraft/config.py
index 3564d9c..1409cde 100644
--- a/lpcraft/config.py
+++ b/lpcraft/config.py
@@ -373,7 +373,7 @@ class Config(ModelConfigDefaults):
         """Filter out extension field."""
         expanded_values = values.copy()
         for k, v in values.items():
-            if k.startswith("."):
+            if k.startswith(".") or k == "include":
                 expanded_values.pop(k)
         return expanded_values
 
diff --git a/lpcraft/tests/test_config.py b/lpcraft/tests/test_config.py
index 8695520..6ca9de8 100644
--- a/lpcraft/tests/test_config.py
+++ b/lpcraft/tests/test_config.py
@@ -16,6 +16,7 @@ from testtools.matchers import (
     MatchesStructure,
 )
 
+<<<<<<< lpcraft/tests/test_config.py
 from lpcraft.config import (
     LAUNCHPAD_PPA_BASE_URL,
     Config,
@@ -25,6 +26,10 @@ from lpcraft.config import (
     get_ppa_url_parts,
 )
 from lpcraft.errors import CommandError
+=======
+from lpcraft.config import Config, OutputDistributeEnum, PackageRepository
+from lpcraft.errors import CommandError, ConfigurationError
+>>>>>>> lpcraft/tests/test_config.py
 
 
 class TestConfig(TestCase):
@@ -35,8 +40,8 @@ class TestConfig(TestCase):
         # So switch to the created project directory.
         os.chdir(self.tempdir)
 
-    def create_config(self, text):
-        path = self.tempdir / ".launchpad.yaml"
+    def create_config(self, text, filename=".launchpad.yaml"):
+        path = self.tempdir / filename
         path.write_text(text)
         return path
 
@@ -909,3 +914,207 @@ class TestConfig(TestCase):
                 ),
             ),
         )
+
+    def test_yaml_include(self):
+        # Include a config file given a relative path
+        self.create_config(
+            dedent(
+                """
+                .test:
+                    series: bionic
+                    architectures: [amd64]
+                """
+            ),
+            ".included.launchpad.yaml",
+        )
+
+        path = self.create_config(
+            dedent(
+                """
+                pipeline:
+                    - test
+
+                include:
+                  - .included.launchpad.yaml
+
+                jobs:
+                    test-a:
+                        series: focal
+                        architectures: [amd64]
+
+                    test-b:
+                        extends: .test
+                        packages: [file]
+                """
+            )
+        )
+
+        config = Config.load(path)
+
+        self.assertThat(
+            config,
+            MatchesStructure(
+                pipeline=Equals([["test"]]),
+                jobs=MatchesDict(
+                    {
+                        "test-a": MatchesListwise(
+                            [
+                                MatchesStructure.byEquality(
+                                    series="focal",
+                                    architectures=["amd64"],
+                                )
+                            ]
+                        ),
+                        "test-b": MatchesListwise(
+                            [
+                                MatchesStructure.byEquality(
+                                    series="bionic",
+                                    architectures=["amd64"],
+                                    packages=["file"],
+                                )
+                            ]
+                        ),
+                    }
+                ),
+            ),
+        )
+
+    def test_yaml_include_abs_path(self):
+        # Include a config file given an absolute path
+        tempdir = Path(self.useFixture(TempDir()).path)
+        included_config_path = tempdir / ".included.launchpad.yaml"
+        included_config_path.write_text(
+            dedent(
+                """
+                .test:
+                    series: bionic
+                    architectures: [amd64]
+                """
+            )
+        )
+
+        base_config_path = self.create_config(
+            dedent(
+                f"""
+                pipeline:
+                    - test
+
+                include:
+                  - '{included_config_path}'
+
+                jobs:
+                    test-a:
+                        series: focal
+                        architectures: [amd64]
+
+                    test-b:
+                        extends: .test
+                        packages: [file]
+                """
+            )
+        )
+
+        config = Config.load(base_config_path)
+
+        self.assertNotEqual(
+            base_config_path.parent, included_config_path.parent
+        )
+
+        self.assertThat(
+            config,
+            MatchesStructure(
+                pipeline=Equals([["test"]]),
+                jobs=MatchesDict(
+                    {
+                        "test-a": MatchesListwise(
+                            [
+                                MatchesStructure.byEquality(
+                                    series="focal",
+                                    architectures=["amd64"],
+                                )
+                            ]
+                        ),
+                        "test-b": MatchesListwise(
+                            [
+                                MatchesStructure.byEquality(
+                                    series="bionic",
+                                    architectures=["amd64"],
+                                    packages=["file"],
+                                )
+                            ]
+                        ),
+                    }
+                ),
+            ),
+        )
+
+    def test_yaml_bad_include(self):
+        # Include a config file that doesn't exists.
+        path = self.create_config(
+            dedent(
+                """
+                pipeline:
+                    - test
+
+                include:
+                  - bad_include.launchpad.yaml
+
+                jobs:
+                    test:
+                        series: focal
+                        architectures: [amd64]
+                """
+            )
+        )
+
+        self.assertRaisesRegex(
+            ConfigurationError,
+            "Couldn't find config file 'bad_include.launchpad.yaml'.",
+            Config.load,
+            path,
+        )
+
+    def test_yaml_include_not_mapping(self):
+        # Include a config file that is not defining a mapping
+        self.create_config(
+            dedent(
+                """
+                - foo
+                """
+            ),
+            ".launchpad.included.yaml",
+        )
+
+        path = self.create_config(
+            dedent(
+                """
+                include:
+                  - .launchpad.included.yaml
+                """
+            )
+        )
+
+        self.assertRaisesRegex(
+            ConfigurationError,
+            "Config file '.launchpad.included.yaml' does not define a mapping",
+            Config.load,
+            path,
+        )
+
+    def test_yaml_bad_extends(self):
+        # Extends a node that doesn't exists.
+        path = self.create_config(
+            dedent(
+                """
+                test:
+                    extends: .test
+                """
+            )
+        )
+
+        self.assertRaisesRegex(
+            ConfigurationError,
+            "Couldn't find extension '.test'.",
+            Config.load,
+            path,
+        )
diff --git a/lpcraft/tests/test_utils.py b/lpcraft/tests/test_utils.py
index 52b42e3..86cbda1 100644
--- a/lpcraft/tests/test_utils.py
+++ b/lpcraft/tests/test_utils.py
@@ -11,7 +11,7 @@ from systemfixtures import FakeProcesses
 from testtools import TestCase
 
 from lpcraft.errors import ConfigurationError
-from lpcraft.utils import ask_user, get_host_architecture, load_yaml
+from lpcraft.utils import ask_user, find, get_host_architecture, load_yaml
 
 
 class TestLoadYAML(TestCase):
@@ -66,6 +66,26 @@ class TestLoadYAML(TestCase):
             path,
         )
 
+    def test_find(self):
+        self.assertEqual(42, next(find({"a": 42}, "a")))
+        self.assertEqual(42, next(find({"b": 41, "a": 42}, "a")))
+
+        self.assertEqual(42, next(find({"b": {"a": 42}}, "a")))
+
+        self.assertEqual(42, next(find([{"b": 41, "a": 42}], "a")))
+        self.assertEqual(42, next(find([{"b": 41}, {"a": 42}], "a")))
+        self.assertEqual(42, next(find([("b", {"a": 42})], "a")))
+
+        def find_raises_map():
+            next(find({"a": 42}, "b"))
+
+        self.assertRaises(StopIteration, find_raises_map)
+
+        def find_raises_list():
+            next(find([1, 2, 3], "b"))
+
+        self.assertRaises(StopIteration, find_raises_list)
+
 
 class TestGetHostArchitecture(TestCase):
     def setUp(self):
diff --git a/lpcraft/utils.py b/lpcraft/utils.py
index a369cbd..9b5ef19 100644
--- a/lpcraft/utils.py
+++ b/lpcraft/utils.py
@@ -18,6 +18,71 @@ import yaml
 from lpcraft.errors import ConfigurationError
 
 
+def find(d: Any, key: Any) -> Any:
+    """Recursively search in dict/list/tuple."""
+    if isinstance(d, dict):
+        if key in d:
+            yield d[key]
+        for v in d.values():
+            for i in find(v, key):
+                yield i
+    if isinstance(d, (list, tuple)):
+        for v in d:
+            for i in find(v, key):
+                yield i
+
+
+def expand_yaml_extends(loaded: Any, root: Dict[Any, Any]) -> None:
+    """Expand 'extends' entries.
+
+    extends entries are searched for throughout the entire configuration.
+    When found, their associated value (pseudo-alias) is then
+    searched for throughout the entire configuration.
+    The value associated to the pseudo-alias is copied
+    at the level the initial 'extends' key was found at.
+    Finally the 'extends' entry is entirely removed.
+    """
+    tag = "extends"
+    if isinstance(loaded, dict):
+        for _, v in loaded.items():
+            if isinstance(v, (dict, list, tuple)):
+                expand_yaml_extends(v, root)
+        if tag in loaded.keys():
+            v = loaded.pop(tag)
+
+            try:
+                val = next(find(root, v))
+            except StopIteration:
+                raise ConfigurationError(
+                    f"Couldn't find extension {str(v)!r}."
+                )
+
+            loaded.update(val)
+    elif isinstance(loaded, (list, tuple)):
+        for v in loaded:
+            expand_yaml_extends(v, root)
+
+
+def load_yaml_includes(loaded: Dict[str, Any]) -> None:
+    """Load all configuration files listed as include."""
+    for include in loaded.get("include", []):
+        include_path = Path(include)
+        if not include_path.is_file():
+            raise ConfigurationError(
+                f"Couldn't find config file {str(include_path)!r}."
+            )
+        with include_path.open("rb") as f:
+
+            included = yaml.safe_load(f)
+
+            if not isinstance(included, dict):
+                raise ConfigurationError(
+                    f"Config file {str(include_path)!r} does not define a mapping"  # noqa: E501
+                )
+
+            loaded.update(included)
+
+
 def load_yaml(path: Path) -> Dict[Any, Any]:
     """Return the content of a YAML file."""
     if not path.is_file():
@@ -25,10 +90,16 @@ def load_yaml(path: Path) -> Dict[Any, Any]:
     try:
         with path.open("rb") as f:
             loaded = yaml.safe_load(f)
+
         if not isinstance(loaded, dict):
             raise ConfigurationError(
                 f"Config file {str(path)!r} does not define a mapping"
             )
+
+        load_yaml_includes(loaded)
+
+        expand_yaml_extends(loaded, loaded)
+
         return loaded
     except (yaml.error.YAMLError, OSError) as e:
         raise ConfigurationError(