Merge ~jugmac00/lpci:add-conda-build-plugin into lpci:main

[Jürgen Gmach]

- Add conda build plugin
- Test existing recipe folder with missing meta.yaml

[Jürgen Gmach (jugmac00)]

self approve - implementation already approved via https://code.launchpad.net/~techalchemy/lpcraft/+git/lpcraft/+merge/423301

I only added one test to cover one previously uncovered line.

review: Approve

diff --git a/lpcraft/plugin/tests/test_plugins.py b/lpcraft/plugin/tests/test_plugins.py
index fe7d930..e0ec098 100644
--- a/lpcraft/plugin/tests/test_plugins.py
+++ b/lpcraft/plugin/tests/test_plugins.py
@@ -483,3 +483,422 @@ class TestPlugins(CommandBaseTestCase):
             plugin_match[0].conda_channels,
         )
         self.assertEqual(["PYTHON=3.8", "pip"], plugin_match[0].conda_packages)
+
+    @patch("lpcraft.commands.run.get_provider")
+    @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+    def test_conda_build_plugin(
+        self, mock_get_host_architecture, mock_get_provider
+    ):
+        launcher = Mock(spec=launch)
+        provider = makeLXDProvider(lxd_launcher=launcher)
+        mock_get_provider.return_value = provider
+        execute_run = launcher.return_value.execute_run
+        execute_run.return_value = subprocess.CompletedProcess([], 0)
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    build-target: info/recipe/parent
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+            """
+        )
+        Path(".launchpad.yaml").write_text(config)
+        Path("info/recipe/parent").mkdir(parents=True)
+        Path("info/recipe/meta.yaml").touch()
+        Path("info/recipe/parent/meta.yaml").touch()
+        pre_run_command = dedent(
+            """
+        if [ ! -d "$HOME/miniconda3" ]; then
+            wget -O /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+            chmod +x /tmp/miniconda.sh
+            /tmp/miniconda.sh -b
+        fi
+        export PATH=$HOME/miniconda3/bin:$PATH
+        conda remove --all -q -y -n $CONDA_ENV
+        conda create -n $CONDA_ENV -q -y -c conda-forge -c defaults PYTHON=3.8 conda-build mamba pip
+        source activate $CONDA_ENV
+        """  # noqa:E501
+        )
+        run_command = dedent(
+            """
+            export PATH=$HOME/miniconda3/bin:$PATH
+            source activate $CONDA_ENV
+            conda-build --no-anaconda-upload --output-folder dist -c conda-forge -c defaults info/recipe/parent
+            pip install --upgrade pytest
+        """  # noqa: E501
+        )
+        post_run_command = (
+            "export PATH=$HOME/miniconda3/bin:$PATH; "
+            "source activate $CONDA_ENV; conda env export"
+        )
+
+        self.run_command("run")
+
+        self.assertEqual(
+            [
+                call(
+                    ["apt", "update"],
+                    cwd=PosixPath("/root/lpcraft/project"),
+                    env={"CONDA_ENV": "lpci"},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    [
+                        "apt",
+                        "install",
+                        "-y",
+                        "git",
+                        "python3-dev",
+                        "python3-pip",
+                        "python3-venv",
+                        "wget",
+                        "automake",
+                        "build-essential",
+                        "cmake",
+                        "gcc",
+                        "g++",
+                        "libc++-dev",
+                        "libc6-dev",
+                        "libffi-dev",
+                        "libjpeg-dev",
+                        "libpng-dev",
+                        "libreadline-dev",
+                        "libsqlite3-dev",
+                        "libtool",
+                        "zlib1g-dev",
+                    ],
+                    cwd=PosixPath("/root/lpcraft/project"),
+                    env={"CONDA_ENV": "lpci"},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    [
+                        "bash",
+                        "--noprofile",
+                        "--norc",
+                        "-ec",
+                        pre_run_command,
+                    ],
+                    cwd=PosixPath("/root/lpcraft/project"),
+                    env={"CONDA_ENV": "lpci"},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    [
+                        "bash",
+                        "--noprofile",
+                        "--norc",
+                        "-ec",
+                        run_command,
+                    ],
+                    cwd=PosixPath("/root/lpcraft/project"),
+                    env={"CONDA_ENV": "lpci"},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    [
+                        "bash",
+                        "--noprofile",
+                        "--norc",
+                        "-ec",
+                        post_run_command,
+                    ],
+                    cwd=PosixPath("/root/lpcraft/project"),
+                    env={"CONDA_ENV": "lpci"},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+            ],
+            execute_run.call_args_list,
+        )
+
+    def test_conda_build_plugin_finds_recipe(self):
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+        """
+        )
+        config_path = Path(".launchpad.yaml")
+        config_path.write_text(config)
+        Path("include/fake_subdir").mkdir(parents=True)
+        meta_yaml = Path("info/recipe/meta.yaml")
+        meta_yaml.parent.mkdir(parents=True)
+        meta_yaml.touch()
+        config_obj = lpcraft.config.Config.load(config_path)
+        self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+        pm = get_plugin_manager(config_obj.jobs["build"][0])
+        plugins = pm.get_plugins()
+        plugin_match = [
+            _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+        ]
+        self.assertEqual("info/recipe", plugin_match[0].build_target)
+
+    def test_conda_build_plugin_finds_recipe_with_fake_parent(self):
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+        """
+        )
+        config_path = Path(".launchpad.yaml")
+        config_path.write_text(config)
+        meta_yaml = Path("info/recipe/meta.yaml")
+        meta_yaml.parent.mkdir(parents=True)
+        parent_path = meta_yaml.parent.joinpath("parent")
+        parent_path.mkdir()
+        parent_path.joinpath("some_file.yaml").touch()
+        meta_yaml.touch()
+        config_obj = lpcraft.config.Config.load(config_path)
+        self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+        pm = get_plugin_manager(config_obj.jobs["build"][0])
+        plugins = pm.get_plugins()
+        plugin_match = [
+            _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+        ]
+        self.assertEqual("info/recipe", plugin_match[0].build_target)
+
+    def test_conda_build_plugin_finds_parent_recipe(self):
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+        """
+        )
+        config_path = Path(".launchpad.yaml")
+        config_path.write_text(config)
+        Path("include/fake_subdir").mkdir(parents=True)
+        meta_yaml = Path("info/recipe/meta.yaml")
+        parent_meta_yaml = meta_yaml.parent.joinpath("parent/meta.yaml")
+        parent_meta_yaml.parent.mkdir(parents=True)
+        meta_yaml.touch()
+        parent_meta_yaml.touch()
+        config_obj = lpcraft.config.Config.load(config_path)
+        self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+        pm = get_plugin_manager(config_obj.jobs["build"][0])
+        plugins = pm.get_plugins()
+        plugin_match = [
+            _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+        ]
+        self.assertEqual("info/recipe/parent", plugin_match[0].build_target)
+
+    def test_conda_build_plugin_uses_child_vars_with_parent_recipe(self):
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+        """
+        )
+        run_command = dedent(
+            """
+            export PATH=$HOME/miniconda3/bin:$PATH
+            source activate $CONDA_ENV
+            conda-build --no-anaconda-upload --output-folder dist -c conda-forge -c defaults -m info/recipe/parent/conda_build_config.yaml -m info/recipe/conda_build_config.yaml info/recipe/parent
+            pip install --upgrade pytest
+        """  # noqa: E501
+        )
+        config_path = Path(".launchpad.yaml")
+        config_path.write_text(config)
+        Path("include/fake_subdir").mkdir(parents=True)
+        meta_yaml = Path("info/recipe/meta.yaml")
+        variant_config = meta_yaml.with_name("conda_build_config.yaml")
+        parent_meta_yaml = meta_yaml.parent.joinpath("parent/meta.yaml")
+        parent_variant_config = parent_meta_yaml.with_name(
+            "conda_build_config.yaml"
+        )
+        parent_meta_yaml.parent.mkdir(parents=True)
+        meta_yaml.touch()
+        variant_config.touch()
+        parent_variant_config.touch()
+        parent_meta_yaml.touch()
+        config_obj = lpcraft.config.Config.load(config_path)
+        self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+        pm = get_plugin_manager(config_obj.jobs["build"][0])
+        plugins = pm.get_plugins()
+        plugin_match = [
+            _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+        ]
+        self.assertEqual(
+            [parent_variant_config.as_posix(), variant_config.as_posix()],
+            plugin_match[0].build_configs,
+        )
+        self.assertEqual(run_command, plugin_match[0].lpcraft_execute_run())
+
+    def test_conda_build_plugin_renames_recipe_templates(self):
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+        """
+        )
+        config_path = Path(".launchpad.yaml")
+        config_path.write_text(config)
+        meta_yaml = Path("info/recipe/meta.yaml")
+        template_meta_yaml = meta_yaml.with_name("meta.yaml.template")
+        meta_yaml.parent.mkdir(parents=True)
+        meta_yaml.touch()
+        template_meta_yaml.touch()
+        config_obj = lpcraft.config.Config.load(config_path)
+        self.assertEqual(config_obj.jobs["build"][0].plugin, "conda-build")
+        pm = get_plugin_manager(config_obj.jobs["build"][0])
+        plugins = pm.get_plugins()
+        plugin_match = [
+            _ for _ in plugins if _.__class__.__name__ == "CondaBuildPlugin"
+        ]
+        self.assertEqual("info/recipe", plugin_match[0].build_target)
+        self.assertFalse(template_meta_yaml.is_file())
+
+    def test_conda_build_plugin_raises_error_if_no_recipe(self):
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+        """
+        )
+        config_path = Path(".launchpad.yaml")
+        config_path.write_text(config)
+        config_obj = lpcraft.config.Config.load(config_path)
+        self.assertRaisesRegex(
+            RuntimeError,
+            "No build target found",
+            get_plugin_manager,
+            config_obj.jobs["build"][0],
+        )
+
+    def test_conda_build_plugin_raises_error_if_no_recipe_in_recipe_folder(
+        self,
+    ):
+        config = dedent(
+            """
+            pipeline:
+                - build
+
+            jobs:
+                build:
+                    series: focal
+                    architectures: amd64
+                    plugin: conda-build
+                    conda-channels:
+                        - conda-forge
+                    conda-packages:
+                        - mamba
+                        - pip
+                    conda-python: 3.8
+                    run: |
+                        pip install --upgrade pytest
+        """
+        )
+        config_path = Path(".launchpad.yaml")
+        config_path.write_text(config)
+        Path("include/fake_subdir").mkdir(parents=True)
+        # there is a recipe folder, but no meta.yaml file
+        meta_yaml = Path("info/recipe/")
+        meta_yaml.mkdir(parents=True)
+        config_obj = lpcraft.config.Config.load(config_path)
+        self.assertRaisesRegex(
+            RuntimeError,
+            "No build target found",
+            get_plugin_manager,
+            config_obj.jobs["build"][0],
+        )
diff --git a/lpcraft/plugins/plugins.py b/lpcraft/plugins/plugins.py
index c114030..0797fba 100644
--- a/lpcraft/plugins/plugins.py
+++ b/lpcraft/plugins/plugins.py
@@ -3,9 +3,15 @@
 
 from __future__ import annotations  # isort:skip
 
-__all__ = ["ToxPlugin", "PyProjectBuildPlugin", "MiniCondaPlugin"]
+__all__ = [
+    "ToxPlugin",
+    "PyProjectBuildPlugin",
+    "MiniCondaPlugin",
+    "CondaBuildPlugin",
+]
 
 import textwrap
+from pathlib import Path
 from typing import TYPE_CHECKING, ClassVar, List, Optional, cast
 
 import pydantic
@@ -213,3 +219,174 @@ class MiniCondaPlugin(BasePlugin):
             "export PATH=$HOME/miniconda3/bin:$PATH; "
             f"source activate $CONDA_ENV; conda env export{run}"
         )
+
+
+@register(name="conda-build")
+class CondaBuildPlugin(MiniCondaPlugin):
+    """Sets up `miniconda3` and performs a `conda-build` on a package.
+
+    Usage:
+        In `.launchpad.yaml`, create the following structure:
+
+        .. code-block:: yaml
+
+           jobs:
+               myjob:
+                  plugin: conda-build
+                  build-target: info/recipe/parent
+                  conda-channels:
+                    - conda-forge
+                    - defaults
+                  conda-packages:
+                    - mamba
+                    - numpy=1.17
+                    - scipy
+                    - pip
+                  conda-python: 3.8
+                  run: |
+                    conda install ....
+                    pip install --upgrade pytest
+                    python -m build .
+    """
+
+    class Config(MiniCondaPlugin.Config):
+        build_target: Optional[StrictStr]
+        conda_channels: Optional[List[StrictStr]]
+        conda_packages: Optional[List[StrictStr]]
+        conda_python: Optional[StrictStr]
+
+    DEFAULT_CONDA_PACKAGES = ("conda-build",)
+
+    def get_plugin_config(self) -> "CondaBuildPlugin.Config":
+        return cast(CondaBuildPlugin.Config, self.config.plugin_config)
+
+    @staticmethod
+    def _has_recipe(dir_: Path) -> bool:
+        return dir_.joinpath("meta.yaml").is_file()
+
+    @staticmethod
+    def _rename_recipe_template(dir_: Path) -> None:
+        # XXX techalchemy 2022-04-01: conda packages which are already built
+        # and subsequently downloaded from the anaconda repositories retain
+        # the templated recipe, at `meta.yaml.template`, but place the
+        # rendered template at `meta.yaml`. The rendered recipes contain
+        # hardcoded paths for a specific build environment and, for our
+        # purposes, are not reusable. We need to render new ones from the
+        # original templates.
+        template_path = dir_.joinpath("meta.yaml.template")
+        if template_path.is_file():
+            template_path.replace(dir_ / "meta.yaml")
+
+    def find_recipe(self) -> Path:
+        def _find_recipe_dir(path: Path) -> Path:
+            for subpath in path.iterdir():
+                if subpath.is_dir():
+                    self._rename_recipe_template(subpath)
+                    if subpath.name == "recipe" and self._has_recipe(subpath):
+                        return subpath
+                    try:
+                        return _find_recipe_dir(subpath)
+                    except FileNotFoundError:
+                        continue
+            raise FileNotFoundError
+
+        return _find_recipe_dir(Path("."))
+
+    def find_build_target(self) -> str:
+        def find_parents(pth: Path) -> Path:
+            for parent in pth.iterdir():
+                if parent.is_dir():
+                    self._rename_recipe_template(parent)
+                    if parent.name == "parent" and self._has_recipe(parent):
+                        return parent
+            raise FileNotFoundError(pth.joinpath("meta.yaml"))
+
+        try:
+            recipe = self.find_recipe()
+        except FileNotFoundError:
+            raise RuntimeError("No build target found")
+        try:
+            # XXX techalchemy 2022-04-01: Some conda packages are built as
+            # part of a parent package build process (e.g. `mkl-include` which
+            # is built by `intel_repack`). If you acquire the child package
+            # and attempt to build it (`mkl-include` in this case) it will
+            # fail; you must build the parent instead if it exists
+            return find_parents(recipe).as_posix()
+        except FileNotFoundError:
+            return recipe.as_posix()
+
+    @property
+    def build_configs(self) -> list[str]:
+        try:
+            recipe = self.find_recipe()
+        except FileNotFoundError:
+            return []
+        configs = sorted(
+            recipe.glob("**/conda_build_config.yaml"), reverse=True
+        )
+        return [_.as_posix() for _ in configs]
+
+    @property
+    def build_target(self) -> str:
+        build_target = self.get_plugin_config().build_target
+        if not build_target:
+            return self.find_build_target()
+        return build_target
+
+    @hookimpl  # type: ignore
+    def lpcraft_set_environment(self) -> dict[str, str]:
+        # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+        rv: dict[str, str] = super().lpcraft_set_environment()
+        return rv
+
+    @hookimpl  # type: ignore
+    def lpcraft_execute_before_run(self) -> str:
+        # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+        rv: str = super().lpcraft_execute_before_run()
+        return rv
+
+    @hookimpl  # type: ignore
+    def lpcraft_execute_after_run(self) -> str:
+        # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+        rv: str = super().lpcraft_execute_after_run()
+        return rv
+
+    @hookimpl  # type: ignore
+    def lpcraft_install_packages(self) -> list[str]:
+        # XXX techalchemy 2022-04-01: mypy is struggling with the super() call
+        base_packages: list[str] = super().lpcraft_install_packages()
+        base_packages.extend(
+            [
+                "automake",
+                "build-essential",
+                "cmake",
+                "gcc",
+                "g++",
+                "libc++-dev",
+                "libc6-dev",
+                "libffi-dev",
+                "libjpeg-dev",
+                "libpng-dev",
+                "libreadline-dev",
+                "libsqlite3-dev",
+                "libtool",
+                "zlib1g-dev",
+            ]
+        )
+        return base_packages
+
+    @hookimpl  # type: ignore
+    def lpcraft_execute_run(self) -> str:
+        conda_channels = " ".join(f"-c {_}" for _ in self.conda_channels)
+        conda_channels = f" {conda_channels}" if conda_channels else ""
+        configs = " ".join(f"-m {_}" for _ in self.build_configs)
+        configs = f" {configs}" if configs else ""
+        build_command = "conda-build --no-anaconda-upload --output-folder dist"
+        run_command = self.config.run or ""
+        return textwrap.dedent(
+            f"""
+            export PATH=$HOME/miniconda3/bin:$PATH
+            source activate $CONDA_ENV
+            {build_command}{conda_channels}{configs} {self.build_target}
+            {run_command}"""
+        )