Merge ~jugmac00/lpci:add-optional-argument-for-add-apt-repository into lpci:main

[Jürgen Gmach]

Add optional argument to add apt repository

[Jürgen Gmach (jugmac00)]

WIP - need to add tests - ran out of time.

[Jürgen Gmach (jugmac00)]

superseded by https://code.launchpad.net/~jugmac00/lpcraft/+git/lpcraft/+merge/420745

Unmerged commits

d270d8e... by Jürgen Gmach on 2022-04-27

Add optional CLI argument to replace `/etc/apt/sources.list`

Also add minimal CLI interface documentation, which at some point should
be autogenerated.

	test:0 (build)
	build:0 (build)

1 → 2 of 2 results

First • Previous • Next • Last

diff --git a/NEWS.rst b/NEWS.rst
index e67e124..1bbb813 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -5,7 +5,10 @@ Version history
 0.0.11 (unreleased)
 ===================
 
-- Nothing yet.
+- Add new optional and stackable argument ``--replace-sources-list`` which
+  overwrites ``/etc/apt/sources.list``.
+
+- Add minimal CLI interface documentation.
 
 0.0.10  (2022-04-27)
 ====================
diff --git a/docs/cli-interface.rst b/docs/cli-interface.rst
new file mode 100644
index 0000000..37f1136
--- /dev/null
+++ b/docs/cli-interface.rst
@@ -0,0 +1,42 @@
+=======================
+lpcraft - CLI interface
+=======================
+
+Please note that this is only a small selection of the available commands and
+options.
+
+Please run ``lpcraft --help`` to see all commands.
+
+lpcraft run
+-----------
+
+This command runs all jobs listed via pipelines from a configuration file.
+
+**Example:**
+
+``lpcraft run``
+
+lpcraft run optional arguments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- ``--replace-sources-list SOURCE_LINE``, e.g.
+  ``lpcraft run --replace-sources-list "deb http://archive.ubuntu.com/ubuntu/ focal main restricted"``.
+  Please note that the option is stackable, so you can repeat it several times.
+
+lpcraft run-one
+---------------
+
+This commands runs one specified job.
+
+**Example:**
+
+``lpcraft run-one test 0``
+
+where ``test`` is the job name and ``0`` is the index of the job/matrix.
+
+lpcraft run-one optional arguments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- ``--replace-sources-list SOURCE_LINE``, e.g.
+  ``lpcraft run-one --replace-sources list "deb http://archive.ubuntu.com/ubuntu/ focal main restricted" test 0``.
+  Please note that the option is stackable, so you can repeat it several times.
diff --git a/docs/index.rst b/docs/index.rst
index b8cfbab..288b9bd 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -46,6 +46,7 @@ Example configuration
 
     self
     configuration
+    cli-interface
     plugins
     CONTRIBUTING
     release-process
diff --git a/lpcraft/commands/run.py b/lpcraft/commands/run.py
index e5bc296..cc37319 100644
--- a/lpcraft/commands/run.py
+++ b/lpcraft/commands/run.py
@@ -194,9 +194,15 @@ def _copy_output_properties(
 
 
 def _run_job(
-    job_name: str, job: Job, provider: Provider, output: Optional[Path]
+    job_name: str,
+    job: Job,
+    provider: Provider,
+    output: Optional[Path],
+    replacement_repositories: Optional[List[str]] = None,
 ) -> None:
     """Run a single job."""
+    # XXX jugmac00 2022-04-27: we should create a configuration object to be
+    # passed in and not so many arguments
     host_architecture = get_host_architecture()
     if host_architecture not in job.architectures:
         return
@@ -253,6 +259,37 @@ def _run_job(
             )
         packages = list(itertools.chain(*pm.hook.lpcraft_install_packages()))
         if packages:
+            if replacement_repositories:
+                # replace sources.list
+                lines = "\n".join(replacement_repositories)
+                with emit.open_stream(
+                    "Replacing /etc/apt/sources.list"
+                ) as stream:
+                    instance.push_file_io(
+                        destination=PurePath("/etc/apt/sources.list"),
+                        content=io.BytesIO(lines.encode()),
+                        file_mode="0644",
+                        group="root",
+                        user="root",
+                    )
+                # update local repository information
+                apt_update = ["apt", "update"]
+                with emit.open_stream(f"Running {apt_update}") as stream:
+                    proc = instance.execute_run(
+                        apt_update,
+                        cwd=remote_cwd,
+                        env=environment,
+                        stdout=stream,
+                        stderr=stream,
+                    )
+                if proc.returncode != 0:
+                    raise CommandError(
+                        f"Job {job_name!r} for "
+                        f"{job.series}/{host_architecture} failed with "
+                        f"exit status {proc.returncode} "
+                        f"while running `{shlex.join(apt_update)}`.",
+                        retcode=proc.returncode,
+                    )
             packages_cmd = ["apt", "install", "-y"] + packages
             emit.progress("Installing system packages")
             with emit.open_stream(f"Running {packages_cmd}") as stream:
@@ -344,6 +381,9 @@ def run(args: Namespace) -> int:
                             job,
                             provider,
                             getattr(args, "output_directory", None),
+                            replacement_repositories=getattr(
+                                args, "replace_sources_list", None
+                            ),
                         )
                 except CommandError as e:
                     if len(stage) == 1:
@@ -391,7 +431,13 @@ def run_one(args: Namespace) -> int:
 
     try:
         _run_job(
-            args.job, job, provider, getattr(args, "output_directory", None)
+            args.job,
+            job,
+            provider,
+            getattr(args, "output_directory", None),
+            replacement_repositories=getattr(
+                args, "replace_sources_list", None
+            ),
         )
     finally:
         should_clean_environment = getattr(args, "clean", False)
diff --git a/lpcraft/commands/tests/test_run.py b/lpcraft/commands/tests/test_run.py
index b5ae1db..fca99f4 100644
--- a/lpcraft/commands/tests/test_run.py
+++ b/lpcraft/commands/tests/test_run.py
@@ -419,6 +419,110 @@ class TestRun(RunBaseTestCase):
 
     @patch("lpcraft.commands.run.get_provider")
     @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+    def test_replace_sources_list(
+        self, mock_get_host_architecture, mock_get_provider
+    ):
+        launcher = Mock(spec=launch)
+        provider = makeLXDProvider(lxd_launcher=launcher)
+        mock_get_provider.return_value = provider
+        execute_run = launcher.return_value.execute_run
+        execute_run.return_value = subprocess.CompletedProcess([], 0)
+        launcher.return_value.pull_file
+        config = dedent(
+            """
+            pipeline:
+                - test
+            jobs:
+                test:
+                    series: focal
+                    architectures: amd64
+                    run: ls -la
+                    packages: [git]
+            """
+        )
+        Path(".launchpad.yaml").write_text(config)
+
+        result = self.run_command("run", "--replace-sources-list", "repo info")
+
+        self.assertEqual(0, result.exit_code)
+        self.assertEqual(
+            [
+                call(
+                    ["apt", "update"],
+                    cwd=Path("/root/lpcraft/project"),
+                    env={},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    ["apt", "install", "-y", "git"],
+                    cwd=Path("/root/lpcraft/project"),
+                    env={},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    ["bash", "--noprofile", "--norc", "-ec", "ls -la"],
+                    cwd=Path("/root/lpcraft/project"),
+                    env={},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+            ],
+            execute_run.call_args_list,
+        )
+
+        mock_info = launcher.return_value.push_file_io.call_args_list[0][1]
+        self.assertEqual(
+            Path("/etc/apt/sources.list"), mock_info["destination"]
+        )
+        self.assertEqual("repo info", mock_info["content"].read().decode())
+        self.assertEqual("0644", mock_info["file_mode"])
+        self.assertEqual("root", mock_info["group"])
+        self.assertEqual("root", mock_info["user"])
+
+    @patch("lpcraft.commands.run.get_provider")
+    @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+    def test_updating_package_info_fails(
+        self, mock_get_host_architecture, mock_get_provider
+    ):
+        launcher = Mock(spec=launch)
+        provider = makeLXDProvider(lxd_launcher=launcher)
+        mock_get_provider.return_value = provider
+        execute_run = launcher.return_value.execute_run
+        execute_run.return_value = subprocess.CompletedProcess([], 100)
+        config = dedent(
+            """
+            pipeline:
+                - test
+            jobs:
+                test:
+                    series: focal
+                    architectures: amd64
+                    run: ls -la
+                    packages: [git]
+            """
+        )
+        Path(".launchpad.yaml").write_text(config)
+
+        result = self.run_command("run", "--replace-sources-list", "repo info")
+
+        self.assertEqual(100, result.exit_code)
+        self.assertEqual(
+            [
+                call(
+                    ["apt", "update"],
+                    cwd=Path("/root/lpcraft/project"),
+                    env={},
+                    stdout=ANY,
+                    stderr=ANY,
+                )
+            ],
+            execute_run.call_args_list,
+        )
+
+    @patch("lpcraft.commands.run.get_provider")
+    @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
     def test_default_to_run_command(
         self, mock_get_host_architecture, mock_get_provider
     ):
@@ -2149,3 +2253,69 @@ class TestRunOne(RunBaseTestCase):
             project_path=self.tmp_project_path,
             instances=instance_names,
         )
+
+    @patch("lpcraft.commands.run.get_provider")
+    @patch("lpcraft.commands.run.get_host_architecture", return_value="amd64")
+    def test_replace_sources_list(
+        self, mock_get_host_architecture, mock_get_provider
+    ):
+        launcher = Mock(spec=launch)
+        provider = makeLXDProvider(lxd_launcher=launcher)
+        mock_get_provider.return_value = provider
+        execute_run = launcher.return_value.execute_run
+        execute_run.return_value = subprocess.CompletedProcess([], 0)
+        launcher.return_value.pull_file
+        config = dedent(
+            """
+            pipeline:
+                - test
+            jobs:
+                test:
+                    series: focal
+                    architectures: amd64
+                    run: ls -la
+                    packages: [git]
+            """
+        )
+        Path(".launchpad.yaml").write_text(config)
+
+        result = self.run_command(
+            "run-one", "--replace-sources-list", "repo info", "test", "0"
+        )
+
+        self.assertEqual(0, result.exit_code)
+        self.assertEqual(
+            [
+                call(
+                    ["apt", "update"],
+                    cwd=Path("/root/lpcraft/project"),
+                    env={},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    ["apt", "install", "-y", "git"],
+                    cwd=Path("/root/lpcraft/project"),
+                    env={},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+                call(
+                    ["bash", "--noprofile", "--norc", "-ec", "ls -la"],
+                    cwd=Path("/root/lpcraft/project"),
+                    env={},
+                    stdout=ANY,
+                    stderr=ANY,
+                ),
+            ],
+            execute_run.call_args_list,
+        )
+
+        mock_info = launcher.return_value.push_file_io.call_args_list[0][1]
+        self.assertEqual(
+            Path("/etc/apt/sources.list"), mock_info["destination"]
+        )
+        self.assertEqual("repo info", mock_info["content"].read().decode())
+        self.assertEqual("0644", mock_info["file_mode"])
+        self.assertEqual("root", mock_info["group"])
+        self.assertEqual("root", mock_info["user"])
diff --git a/lpcraft/main.py b/lpcraft/main.py
index 42a68e5..4a90d01 100644
--- a/lpcraft/main.py
+++ b/lpcraft/main.py
@@ -97,6 +97,11 @@ def main(argv: Optional[List[str]] = None) -> int:
             "for the pipeline after the running it."
         ),
     )
+    parser_run.add_argument(
+        "--replace-sources-list",
+        action="append",
+        help="Overwrite /etc/apt/sources.list.",
+    )
     parser_run.set_defaults(func=run)
 
     parser_run_one = subparsers.add_parser(
@@ -129,6 +134,11 @@ def main(argv: Optional[List[str]] = None) -> int:
         metavar="N",
         help="Run only the Nth job with the given name (indexing from 0).",
     )
+    parser_run_one.add_argument(
+        "--replace-sources-list",
+        action="append",
+        help="Overwrite /etc/apt/sources.list.",
+    )
     parser_run_one.set_defaults(func=run_one)
 
     parser_version = subparsers.add_parser(