Merge ~lloydwaltersj/maas-ci/+git/system-tests:refactor-image-promotion-workflows into ~lloydwaltersj/maas-ci/+git/system-tests:refactor-packer-testing-workflows

[Jack Lloyd-Walters]

refactors the image promotion workflows

refactor the image promotion workflow to align with https://code.launchpad.net/~lloydwaltersj/maas-ci/+git/system-tests/+merge/457784

Targets that MP, as it requires most of the refactor changes made there.

3619cc7... by Jack Lloyd-Walters on 2024-01-04

remove return all logs left in function

915d20e... by Jack Lloyd-Walters on 2024-01-04

fetch centos eol

c952750... by Jack Lloyd-Walters on 2024-01-05

don't promote if nothing changed

a3e7e60... by Jack Lloyd-Walters on 2024-01-05

move to combined area

0669c9e... by Jack Lloyd-Walters on 2024-01-05

move to combined area

[Jack Lloyd-Walters (lloydwaltersj)]

jenkins: !test

f7c92e7... by Jack Lloyd-Walters on 2024-01-09

convert classes to CamelCase

4f099a1... by Jack Lloyd-Walters on 2024-01-11

add canonilibraries

8f232f6... by Jack Lloyd-Walters on 2024-01-11

fixes

7c1cc6b... by Jack Lloyd-Walters on 2024-01-11

remove http:// from localhost url

5659603... by Jack Lloyd-Walters on 2024-01-12

fix workflow trigger

75272ef... by Jack Lloyd-Walters on 2024-01-12

linting

60d5983... by Jack Lloyd-Walters on 2024-01-12

fix id

Unmerged commits

60d5983... by Jack Lloyd-Walters on 2024-01-12

fix id

lint:0 (build)

1 → 1 of 1 result

First • Previous • Next • Last

75272ef... by Jack Lloyd-Walters on 2024-01-12

linting

lint:0 (build)

1 → 1 of 1 result

First • Previous • Next • Last

5659603... by Jack Lloyd-Walters on 2024-01-12

fix workflow trigger

lint:0 (build)

1 → 1 of 1 result

First • Previous • Next • Last

7c1cc6b... by Jack Lloyd-Walters on 2024-01-11

remove http:// from localhost url

8f232f6... by Jack Lloyd-Walters on 2024-01-11

fixes

4f099a1... by Jack Lloyd-Walters on 2024-01-11

add canonilibraries

f7c92e7... by Jack Lloyd-Walters on 2024-01-09

convert classes to CamelCase

0669c9e... by Jack Lloyd-Walters on 2024-01-05

move to combined area

f3b54e4... by Jack Lloyd-Walters on 2024-01-03

refactor and new workflows

a3e7e60... by Jack Lloyd-Walters on 2024-01-05

move to combined area

diff --git a/temporal/common/argument_parser.py b/temporal/common/argument_parser.py
index 64a4e3d..cf19172 100644
--- a/temporal/common/argument_parser.py
+++ b/temporal/common/argument_parser.py
@@ -5,7 +5,7 @@ import sys
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import TypeVar
+from typing import Any, TypeVar
 
 from dateutil.relativedelta import relativedelta
 from temporalio import workflow
@@ -22,7 +22,36 @@ def readable_list(*items: str) -> str:
     return ", ".join(items[:-1]) + f", and {items[-1]}"
 
 
-def iso8601todelta(iso_string: str) -> relativedelta:
+class RelativeDelta(relativedelta):
+    """Make relativedelta serialisable"""
+
+    @classmethod
+    def parse_obj(cls: type[T], obj: Any) -> T:
+        return cls(**obj)
+
+    def dict(self) -> dict[str, Any]:
+        return {
+            "years": self.years,
+            "months": self.months,
+            "days": self.days,
+            "leapdays": self.leapdays,
+            "weeks": self.weeks,
+            "hours": self.hours,
+            "minutes": self.minutes,
+            "seconds": self.seconds,
+            "microseconds": self.microseconds,
+            "year": self.year,
+            "month": self.month,
+            "day": self.day,
+            "weekday": self.weekday,
+            "hour": self.hour,
+            "minute": self.minute,
+            "second": self.second,
+            "microsecond": self.microsecond,
+        }
+
+
+def iso8601todelta(iso_string: str) -> RelativeDelta:
     # search for an ISO8601 duration: P[n]Y[n]M[n]W[n]DT[n]H[n]M[n]S
     if duration := re.search(
         r"P(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T)?(\d+H)?(\d+M)?(\d+S)?", iso_string
@@ -48,7 +77,7 @@ def iso8601todelta(iso_string: str) -> relativedelta:
                 return int(digit.group())
             return 0
 
-        return relativedelta(
+        return RelativeDelta(
             years=num(years),
             months=num(months),
             weeks=num(weeks),
diff --git a/temporal/common/common.py b/temporal/common/common.py
index 86bde71..859ee00 100644
--- a/temporal/common/common.py
+++ b/temporal/common/common.py
@@ -1,5 +1,3 @@
-"""Common functionality for temporal."""
-
 import asyncio
 import re
 import sys
diff --git a/temporal/common/github_functions.py b/temporal/common/github_functions.py
index 4d71b9f..0c1acfb 100644
--- a/temporal/common/github_functions.py
+++ b/temporal/common/github_functions.py
@@ -6,23 +6,27 @@ from pathlib import Path
 from subprocess import CompletedProcess, run
 from typing import Iterator
 
+COMMIT_MESSAGE_REGEX = re.compile(r"[0-9a-f]+\s([^\n]+)")
+
 
 @dataclass
 class GitRepo:
-    _repo_url: str
-    _repo_path: str | Path
+    # reccomended to accese these values using the parameters instead
+    # url() and path(), path_str(), directory() or directory_str()
+    repo_url_string: str
+    repo_path_string: str | Path
 
     # properties
     @cached_property
     def url(self) -> str:
-        return self._repo_url
+        return self.repo_url_string
 
     def _as_path_(self, file_path: str | Path) -> Path:
         return file_path if isinstance(file_path, Path) else Path(file_path)
 
     @cached_property
     def path(self) -> Path:
-        return self._as_path_(self._repo_path)
+        return self._as_path_(self.repo_path_string)
 
     @cached_property
     def path_str(self) -> str:
@@ -83,8 +87,11 @@ class GitRepo:
         return "main" if "main" in self.branches else "master"
 
     def remote_of(self, branch: str) -> str | None:
-        if remote_branch := re.search(
-            f"remotes/(.*{branch})", self.execute(["git", "branch", "-a"])
+        # we can't determine remote branches if the repo isn't cloned.
+        if self.exists and (
+            remote_branch := re.search(
+                f"remotes/(.*{branch})", "\n".join(self.branches)
+            )
         ):
             return remote_branch.group()
         return None
@@ -171,6 +178,10 @@ class GitRepo:
         self.checkout(branch)
         return new_branch
 
+    def clean_checkout(self, branch: str) -> bool:
+        self.reset_to_remote()
+        return self.checkout(branch)
+
     def pull(self) -> None:
         self.execute(["git", "pull"])
 
@@ -196,8 +207,8 @@ class GitRepo:
     def all_commits(self) -> list[str]:
         return [
             str(msg)
-            for msg in re.findall(
-                r"[0-9a-f]+\s([^\n]+)", self.execute(["git", "log", "--oneline"])
+            for msg in COMMIT_MESSAGE_REGEX.findall(
+                self.execute(["git", "log", "--oneline"])
             )
         ]
 
@@ -221,7 +232,13 @@ class GitRepo:
             and (cmts := self.all_commits)
             and msg in cmts
         ):
-            return cmts.index(msg)
+            idx = 0
+            for newidx, cmt in enumerate(cmts):
+                if idx == newidx and cmt == msg:
+                    idx += 1
+                else:
+                    break
+            return idx
         return None
 
     # staging
@@ -231,8 +248,8 @@ class GitRepo:
 
     def staged_file(self) -> dict[str, list[str]]:
         staged: dict[str, list[str]] = {"changed": [], "staged": []}
-        for _file in self.files:
-            staged["changed" if _file[0] == " " else "staged"].append(_file)
+        for file_ in self.files:
+            staged["changed" if file_[0] == " " else "staged"].append(file_)
         return staged
 
     def _file_path_(self, filename: str | Path) -> Path:
@@ -258,10 +275,17 @@ class GitRepo:
         base_branch: str | None = None,
         add_files: str | list[str] | None = None,
         orphan: bool = False,
+        clean_tree: bool = True,
     ) -> Iterator[None]:
+        if clean_tree:
+            self.reset_to_remote()
         # checkout the branch
         if self.branch_exists(branch):
-            new_branch = self.checkout_from(branch, base_branch or self.main)
+            from_branch = base_branch or self.main
+            if branch == from_branch:
+                new_branch = self.checkout(branch)
+            else:
+                new_branch = self.checkout_from(branch, base_branch or self.main)
         elif orphan:
             new_branch = self.checkout_orphan(branch)
         force_push: bool = self.diverged
diff --git a/temporal/common/jenkins_functions.py b/temporal/common/jenkins_functions.py
index 1964793..f88f4da 100644
--- a/temporal/common/jenkins_functions.py
+++ b/temporal/common/jenkins_functions.py
@@ -95,8 +95,8 @@ class JenkinsBuild(Build, Serializable):  # type:ignore [misc]
         return int(self.get_number())
 
     def _artifact_(self, name: str) -> Artifact | None:
-        for a_name, artifact in self._artifacts_.items():
-            if a_name == name:
+        for artifact_name, artifact in self._artifacts_.items():
+            if artifact_name == name:
                 return artifact
         return None
 
@@ -127,7 +127,9 @@ class JenkinsBuild(Build, Serializable):  # type:ignore [misc]
 
     @cached_property
     def results(self) -> dict[str, Any]:
-        return {k: v.__dict__ for k, v in self.get_resultset().items()}
+        if self.has_resultset():
+            return {k: v.__dict__ for k, v in self.get_resultset().items()}
+        return {}
 
     @cached_property
     def result_statuses(self) -> dict[str, bool]:
diff --git a/temporal/common/jenkins_workflows.py b/temporal/common/jenkins_workflows.py
index e1b97ed..cfd8534 100644
--- a/temporal/common/jenkins_workflows.py
+++ b/temporal/common/jenkins_workflows.py
@@ -22,7 +22,7 @@ class JenkinsTesterWorkflowParams(TestWorkflowParams):
 
     # job details
     job_name: str = ""
-    job_parameters: dict[str, str] = field(default_factory=dict)
+    job_parameters: dict[str, str | int | bool] = field(default_factory=dict)
     job_build: int | None = None
 
     # Any jobs that will be triggered by the primary job
@@ -163,7 +163,7 @@ async def fetch_downstream_build(params: FetchDownstreamBuildParams) -> int:
     ds_job_name = params.downstream_job
 
     for test_num in range(
-        max(1, params.upstream_build), server.next_build_number(ds_job_name)
+        max(1, params.downstream_build), server.next_build_number(ds_job_name)
     )[::-1]:
         build: JenkinsBuild = server.build(ds_job_name, test_num)
         if (us_job_name, us_build_num) in build.upstream_jobs:
@@ -189,7 +189,7 @@ async def fetch_job_details(params: FetchJobDetailsParams) -> dict[str, Any]:
             return "-".join(groups)
         return default or to_search
 
-    # if we're searching for specific log sections
+    # only return logs we've specifically asked for
     output_logs: dict[str, str] = {}
     if log_parts := params.log_parts:
         for log_name, log_content in build.logs.items():
@@ -227,9 +227,6 @@ async def fetch_job_details(params: FetchJobDetailsParams) -> dict[str, Any]:
                     # otherwise return the whole log
                     else:
                         output_logs[name] = log_content
-    # otherwise, return all the logs
-    else:
-        output_logs = build.logs
     # return the job details
     return {
         "logs": output_logs,
@@ -327,6 +324,12 @@ class JenkinsTesterWorkflow:
             start_to_close_timeout=params.to_delta(params.fetch_results_timeout),
         )
 
+        # if we're testing child jobs, the parent must have passed also
+        if sub_jobs:
+            assert job_details[
+                "passed"
+            ], "Cannot search for children of a failed parent!"
+
         # itterate on any sub jobs
         for downstream_name, downstream_number in sub_job_min_numbers.items():
             # fetch an old build or request new
diff --git a/temporal/start_packer_cron.py b/temporal/start_packer_cron.py
index 354a9f7..1564867 100644
--- a/temporal/start_packer_cron.py
+++ b/temporal/start_packer_cron.py
@@ -32,7 +32,7 @@ async def main(
         await client.start_workflow(
             SchedulePackerImageTests.run,
             params,
-            id="Determine packer tests",
+            id="Determine-packer-tests",
             task_queue=task_queue,
             start_delay=schedule.start - now
             if schedule.start and (now := datetime.utcnow()) and schedule.start > now
@@ -45,7 +45,7 @@ async def main(
                 action=ScheduleActionStartWorkflow(
                     SchedulePackerImageTests.run,
                     params,
-                    id="Determine packer tests",
+                    id="Determine-packer-tests",
                     task_queue=task_queue,
                 ),
                 spec=ScheduleSpec(
@@ -59,7 +59,9 @@ async def main(
 
 if __name__ == "__main__":
     # parser
-    parser = WorkflowParser(description="Starts an image test for the supplied images.")
+    parser = WorkflowParser(
+        description="Starts a cron job to select packer images for testing."
+    )
     parser.add_cron()
 
     # test specific
diff --git a/temporal/start_promotion_cron.py b/temporal/start_promotion_cron.py
new file mode 100644
index 0000000..89878e9
--- /dev/null
+++ b/temporal/start_promotion_cron.py
@@ -0,0 +1,182 @@
+import asyncio
+from datetime import datetime
+
+from temporalio import workflow
+from temporalio.client import (
+    Schedule,
+    ScheduleActionStartWorkflow,
+    ScheduleIntervalSpec,
+    ScheduleSpec,
+)
+
+with workflow.unsafe.imports_passed_through():
+    from .common.argument_parser import WorkflowParser, iso8601todelta
+    from .common.common import get_authenticated_client, task_queue, validateurl
+    from .workflows.promotion_image_cron import (
+        ScheduleImagePromotionTests,
+        ScheduleImagePromotionTestsParams,
+    )
+
+
+async def main(
+    parser: WorkflowParser,
+) -> None:
+    params, conn, schedule = parser.parse_argv(ScheduleImagePromotionTestsParams)
+    client = await get_authenticated_client(
+        connection=conn,
+        root_cas_key=parser.lets_encrypt_root_cas,
+        task_queue=task_queue,
+    )
+
+    if not schedule.every:
+        await client.start_workflow(
+            ScheduleImagePromotionTests.run,
+            params,
+            id="Determine-promotion-targets",
+            task_queue=task_queue,
+            start_delay=schedule.start - now
+            if schedule.start and (now := datetime.utcnow()) and schedule.start > now
+            else None,
+        )
+    else:
+        await client.create_schedule(
+            "Promotion-test-scheduler",
+            schedule=Schedule(
+                action=ScheduleActionStartWorkflow(
+                    ScheduleImagePromotionTests.run,
+                    params,
+                    id="Determine-promotion-targets",
+                    task_queue=task_queue,
+                ),
+                spec=ScheduleSpec(
+                    start_at=schedule.start,
+                    end_at=schedule.end,
+                    intervals=[ScheduleIntervalSpec(every=schedule.every)],
+                ),
+            ),
+        )
+
+
+if __name__ == "__main__":
+    # parser
+    parser = WorkflowParser(
+        description="Starts a cron job to select promotion images for testing."
+    )
+    parser.add_cron()
+
+    # for this parser
+    parser.add_argument(
+        "--image-stream",
+        "--stream",
+        type=validateurl,
+        default="https://images.maas.io/ephemeral-v3/candidate/",
+        help="The image stream to pull images from.",
+    )
+    parser.add_argument(
+        "--maas-snap-channel",
+        "--snap-channel",
+        "--sc",
+        type=str,
+        default="latest/edge",
+        help="The snap channel to install MAAS for testing with.",
+    )
+    parser.add_argument(
+        "--silent-messages",
+        "--sm",
+        action="store_true",
+        help="""Flag to prevent Mattermost messages being sent.""",
+    )
+    parser.add_argument(
+        "--include-unreleased",
+        "--iu",
+        action="store_true",
+        help="""Flag to include unreleased images in the tests.""",
+    )
+
+    # test specific
+    testing_group = parser.add_argument_group(
+        "Testing and building configurations", "Modify the testing and building jobs."
+    )
+    testing_group.add_argument(
+        "--test",
+        action="store_true",
+        help="""Flag to test the results of the input arguments
+        without progressing further.""",
+    )
+    testing_group.add_argument(
+        "--patch-file",
+        "--pf",
+        type=str,
+        default="conf/cpc-patch.yaml",
+        help="The file containing the arches for the stream.",
+    )
+    testing_group.add_argument(
+        "--packer-file",
+        "--pkf",
+        type=str,
+        default="conf/packer-maas.yaml",
+        help="The file containing release details about packer-based images.",
+    )
+    testing_group.add_argument(
+        "--ubuntu-file",
+        "--uf",
+        type=str,
+        default="conf/meph-v3.yaml",
+        help="The file containing release details about ubuntu images.",
+    )
+    testing_group.add_argument(
+        "--support-standard",
+        "--sstand",
+        type=iso8601todelta,
+        default="P9M",
+        help="""ISO Duration -  P[n]Y[n]M[n]W[n]DT[n]H[n]M[n]S
+        Length of standard support window for ubuntu images.""",
+    )
+    testing_group.add_argument(
+        "--support-eol",
+        "--seol",
+        type=iso8601todelta,
+        default="P5Y",
+        help="""ISO Duration -  P[n]Y[n]M[n]W[n]DT[n]H[n]M[n]S
+        Length of end of life window for ubuntu images.""",
+    )
+    testing_group.add_argument(
+        "--support-esm",
+        "--sesm",
+        type=iso8601todelta,
+        default="P10Y",
+        help="""ISO Duration -  P[n]Y[n]M[n]W[n]DT[n]H[n]M[n]S
+        Length of extended security maintenance window for ubuntu images.""",
+    )
+
+    # jenkins
+    parser.add_jenkins(jenkins_url="http://maas-integration-ci.internal:8080")
+
+    # git repos
+    parser.add_repository(
+        repo_name="image repo",
+        repo_usage="pushing successful candidate images",
+        repo_url="https://git.launchpad.net/maas-images",
+        repo_location="~/maas-images",
+    )
+    parser.add_repository(
+        repo_name="maas git",
+        repo_usage="pulling maas source",
+        repo_url="https://git.launchpad.net/maas",
+        repo_branch="master",
+    )
+    parser.add_repository(
+        repo_name="system test",
+        repo_usage="testing images",
+        repo_url="https://git.launchpad.net/~maas-committers/maas-ci/+git/system-tests",
+        repo_branch="master",
+    )
+
+    # timeouts
+    parser.add_timeouts(parameter_class=ScheduleImagePromotionTestsParams)
+
+    # connectivity
+    parser.add_connections(url="localhost:7233", namespace="image-testing")
+
+    # execute
+    asyncio.run(main(parser))
diff --git a/temporal/test_promotion_images.py b/temporal/test_promotion_images.py
new file mode 100644
index 0000000..f57f112
--- /dev/null
+++ b/temporal/test_promotion_images.py
@@ -0,0 +1,153 @@
+import asyncio
+from datetime import datetime
+
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from .common.argument_parser import WorkflowParser
+    from .common.common import get_authenticated_client, task_queue, validateurl
+    from .common.github_functions import GitRepo
+    from .workflows.promotion_image_tester import (
+        TestPromotionImages,
+        TestPromotionImagesParams,
+        image_names,
+    )
+
+
+async def main(
+    parser: WorkflowParser,
+) -> None:
+    params, conn, _ = parser.parse_argv(TestPromotionImagesParams)
+    client = await get_authenticated_client(
+        connection=conn,
+        root_cas_key=parser.lets_encrypt_root_cas,
+        task_queue=task_queue,
+    )
+    dt = datetime.utcnow().strftime("%Y%m%d-%H%M%S%f")[:-3]
+
+    await client.start_workflow(
+        TestPromotionImages.run,
+        params,
+        id="-".join(["image", "promotion"] + params.images + [dt]),
+        task_queue=task_queue,
+    )
+
+
+def fetch_promotion_images(
+    repo_url: str = "https://git.launchpad.net/maas-images",
+    packer_file: str = "conf/packer-maas.yaml",
+    ubuntu_file: str = "conf/meph-v3.yaml",
+) -> list[str]:
+    image_repo = GitRepo(repo_url, "~/maas-images")
+    image_repo.clone_clean()
+    images = image_names(
+        image_repo=image_repo, packer_file=packer_file, ubuntu_file=ubuntu_file
+    )
+    return list(dict(sorted(images.items())).values())
+
+
+if __name__ == "__main__":
+    images = fetch_promotion_images()
+
+    # parser
+    parser = WorkflowParser(description="Tests and promotes the supplied images.")
+
+    # base args
+    parser.add_argument(
+        "images",
+        type=str,
+        help="Name of an image to be promoted.",
+        choices=images,
+        nargs="+",
+    )
+    parser.add_argument(
+        "--image-stream",
+        "--stream",
+        type=validateurl,
+        default="https://images.maas.io/ephemeral-v3/candidate/",
+        help="The image stream to pull images from.",
+    )
+    parser.add_argument(
+        "--maas-snap-channel",
+        "--snap-channel",
+        "--sc",
+        type=str,
+        default="latest/edge",
+        help="The snap channel to install MAAS for testing with.",
+    )
+    parser.add_argument(
+        "--silent-messages",
+        "--sm",
+        action="store_true",
+        help="""Flag to prevent Mattermost messages being sent.""",
+    )
+    parser.add_argument(
+        "--include-unreleased",
+        "--iu",
+        action="store_true",
+        help="""Flag to include unreleased images in the tests.""",
+    )
+
+    # test specific
+    testing_group = parser.add_argument_group(
+        "Testing and building configurations", "Modify the testing and building jobs."
+    )
+    testing_group.add_argument(
+        "--test",
+        action="store_true",
+        help="""Flag to test the results of the input arguments
+        without progressing further.""",
+    )
+    testing_group.add_argument(
+        "--patch-file",
+        "--pf",
+        type=str,
+        default="conf/cpc-patch.yaml",
+        help="The file containing the arches for the stream.",
+    )
+    testing_group.add_argument(
+        "--packer-file",
+        "--pkf",
+        type=str,
+        default="conf/packer-maas.yaml",
+        help="The file containing release details about packer-based images.",
+    )
+    testing_group.add_argument(
+        "--ubuntu-file",
+        "--uf",
+        type=str,
+        default="conf/meph-v3.yaml",
+        help="The file containing release details about ubuntu images.",
+    )
+
+    # jenkins
+    parser.add_jenkins(jenkins_url="http://maas-integration-ci.internal:8080")
+
+    # git repos
+    parser.add_repository(
+        repo_name="image repo",
+        repo_usage="pushing successful candidate images",
+        repo_url="https://git.launchpad.net/maas-images",
+        repo_location="~/maas-images",
+    )
+    parser.add_repository(
+        repo_name="maas git",
+        repo_usage="pulling maas source",
+        repo_url="https://git.launchpad.net/maas",
+        repo_branch="master",
+    )
+    parser.add_repository(
+        repo_name="system test",
+        repo_usage="testing images",
+        repo_url="https://git.launchpad.net/~maas-committers/maas-ci/+git/system-tests",
+        repo_branch="master",
+    )
+
+    # timeouts
+    parser.add_timeouts(parameter_class=TestPromotionImagesParams)
+
+    # connectivity
+    parser.add_connections(url="localhost:7233", namespace="image-testing")
+
+    # execute
+    asyncio.run(main(parser))
diff --git a/temporal/workflows/matrix_testing_worker.py b/temporal/workflows/matrix_testing_worker.py
index a5da67a..31ada0d 100644
--- a/temporal/workflows/matrix_testing_worker.py
+++ b/temporal/workflows/matrix_testing_worker.py
@@ -6,10 +6,20 @@ with workflow.unsafe.imports_passed_through():
     from .packer_image_cron import workflows as packer_cron_workflows
     from .packer_image_tester import activities as packer_image_activities
     from .packer_image_tester import workflows as packer_image_workflows
+    from .promotion_image_cron import activities as promotion_cron_activities
+    from .promotion_image_cron import workflows as promotion_cron_workflows
+    from .promotion_image_tester import activities as promotion_image_activities
+    from .promotion_image_tester import workflows as promotion_image_workflows
 
 if __name__ == "__main__":
     start_worker(
         task_queue=task_queue,
-        workflows=packer_image_workflows + packer_cron_workflows,
-        activities=packer_image_activities + packer_cron_activies,
+        workflows=packer_image_workflows
+        + packer_cron_workflows
+        + promotion_image_workflows
+        + promotion_cron_workflows,
+        activities=packer_image_activities
+        + packer_cron_activies
+        + promotion_image_activities
+        + promotion_cron_activities,
     )
diff --git a/temporal/workflows/packer_image_cron.py b/temporal/workflows/packer_image_cron.py
index c3a1ed2..2b0b16c 100644
--- a/temporal/workflows/packer_image_cron.py
+++ b/temporal/workflows/packer_image_cron.py
@@ -12,6 +12,7 @@ with workflow.unsafe.imports_passed_through():
         nested_dict,
         read_yaml,
         snap_channels,
+        task_queue,
         todict,
     )
     from ..common.github_workflows import (
@@ -328,6 +329,10 @@ class SchedulePackerImageTests:
                     results_file=params.results_file,
                     results_repo=params.results_repo,
                     results_location=params.results_location,
+                    packer_maas_repo=params.packer_maas_repo,
+                    packer_maas_branch=params.packer_maas_branch,
+                    system_test_repo=params.system_test_repo,
+                    system_test_branch=params.system_test_branch,
                     # jenkins details
                     jenkins_url=params.jenkins_url,
                     jenkins_user=params.jenkins_user,
@@ -343,8 +348,8 @@ class SchedulePackerImageTests:
                     fetch_results_timeout=params.fetch_results_timeout,
                     default_timeout=params.default_timeout,
                 ),
-                task_queue="mono_queue",
-                id=f"Starting {test_image}-{test_channel} Test",
+                task_queue=task_queue,
+                id=f"Starting-{test_image}-{test_channel}-Test",
             )
 
 
diff --git a/temporal/workflows/packer_image_tester.py b/temporal/workflows/packer_image_tester.py
index f8fded6..98c74bc 100644
--- a/temporal/workflows/packer_image_tester.py
+++ b/temporal/workflows/packer_image_tester.py
@@ -13,6 +13,7 @@ with workflow.unsafe.imports_passed_through():
         matches_image,
         nested_dict,
         read_yaml,
+        task_queue,
         todict,
     )
     from ..common.github_workflows import ReportToMatrix, ReportToMatrixParams
@@ -38,6 +39,16 @@ STATUS_MAPPING = {
 }
 POSITIVE_STATUSES = ["passed", "fixed"]
 
+RESULT_NAME_SEARCH = re.compile(
+    r"\[(?P<machine>.*)\.(?P<arch>.*)\-(?P<image>.*)\-(?P<step>.*)\]"
+)
+FEATURE_NAME_SEARCH = re.compile(
+    r"(?P<machine>.*)\-(?P<step>.*)\-(?P<feature_group>.*)\-(?P<feature_config>.*)"
+)
+MAAS_VERSIONS = re.compile(
+    r"maas\s+(?P<long_version>(?P<short_version>\d+\.\d+)[^\s]+)"
+)
+
 
 @dataclass
 class TestPackerImagesParams(TestWorkflowParams):
@@ -181,13 +192,6 @@ async def parse_test_results(params: ParseTestParams) -> dict[str, dict[str, Any
     image_results = params.image_results
     name_from_mapping = params.name_from_mapping
 
-    result_name_search = re.compile(
-        r"\[(?P<machine>.*)\.(?P<arch>.*)\-(?P<image>.*)\-(?P<step>.*)\]"
-    )
-    feature_name_search = re.compile(
-        r"(?P<machine>.*)\-(?P<step>.*)\-(?P<feature_group>.*)\-(?P<feature_config>.*)"
-    )
-
     output_results = nested_dict()
     for image in params.images:
         image_oseries_name = name_from_mapping.get(image, image)
@@ -199,7 +203,7 @@ async def parse_test_results(params: ParseTestParams) -> dict[str, dict[str, Any
         for result_name, result_status in sorted(
             this_test_results.get("result_statuses", {}).items()
         ):
-            if search := result_name_search.search(result_name):
+            if search := RESULT_NAME_SEARCH.search(result_name):
                 if not matches_image(
                     search.group("image"), [image, image_oseries_name]
                 ):
@@ -234,7 +238,7 @@ async def parse_test_results(params: ParseTestParams) -> dict[str, dict[str, Any
                         this_test_results.get("logs", {}).items()
                     ):
                         if (
-                            feature_search := feature_name_search.search(feature_name)
+                            feature_search := FEATURE_NAME_SEARCH.search(feature_name)
                         ) and (feature_search.group("machine") == this_machine):
                             feature, config = feature_search.group(
                                 "feature_group", "feature_config"
@@ -315,12 +319,7 @@ async def parse_test_results(params: ParseTestParams) -> dict[str, dict[str, Any
             maas_version := image_results.get(image, {})
             .get("logs", {})
             .get("env_builder-maas")
-        ) and (
-            search := re.search(
-                r"maas\s+(?P<long_version>(?P<short_version>\d+\.\d+)[^\s]+)",
-                maas_version,
-            )
-        ):
+        ) and (search := MAAS_VERSIONS.search(maas_version)):
             this_image_details["maas_long_version"] = search.group("long_version")
             this_image_details["maas_short_version"] = search.group("short_version")
 
@@ -339,6 +338,9 @@ async def parse_failed_results(params: ParseTestParams) -> dict[str, dict[str, A
     image_results = params.image_results
     name_from_mapping = params.name_from_mapping
 
+    build_status = str(build_details["status"])
+    test_status = str(image_results["status"])
+
     output_results: dict[str, dict[str, Any]] = {}
     for image in params.images:
         build_logs: dict[str, str] = build_details["logs"]
@@ -356,6 +358,7 @@ async def parse_failed_results(params: ParseTestParams) -> dict[str, dict[str, A
             ]
             + ["failed"]
         )[0]
+        tests_run = image_results.get(image, {}).get("results", {})
         features: list[str] = [
             feat
             for feat_name, feat in image_results.get(image, {}).get("logs", {}).items()
@@ -363,16 +366,26 @@ async def parse_failed_results(params: ParseTestParams) -> dict[str, dict[str, A
         ]
 
         summary: str = ""
-        if build_details["status"] == "aborted":
-            summary = "Build job aborted."
+        if build_status == "aborted":
+            summary = "Build job was aborted."
+        elif test_status == "aborted":
+            summary = "Test job was aborted."
         elif not build_details["passed"]:
             summary = "Build job failed."
         elif image_build_status not in POSITIVE_STATUSES:
             summary = "Could not build image."
         elif not image_results.get(image, {}):
-            summary = "Could not test image."
+            summary = "Could not run image test."
+        elif not ["deploy" in run for run in tests_run.keys()]:
+            summary = "Test failed before image deployment."
+        elif [
+            run["status"] not in POSITIVE_STATUSES
+            for name, run in tests_run.items()
+            if "deploy" in name
+        ]:
+            summary = "Could not deploy image."
         elif not features:
-            summary = "Could not test any features."
+            summary = "Could not test image features."
 
         image_details: dict[str, Any] = {
             "architectures": [],
@@ -391,12 +404,7 @@ async def parse_failed_results(params: ParseTestParams) -> dict[str, dict[str, A
             maas_version := image_results.get(image, {})
             .get("logs", {})
             .get("env_builder-maas")
-        ) and (
-            search := re.search(
-                r"maas\s+(?P<long_version>(?P<short_version>\d+\.\d+)[^\s]+)",
-                maas_version,
-            )
-        ):
+        ) and (search := MAAS_VERSIONS.search(maas_version)):
             image_details["maas_long_version"] = search.group("long_version")
             image_details["maas_short_version"] = search.group("short_version")
 
@@ -455,15 +463,16 @@ class TestPackerImages:
                     max_retry_attempts=params.max_retry_attempts,
                     heartbeat_delay=params.heartbeat_delay,
                     # timeouts
+                    default_timeout=params.default_timeout,
                     login_timeout=params.login_timeout,
-                    request_build_timeout=params.request_build_timeout,
+                    read_file_timeout=params.read_file_timeout,
                     start_build_timeout=params.start_build_timeout,
-                    complete_build_timeout=params.complete_build_timeout,
                     fetch_results_timeout=params.fetch_results_timeout,
-                    default_timeout=params.default_timeout,
+                    request_build_timeout=params.request_build_timeout,
+                    complete_build_timeout=params.complete_build_timeout,
                 ),
-                task_queue="mono_queue",
-                id=f"Determining parameters: {', '.join(job_search)}",
+                task_queue=task_queue,
+                id=f"Determining-Parameters:{'-'.join(job_search)}",
             )
 
         # build the images
@@ -500,15 +509,16 @@ class TestPackerImages:
                 max_retry_attempts=params.max_retry_attempts,
                 heartbeat_delay=params.heartbeat_delay,
                 # timeouts
+                default_timeout=params.default_timeout,
                 login_timeout=params.login_timeout,
-                request_build_timeout=params.request_build_timeout,
+                read_file_timeout=params.read_file_timeout,
                 start_build_timeout=params.start_build_timeout,
-                complete_build_timeout=params.complete_build_timeout,
                 fetch_results_timeout=params.fetch_results_timeout,
-                default_timeout=params.default_timeout,
+                request_build_timeout=params.request_build_timeout,
+                complete_build_timeout=params.complete_build_timeout,
             ),
-            task_queue="mono_queue",
-            id=f"Building: {', '.join(params.images)}",
+            task_queue=task_queue,
+            id=f"Building:{'-'.join(params.images)}",
         )
 
         # determine the images that built
@@ -549,15 +559,16 @@ class TestPackerImages:
                     max_retry_attempts=params.max_retry_attempts,
                     heartbeat_delay=params.heartbeat_delay,
                     # timeouts
+                    default_timeout=params.default_timeout,
                     login_timeout=params.login_timeout,
-                    request_build_timeout=params.request_build_timeout,
+                    read_file_timeout=params.read_file_timeout,
                     start_build_timeout=params.start_build_timeout,
-                    complete_build_timeout=params.complete_build_timeout,
                     fetch_results_timeout=params.fetch_results_timeout,
-                    default_timeout=params.default_timeout,
+                    request_build_timeout=params.request_build_timeout,
+                    complete_build_timeout=params.complete_build_timeout,
                 ),
-                task_queue="mono_queue",
-                id=f"Testing: {image}",
+                task_queue=task_queue,
+                id=f"Testing:{image}",
             )
             image_test_results |= {image: image_details}
 
@@ -617,8 +628,8 @@ class TestPackerImages:
                     fetch_results_timeout=params.fetch_results_timeout,
                     default_timeout=params.default_timeout,
                 ),
-                task_queue="mono_queue",
-                id=f"Reporting: {', '.join(output_results.keys())}",
+                task_queue=task_queue,
+                id=f"Reporting:{'-'.join(output_results.keys())}",
             )
         return output_results
 
diff --git a/temporal/workflows/promotion_image_cron.py b/temporal/workflows/promotion_image_cron.py
new file mode 100644
index 0000000..4e5b92e
--- /dev/null
+++ b/temporal/workflows/promotion_image_cron.py
@@ -0,0 +1,242 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+import yaml
+from temporalio import activity, workflow
+from temporalio.common import RetryPolicy
+
+with workflow.unsafe.imports_passed_through():
+    from ..common.argument_parser import RelativeDelta
+    from ..common.common import TestWorkflowParams, task_queue
+    from ..common.github_functions import GitRepo
+    from ..common.github_workflows import FetchGitRepoParams, fetch_git_repo
+    from .promotion_image_tester import (
+        PATCH_DICT_TYPE,
+        DetermineReleaseNamesParams,
+        FetchMephChangesParams,
+        TestPromotionImages,
+        TestPromotionImagesParams,
+    )
+    from .promotion_image_tester import activities as test_promotion_activities
+    from .promotion_image_tester import (
+        determine_release_names,
+        fetch_meph_changes,
+        image_details,
+    )
+    from .promotion_image_tester import workflows as test_promotion_workflows
+
+
+@dataclass
+class ScheduleImagePromotionTestsParams(TestWorkflowParams):
+    image_stream: str = "https://images.maas.io/ephemeral-v3/candidate/"
+
+    maas_snap_channel: str = "latest/edge"
+
+    # files to be accessed
+    patch_file: str = "conf/cpc-patch.yaml"
+    packer_file: str = "conf/packer-maas.yaml"
+    ubuntu_file: str = "conf/meph-v3.yaml"
+
+    # jenkins details
+    jenkins_url: str = ""
+    jenkins_user: str = ""
+    jenkins_pass: str = ""
+
+    # repo
+    system_test_repo: str = (
+        "https://git.launchpad.net/~maas-committers/maas-ci/+git/system-tests"
+    )
+    system_test_branch: str = "master"
+
+    maas_git_repo: str = "https://git.launchpad.net/maas"
+    maas_git_branch: str = "master"
+
+    image_repo_url: str = "https://git.launchpad.net/maas-images"
+    image_repo_location: str = "~/maas-images"
+
+    # what we report to mattermost
+    silent_messages: bool = False
+    include_unreleased: bool = False
+
+    # support windows
+    support_standard: RelativeDelta = RelativeDelta(months=9)
+    support_eol: RelativeDelta = RelativeDelta(years=5)
+    support_esm: RelativeDelta = RelativeDelta(years=10)
+
+
+@dataclass
+class DetermineTestableImagesParams:
+    image_repo: GitRepo
+    packer_file: str
+
+    release_names: dict[str, str]
+    old_patch: PATCH_DICT_TYPE
+    new_patch: PATCH_DICT_TYPE
+
+    include_unreleased: bool
+    support_standard: RelativeDelta
+    support_eol: RelativeDelta
+    support_esm: RelativeDelta
+
+
+def ymd(time: datetime) -> str:
+    return time.strftime("%Y-%m-%d")
+
+
[e-mail]
+async def determine_testable_images(
+    params: DetermineTestableImagesParams,
+) -> dict[str, bool]:
+    """Determine which images are testable, that is:
+    1. The image has new versions in the stream.
+    2. The image is currently supported, and released"""
+    today = ymd(datetime.now())
+
+    with params.image_repo.directory.joinpath(params.packer_file).open() as f:
+        packer_releases: dict[str, dict[str, datetime]] = yaml.safe_load(f).get(
+            "packer-maas", {}
+        )
+
+    testable: dict[str, bool] = {}
+    for image in image_details(
+        release_names=params.release_names,
+        patch=params.new_patch,
+        old_patch=params.old_patch,
+    ):
+        # unchanged images don't need to be worried about
+        if not image.changed:
+            continue
+        # default
+        lts = False
+        released = False
+        end = "Unkown"
+        # ubuntu image
+        if release_date := datetime.strptime(image.release, "%y.%m"):
+            lts = release_date.year % 2 == 0 and release_date.month == 4
+            released = ymd(release_date) < today
+            end = ymd(release_date + params.support_standard)
+            eol = ymd(release_date + params.support_eol)
+            esm = ymd(release_date + params.support_esm)
+        # a packer image we know about
+        elif packer_eol := packer_releases.get(image.name, {}).get("support_eol"):
+            released = True
+            end = packer_eol.strftime("%Y-%m-%d")
+        # determine end of life dates
+        usable = image.detail.get("support_esm_eol", {}).get(
+            "stable", esm if lts else end
+        )
+        support = image.detail.get("support_eol", {}).get("stable", eol if lts else end)
+        # determine supported status
+        testable[image.name] = (
+            testable.get(image.name, False)
+            or (released or params.include_unreleased)
+            and ((usable > today) or (support > today))
+        )
+    return testable
+
+
[e-mail]
+class ScheduleImagePromotionTests:
+    @workflow.run
+    async def run(self, params: ScheduleImagePromotionTestsParams) -> str:
+        # clone the image repo
+        image_repo = await workflow.execute_activity(
+            fetch_git_repo,
+            FetchGitRepoParams(
+                params.image_repo_url,
+                params.image_repo_location,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.request_build_timeout),
+        )
+
+        # fetch the mapping between image release and release name
+        release_names = await workflow.execute_activity(
+            determine_release_names,
+            DetermineReleaseNamesParams(
+                image_repo=image_repo,
+                packer_file=params.packer_file,
+                ubuntu_file=params.ubuntu_file,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.read_file_timeout),
+        )
+
+        # fetch the promotion changes
+        meph_changes = await workflow.execute_activity(
+            fetch_meph_changes,
+            FetchMephChangesParams(
+                image_repo=image_repo,
+                image_stream=params.image_stream,
+                patch_file=params.patch_file,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.read_file_timeout),
+        )
+
+        # determine which images have been changed
+        testable_images = await workflow.execute_activity(
+            determine_testable_images,
+            DetermineTestableImagesParams(
+                image_repo=image_repo,
+                packer_file=params.packer_file,
+                release_names=release_names,
+                old_patch=meph_changes.get("old", {}),
+                new_patch=meph_changes.get("new", {}),
+                include_unreleased=params.include_unreleased,
+                support_standard=params.support_standard,
+                support_eol=params.support_eol,
+                support_esm=params.support_esm,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.fetch_results_timeout),
+        )
+
+        # test those changed images
+        images = [image for image, testable in testable_images.items() if testable]
+        return await workflow.execute_child_workflow(
+            TestPromotionImages,
+            TestPromotionImagesParams(
+                images=images,
+                image_stream=params.image_stream,
+                maas_snap_channel=params.maas_snap_channel,
+                silent_messages=params.silent_messages,
+                include_unreleased=params.include_unreleased,
+                # files
+                patch_file=params.patch_file,
+                packer_file=params.packer_file,
+                ubuntu_file=params.ubuntu_file,
+                # repos
+                image_repo_url=params.image_repo_url,
+                image_repo_location=params.image_repo_location,
+                maas_git_repo=params.maas_git_repo,
+                maas_git_branch=params.maas_git_branch,
+                system_test_repo=params.system_test_repo,
+                system_test_branch=params.system_test_branch,
+                # jenkins details
+                jenkins_url=params.jenkins_url,
+                jenkins_user=params.jenkins_user,
+                jenkins_pass=params.jenkins_pass,
+                # retries
+                max_retry_attempts=params.max_retry_attempts,
+                heartbeat_delay=params.heartbeat_delay,
+                # timeouts
+                login_timeout=params.login_timeout,
+                request_build_timeout=params.request_build_timeout,
+                start_build_timeout=params.start_build_timeout,
+                complete_build_timeout=params.complete_build_timeout,
+                fetch_results_timeout=params.fetch_results_timeout,
+                default_timeout=params.default_timeout,
+            ),
+            task_queue=task_queue,
+            id=f"Promotion-Request:{'-'.join(images)}",
+        )
+
+
+activities = test_promotion_activities + [
+    fetch_git_repo,
+    determine_release_names,
+    fetch_meph_changes,
+    determine_testable_images,
+]
+workflows = test_promotion_workflows + [ScheduleImagePromotionTests]
diff --git a/temporal/workflows/promotion_image_tester.py b/temporal/workflows/promotion_image_tester.py
new file mode 100644
index 0000000..101f3db
--- /dev/null
+++ b/temporal/workflows/promotion_image_tester.py
@@ -0,0 +1,545 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Iterator
+
+import yaml
+from temporalio import activity, workflow
+from temporalio.common import RetryPolicy
+
+with workflow.unsafe.imports_passed_through():
+    from ..common.common import (
+        TestWorkflowParams,
+        current_time,
+        nested_dict,
+        task_queue,
+        todict,
+    )
+    from ..common.github_functions import GitRepo
+    from ..common.github_workflows import FetchGitRepoParams
+    from ..common.github_workflows import activities as github_activities
+    from ..common.github_workflows import fetch_git_repo
+    from ..common.github_workflows import workflows as github_workflows
+    from ..common.jenkins_workflows import (
+        JenkinsTesterWorkflow,
+        JenkinsTesterWorkflowParams,
+    )
+    from ..common.jenkins_workflows import activites as jenkins_activities
+    from ..common.jenkins_workflows import workflows as jenkins_workflows
+
+PATCH_KEYS = {
+    "centos": "com.ubuntu.maas:centos-bases-download.json",
+    "ubuntu": "com.ubuntu.maas:v3:download.json",
+}
+EOL_KEYS = {"support_eol", "support_esm_eol"}
+PATCH_DICT_TYPE = dict[str, dict[str, dict[str, Any]]]
+
+
+@dataclass
+class TestPromotionImagesParams(TestWorkflowParams):
+    images: list[str] = field(default_factory=list)
+    image_stream: str = "https://images.maas.io/ephemeral-v3/candidate/"
+
+    maas_snap_channel: str = "latest/edge"
+
+    # files to be accessed
+    patch_file: str = "conf/cpc-patch.yaml"
+    packer_file: str = "conf/packer-maas.yaml"
+    ubuntu_file: str = "conf/meph-v3.yaml"
+
+    # jenkins details
+    jenkins_url: str = ""
+    jenkins_user: str = ""
+    jenkins_pass: str = ""
+
+    # repositories
+    system_test_repo: str = (
+        "https://git.launchpad.net/~maas-committers/maas-ci/+git/system-tests"
+    )
+    system_test_branch: str = "master"
+
+    maas_git_repo: str = "https://git.launchpad.net/maas"
+    maas_git_branch: str = "master"
+
+    image_repo_url: str = "https://git.launchpad.net/maas-images"
+    image_repo_location: str = "~/maas-images"
+
+    silent_messages: bool = False
+    include_unreleased: bool = False
+
+
+@dataclass
+class FetchMephChangesParams:
+    image_repo: GitRepo
+    image_stream: str
+    patch_file: str = "conf/cpc-patch.yaml"
+
+
[e-mail]
+async def fetch_meph_changes(
+    params: FetchMephChangesParams,
+) -> dict[str, dict[str, Any]]:
+    params.image_repo.clean_checkout(params.image_repo.main)
+
+    with params.image_repo.directory.joinpath(params.patch_file).open() as f:
+        old_patch_file = yaml.safe_load(f)
+
+    params.image_repo.execute(
+        [
+            "./bin/meph2-util",
+            "diff",
+            params.image_stream,
+            "https://images.maas.io/ephemeral-v3/stable",
+            "--new-versions-only",
+            "--latest-only",
+            "--promote",
+            "-o",
+            params.patch_file,
+        ]
+    )
+
+    with params.image_repo.directory.joinpath(params.patch_file).open() as f:
+        new_patch_file = yaml.safe_load(f)
+
+    return {"old": old_patch_file, "new": new_patch_file}
+
+
+@dataclass
+class DetermineReleaseNamesParams:
+    image_repo: GitRepo
+    packer_file: str = "conf/packer-maas.yaml"
+    ubuntu_file: str = "conf/meph-v3.yaml"
+
+
+def image_names(
+    image_repo: GitRepo, packer_file: str, ubuntu_file: str
+) -> dict[str, str]:
+    with image_repo.directory.joinpath(packer_file).open() as f:
+        packer_releases: dict[str, dict[str, int | str]] = yaml.safe_load(f).get(
+            "packer-maas", {}
+        )
+    with image_repo.directory.joinpath(ubuntu_file).open() as f:
+        ubuntu_releases: list[dict[str, str]] = yaml.safe_load(f).get("releases", {})
+
+    packer_names = {
+        f"{details.get('os')}{details.get('version'):0.1f}": image
+        for image, details in packer_releases.items()
+    }
+    ubuntu_names = {
+        release["version"]: release["release"] for release in ubuntu_releases
+    }
+
+    return packer_names | ubuntu_names
+
+
[e-mail]
+async def determine_release_names(
+    params: DetermineReleaseNamesParams,
+) -> dict[str, str]:
+    return image_names(
+        image_repo=params.image_repo,
+        packer_file=params.packer_file,
+        ubuntu_file=params.ubuntu_file,
+    )
+
+
+@dataclass
+class DetermineArchesForImagesParams:
+    images: list[str]
+    release_names: dict[str, str]
+    meph_changes: dict[str, dict[str, Any]]
+
+
+@dataclass
+class ImageDetail:
+    id: str
+    key: str
+    name: str
+    release: str
+    arch: str
+    detail: dict[str, Any]
+    old_detail: dict[str, Any]
+    changed: bool
+
+
+def release_arch(image_id: str) -> tuple[str, str]:
+    """Return the release name and arch from the image id of
+    the cpc_patch keys"""
+    if "centos" in image_id:
+        _, release, arch = image_id.rsplit(":", 2)
+        release = f"centos{release}"
+    else:
+        _, release, arch, _ = image_id.rsplit(":", 3)
+    return release, arch
+
+
+def image_details(
+    release_names: dict[str, str],
+    patch: PATCH_DICT_TYPE,
+    old_patch: PATCH_DICT_TYPE | None = None,
+    combine_keys: bool = False,
+) -> Iterator[ImageDetail]:
+    for key in PATCH_KEYS.values():
+        patch_keys = set(patch.get(key, {}).keys())
+        if old_patch and combine_keys:
+            patch_keys.update(set(old_patch.get(key, {}).keys()))
+        for image_id in patch_keys:
+            release, arch = release_arch(image_id=image_id)
+            if image_name := release_names.get(release):
+                current = patch.get(key, {}).get(image_id, {})
+                old = (old_patch or {}).get(key, {}).get(image_id, {})
+                yield ImageDetail(
+                    id=image_id,
+                    key=key,
+                    name=image_name,
+                    release=release,
+                    arch=arch,
+                    detail=current,
+                    old_detail=old,
+                    changed=(current != old),
+                )
+
+
[e-mail]
+async def determine_arches_for_images(
+    params: DetermineArchesForImagesParams,
+) -> dict[str, set[str]]:
+    """Return the arches that are supported for an image in the stream"""
+    arches_per_image: dict[str, set[str]] = {}
+    # search all of the keys
+    for image in image_details(
+        release_names=params.release_names, patch=params.meph_changes
+    ):
+        if image.name in params.images:
+            arches_per_image[image.name] = arches_per_image.get(
+                image.name, set()
+            ).union({image.arch})
+    return arches_per_image
+
+
+@dataclass
+class GeneratePromotionPatchParams:
+    image_test_status: dict[str, bool]
+    release_names: dict[str, str]
+    old_patch: PATCH_DICT_TYPE
+    new_patch: PATCH_DICT_TYPE
+    patch_file: str = "conf/cpc-patch.yaml"
+    include_unreleased: bool = False
+
+
+def image_released(release_name: str) -> bool:
+    # we only have access to ubunutu images early
+    if release_date := datetime.strptime(release_name, "%y.%m"):
+        return release_date.strftime("%Y-%m-%d") < current_time().split()[0]
+    return True
+
+
[e-mail]
+async def generate_promotion_patch(
+    params: GeneratePromotionPatchParams,
+) -> PATCH_DICT_TYPE:
+    patch: PATCH_DICT_TYPE = nested_dict()
+
+    for image in image_details(
+        release_names=params.release_names,
+        patch=params.new_patch,
+        old_patch=params.old_patch,
+    ):
+        # skip unreleased images early
+        released = image_released(image.release)
+        if not (released or params.include_unreleased):
+            continue
+
+        # nothing changed
+        if not image.changed:
+            patch[image.key][image.id] = image.detail
+            continue
+
+        # the image is allowed to change
+        if params.image_test_status.get(image.name):
+            patch[image.key][image.id] = image.detail
+        else:
+            patch[image.key][image.id] = image.old_detail
+
+        # always allow esm/eol date changed
+        for eol_key in set(image.detail.keys()).intersection(EOL_KEYS):
+            patch[image.key][image.id][eol_key] = image.detail[eol_key]
+
+        # only allow unreleased images to be candidate
+        if not released:
+            for version in image.detail.get("versions", {}).keys():
+                patch[image.key][image.id][version]["labels"] = ["candidate"]
+            if "labels" in image.detail:
+                patch[image.key][image.id]["labels"] = ["candidate"]
+
+    return todict(patch)
+
+
+@dataclass
+class GeneratePromotionMessageParams:
+    old_patch: PATCH_DICT_TYPE
+    new_patch: PATCH_DICT_TYPE
+    release_names: dict[str, str]
+
+
+class ImageChanges(Enum):
+    # eol changes
+    update_image_eol = 0
+    add_image_eol = 1
+    remove_image_eol = 2
+    # image changes
+    update_image = 3
+    add_image = 4
+    remove_image = 5
+
+
[e-mail]
+async def generate_promotion_message(params: GeneratePromotionMessageParams) -> str:
+    changes: dict[str, set[ImageChanges]] = {
+        image_name: set() for image_name in params.release_names.keys()
+    }
+
+    for image in image_details(
+        release_names=params.release_names,
+        patch=params.new_patch,
+        old_patch=params.old_patch,
+        combine_keys=True,
+    ):
+        if not image.changed:
+            continue
+
+        old_keys = set(image.old_detail.keys())
+        new_keys = set(image.detail.keys())
+
+        # a new image was added
+        if not old_keys and new_keys:
+            changes[image.name].add(ImageChanges.add_image)
+
+        # an old image was removed
+        elif not new_keys and old_keys:
+            changes[image.name].add(ImageChanges.remove_image)
+
+        # an image had eol dates added
+        if not old_keys.intersection(EOL_KEYS) and new_keys.intersection(EOL_KEYS):
+            changes[image.name].add(ImageChanges.add_image_eol)
+
+        # an image had eol dates removed
+        elif not new_keys.intersection(EOL_KEYS) and old_keys.intersection(EOL_KEYS):
+            changes[image.name].add(ImageChanges.remove_image_eol)
+
+        # and image had eol dates changed
+        elif new_keys.intersection(EOL_KEYS) != old_keys.intersection(EOL_KEYS):
+            changes[image.name].add(ImageChanges.update_image_eol)
+
+        # some other aspect of the image changed
+        elif new_keys.difference(EOL_KEYS) != old_keys.difference(EOL_KEYS):
+            changes[image.name].add(ImageChanges.update_image)
+
+    # convert from image: changes to change: images
+    changed: dict[ImageChanges, list[str]] = {}
+    for name, change in changes.items():
+        for c in change:
+            changed[c] = changed.get(c, []) + [name]
+
+    return "\n".join(
+        [f"{c.name}: {', '.join(images)}" for c, images in changed.items()]
+    ).strip("\n")
+
+
+@dataclass
+class PromoteImagesParams:
+    image_repo: GitRepo
+    promotion_patch: PATCH_DICT_TYPE
+    promotion_message: str
+    patch_file: str = "conf/cpc-patch.yaml"
+
+
[e-mail]
+async def promote_images(params: PromoteImagesParams) -> None:
+    now = current_time().split()[0]
+    patch_file = params.image_repo.directory.joinpath(params.patch_file)
+
+    with params.image_repo.checkout_and_commit(
+        message=f"promotion workflow {now}\n{params.promotion_message}",
+        branch=params.image_repo.main,
+        add_files=str(patch_file),
+    ):
+        comments = []
+        with patch_file.open() as f:
+            # maintain the comments in the patch
+            for line in f.readlines():
+                if line[0] not in ["#", "\n"]:
+                    break
+                comments.append(line)
+            current_patch = yaml.safe_load(f)
+
+        # splice the new patch into the old
+        patch = {
+            key: value for key, value in current_patch.items if key not in PATCH_KEYS
+        } | params.promotion_patch
+
+        with patch_file.open("w") as f:
+            f.writelines(comments)
+            yaml.safe_dump(patch, f)
+
+
[e-mail]
+class TestPromotionImages:
+    @workflow.run
+    async def run(self, params: TestPromotionImagesParams) -> str:
+        config_job = "maas-system-tests-config-generator"
+        tester_job = "maas-system-tests-executor"
+
+        # clone the image repo
+        image_repo = await workflow.execute_activity(
+            fetch_git_repo,
+            FetchGitRepoParams(
+                params.image_repo_url,
+                params.image_repo_location,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.request_build_timeout),
+        )
+
+        # fetch the mapping between image release and release name
+        release_names = await workflow.execute_activity(
+            determine_release_names,
+            DetermineReleaseNamesParams(
+                image_repo=image_repo,
+                packer_file=params.packer_file,
+                ubuntu_file=params.ubuntu_file,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.read_file_timeout),
+        )
+
+        # fetch the promotion changes
+        meph_changes = await workflow.execute_activity(
+            fetch_meph_changes,
+            FetchMephChangesParams(
+                image_repo=image_repo,
+                image_stream=params.image_stream,
+                patch_file=params.patch_file,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.read_file_timeout),
+        )
+
+        # determine all of the arches to be tested for each image
+        image_arch_mapping = await workflow.execute_activity(
+            determine_arches_for_images,
+            DetermineArchesForImagesParams(
+                images=params.images,
+                release_names=release_names,
+                meph_changes=meph_changes.get("new", {}),
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.read_file_timeout),
+        )
+
+        # request a test for each image
+        image_test_status: dict[str, bool] = {}
+        for image, arches in image_arch_mapping.items():
+            image_job_details = await workflow.execute_child_workflow(
+                JenkinsTesterWorkflow,
+                JenkinsTesterWorkflowParams(
+                    job_name=config_job,
+                    job_parameters={
+                        "CANDIDATE_IMAGE_NAME": image,
+                        "IMAGE_STREAM_URL": params.image_stream,
+                        "SYSTEMTESTS_GIT_REPO": params.system_test_repo,
+                        "SYSTEMTESTS_GIT_BRANCH": params.system_test_branch,
+                        "MAAS_GIT_REPO": params.maas_git_repo,
+                        "MAAS_GIT_BRANCH": params.maas_git_branch,
+                        "MAAS_SNAP_CHANNEL": params.maas_snap_channel,
+                        "TEST_DB_SNAP_CHANNEL": params.maas_snap_channel,
+                        "ENABLE_VAULT": False,
+                        "REPORT_MATTERMOST": not params.silent_messages,
+                        "GEN_CONFIG_ARGS": " ".join(
+                            f"--architecture {arch}" for arch in arches
+                        ),
+                    },
+                    sub_job_names=[tester_job],
+                    # jenkins details
+                    jenkins_url=params.jenkins_url,
+                    jenkins_user=params.jenkins_user,
+                    jenkins_pass=params.jenkins_pass,
+                    # retries
+                    max_retry_attempts=params.max_retry_attempts,
+                    heartbeat_delay=params.heartbeat_delay,
+                    # timeouts
+                    default_timeout=params.default_timeout,
+                    login_timeout=params.login_timeout,
+                    read_file_timeout=params.read_file_timeout,
+                    start_build_timeout=params.start_build_timeout,
+                    fetch_results_timeout=params.fetch_results_timeout,
+                    request_build_timeout=params.request_build_timeout,
+                    complete_build_timeout=params.complete_build_timeout,
+                ),
+                task_queue=task_queue,
+                id=f"Testing:{image}",
+            )
+
+            # store the result of the image test
+            image_test_status[image] = image_job_details.get(tester_job, {}).get(
+                "passed", False
+            )
+
+        # generate the new patch file for the promotion
+        promotion_patch = await workflow.execute_activity(
+            generate_promotion_patch,
+            GeneratePromotionPatchParams(
+                image_test_status=image_test_status,
+                release_names=release_names,
+                old_patch=meph_changes.get("old", {}),
+                new_patch=meph_changes.get("new", {}),
+                patch_file=params.patch_file,
+                include_unreleased=params.include_unreleased,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.fetch_results_timeout),
+        )
+
+        # generate the promotion summary
+        promotion_message = await workflow.execute_activity(
+            generate_promotion_message,
+            GeneratePromotionMessageParams(
+                old_patch=meph_changes.get("old", {}),
+                new_patch=promotion_patch,
+                release_names=release_names,
+            ),
+            retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+            start_to_close_timeout=params.to_delta(params.fetch_results_timeout),
+        )
+
+        # promote if something changes
+        if promotion_message:
+            await workflow.execute_activity(
+                promote_images,
+                PromoteImagesParams(
+                    image_repo=image_repo,
+                    promotion_patch=promotion_patch,
+                    promotion_message=promotion_message,
+                    patch_file=params.patch_file,
+                ),
+                retry_policy=RetryPolicy(maximum_attempts=params.max_retry_attempts),
+                start_to_close_timeout=params.to_delta(params.fetch_results_timeout),
+            )
+
+        return promotion_message or "No changes made to repo."
+
+
+activities = (
+    jenkins_activities
+    + github_activities
+    + [
+        fetch_git_repo,
+        determine_release_names,
+        fetch_meph_changes,
+        determine_arches_for_images,
+        generate_promotion_patch,
+        generate_promotion_message,
+        promote_images,
+    ]
+)
+workflows = jenkins_workflows + github_workflows + [TestPromotionImages]