Merge ~barryprice/launchpad-mojo-specs/+git/private:vbuilder into launchpad-mojo-specs:master

[Barry Price]

Double PS6 riscv64 builders per wgrant for noble rebuild

Unmerged commits

4020d1e... by Barry Price on 2024-03-31

Double PS6 riscv64 builders per wgrant for noble rebuild

bd86c4a... by Colin Watson on 2023-11-15

vbuilder: Bump lxd-channel to latest/candidate on riscv64

This pulls in https://github.com/canonical/lxd/pull/12530.

4fa0592... by Colin Watson on 2023-11-15

vbuilder: Bump sbuild-stalled-package-timeout to 1500 on riscv64

Emulated builders are slow enough that the default timeout of 150
minutes can be too short.

ea99990... by Colin Watson on 2023-11-13

vbuilder: Allow launchpad-bastion-ps5.internal to rsync logs

2cbd741... by Ines Almeida on 2023-11-13

vbuilder: add 40 arm64 builders to bos03 production

591994d... by Ines Almeida on 2023-11-13

vbuilder: add spec for arm64 bos03 builder in production

4626e30... by Ines Almeida on 2023-11-07

vbuilder: add missing config to vbuilders_bos03.arm64

2f6a2b1... by Ines Almeida on 2023-11-07

vbuilder: add openstack server IP to the bos03 arm64 modifiers

80500c7... by Ines Almeida on 2023-11-06

vbuilder: add missing configuration options and reorder bundle alphabetically

c31f140... by Ines Almeida on 2023-11-02

vbuilder: add arm64 arch to bos03 region in qastaging

diff --git a/vbuilder/bundle.yaml b/vbuilder/bundle.yaml
new file mode 100644
index 0000000..c2d3d6d
--- /dev/null
+++ b/vbuilder/bundle.yaml
@@ -0,0 +1,631 @@
+{%- set log_hosts_allow = "carob.canonical.com launchpad-bastion-ps5.internal" %}
+
+{%- if stage_name == "production" %}
+{%-   set clamav_database_url = "http://clamav-database-mirror.lp.internal/" %}
+{%-   set content_id_template = "launchpad-buildd:production" %}
+{%-   set dns_update_host_bos01 = "10.189.0.2" %}
+{%-   set dns_update_host_bos02 = "10.189.128.2" %}
+{%-   set dns_update_host_bos03 = "10.189.128.2" %}
+{%-   set dns_update_host_lcy02 = "10.131.53.11 10.131.53.12 10.131.53.13" %}
+{%-   set dns_update_key_name = "vbuilder-manage" %}
+{%-   set domain_bos01 = "vbuilder.bos01.scalingstack" %}
+{%-   set domain_bos02 = "vbuilder.bos02.scalingstack" %}
+{%-   set domain_bos03 = "vbuilder.bos03.scalingstack" %}
+{%-   set domain_lcy02 = "vbuilder.lcy02.scalingstack" %}
+{%-   set extra_constraints = "root-disk-source=volume" %}
+{%-   set gss_series = "focal|jammy" %}
+{%-   set instance_key_name_bos01 = "vbuilder-manage-bos01" %}
+{%-   set instance_key_name_bos02 = "vbuilder-manage-bos02" %}
+{%-   set instance_key_name_bos03 = "vbuilder-manage-bos03" %}
+{%-   set instance_key_name_lcy02 = "vbuilder-manage-lcy02" %}
+{%-   set instance_network_bos01 = "10.189.16.0/20" %}
+{%-   set instance_network_bos02 = "10.189.144.0/20" %}
+{%-   set instance_network_bos03 = "10.143.0.0/20" %}
+{%-   set instance_network_lcy02 = "10.133.0.0/16" %}
+{%-   set instance_router = "vbuilder_router" %}
+{%-   set instance_router_bos03 = "router_launchpad-vbuilder-production" %}
+{%-   set instance_router_lcy02 = "router_launchpad-vbuilder-production" %}
+{#-   Output of "openstack image show -c id -f value qemu-riscv64-uboot". #}
+{%-   set kernel_id_bos03_riscv64 = "c5015509-91c3-4684-80cc-b8a2ef44c8b5" %}
+{%-   set launchpad_buildd_repository = "ppa:launchpad/buildd" %}
+{%-   set lp_buildd_managers = "10.131.66.156 10.131.215.202" %}
+{%-   set lp_environment = "production" %}
+{%-   set lp_sshkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQD3tBvyEaZFX8H4krfXGhczq9q/GCfcE0kpsjK8WzM/lQOhWQAVSmm8O9gposKvu4mT28hBWI746+NOPLBrf9ax7YRoU+ZuZesyIIK6ztN07G4aK2vt+1W9yNZKgTJZ8FvxHwlbFy6wMjMP3OzuxWyh0yi03z1YiGPJt0riJRZT+QecaoyYhkTuRbcCoWbhmM0veUjCvfR7LA43YbOfmts7STNCtl1IUJErmUY6fSR1LkyF9uJSdwozAVht242vI8Yg+PrOnKCx3X5w546okP2uMDVXKWeY2g/MhUSr+ZyIaS2JxUygZ7FKS4gNLYb3O4Q6tcIwPE++zsr9HueTfjw2LyeQTZQDQjuv0kJ40B3IFF4E9FMzu8MDwArOKUGzISLrX5VtDZBKINDclfDbrFWHUlVBC2CIIjGPKj3afluP9jadWjsOQx0ooBO0tb0Se+7t3oNjKlknwD85AYCPzIlZt7wo/+e/V/Tilw+UUf00JioEceTdxuOHmbuxP5RzCleg8pYLfe5jiHfBJi0DhO61IYlFMgSToh78EShHWYEdSbD+Ve5GWKvPEU9CsSTleSSqBFdhyggfB5fHiup0efAqMnstDn5sxGYFzfFu7SNVHkMuFeiRHZK+9fF/nfzk0UK5EaGs+4IMhWS1ns8m5O5Li609c/nXK5t5fBOkiWa+RQ== lp_buildd@juju-4112d9-prod-launchpad-manual-servers-4" %}
+{%-   set modifiers_bos01 = '{"arm64": "10.43.0.29", "ppc64el": "10.43.0.36", "s390x": "10.43.0.39"}' %}
+{%-   set modifiers_bos02 = '{"arm64": "10.44.0.22", "ppc64el": "10.44.0.20", "s390x": "10.44.0.18"}' %}
+{%-   set modifiers_bos03 = '{"arm64": "10.143.254.129", "riscv64": "10.143.254.223"}' %}
+{%-   set name_prefix = "launchpad-buildd" %}
+{%-   set openstack_tenant_name = "vbuilder_project" %}
+{%-   set openstack_tenant_name_bos03 = "launchpad-vbuilder-production_project" %}
+{%-   set openstack_tenant_name_lcy02 = "launchpad-vbuilder-production_project" %}
+{%-   set openstack_username = "vbuilder" %}
+{%-   set openstack_username_bos03 = "launchpad-vbuilder-production" %}
+{%-   set openstack_username_lcy02 = "launchpad-vbuilder-production" %}
+{%-   set vbuilders_bos01 = {"amd64": {"series": "focal", "flavor": "vbuilder-gpu", "count": 5}, "arm64": {"series": "focal", "count": 40, "config_drive": false}, "arm64-gpu": {"arch_base": "arm64", "arch_suffix": "-gpu", "series": "focal", "flavor": "vbuilder-nvidia-l4", "count": 2, "config_drive": false}, "ppc64el": {"series": "focal", "count": 20}, "s390x": {"series": "focal", "count": 20}} %}
+{%-   set vbuilders_bos02 = {"arm64": {"series": "focal", "count": 80, "config_drive": false}, "ppc64el": {"series": "focal", "count": 30}, "s390x": {"series": "focal", "count": 20}} %}
+{%-   set vbuilders_bos03 = {"amd64": {"series": "focal", "count": 60}, "arm64": {"series": "focal", "count": 40, "config_drive": false, "flavor": "vbuilder-arm64"}, "riscv64": {"series": "jammy", "count": 120, "config_drive": false}} %}
+{%-   set vbuilders_lcy02 = {"amd64": {"series": "focal", "count": 120}} %}
+{%-   set vbuilder_prefix = "" %}
+{%- elif stage_name == "staging" %}
+{#-   This environment is confusingly named, and is actually connected to Launchpad dogfood. #}
+{%-   set clamav_database_url = "http://clamav-database-mirror.staging.lp.internal/" %}
+{%-   set content_id_template = "launchpad-buildd:staging" %}
+{%-   set dns_update_host_bos01 = "10.189.0.2" %}
+{%-   set dns_update_host_bos02 = "10.189.128.2" %}
+{%-   set dns_update_host_bos03 = "10.189.128.2" %}
+{%-   set dns_update_host_lcy02 = "10.132.31.11 10.132.31.12 10.132.31.13" %}
+{%-   set dns_update_key_name = "vbuilder-staging-manage" %}
+{%-   set domain_bos01 = "vbuilder.staging.bos01.scalingstack" %}
+{%-   set domain_bos02 = "vbuilder.staging.bos02.scalingstack" %}
+{%-   set domain_bos03 = "vbuilder.staging.bos03.scalingstack" %}
+{%-   set domain_lcy02 = "vbuilder.staging.lcy02.scalingstack" %}
+{%-   set extra_constraints = "" %}
+{%-   set gss_series = "focal|jammy" %}
+{%-   set instance_key_name_bos01 = "ppa-manage-test" %}
+{%-   set instance_key_name_bos02 = "ppa-manage-test" %}
+{%-   set instance_key_name_bos03 = "ppa-manage-test" %}
+{%-   set instance_key_name_lcy02 = "ppa-manage-test" %}
+{%-   set instance_network_bos01 = "10.189.34.0/24" %}
+{%-   set instance_network_bos02 = "10.189.162.0/23" %}
+{%-   set instance_network_bos03 = "10.144.2.0/23" %}
+{%-   set instance_network_lcy02 = "10.134.2.0/23" %}
+{%-   set instance_router = "vbuilder_staging_router" %}
+{%-   set instance_router_bos03 = "router_launchpad-vbuilder-staging" %}
+{%-   set instance_router_lcy02 = "router_launchpad-vbuilder-staging" %}
+{%-   set launchpad_buildd_repository = "ppa:launchpad/buildd-staging" %}
+{%-   set lp_buildd_managers = "91.189.90.132" %}
+{%-   set lp_environment = "dogfood" %}
+{%-   set lp_sshkey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDB3Qw9541hRDiGVLGxZm/afNsCirlM6Wa2/VkdU22KQfVu579+ek8fdgvR/si7UOTDgE4j7DGuJW+pk7z6T08Iy5feaI3JpnZV7PX0Qp8CilrcTNWPL1eBoq5HcYDzl+zyXT341l7GBFhwYQ50sF3hq0RV0XvUwxfheyBtdmzkiVE1LXT7kdFvXtxe0fR9ypw+NuMRqqFyZ9w3tee7zclDw1cCcnDf6vmIXbYLF9yNZOQQhwYQFFgIUepdkUg2onyhYXWKj8mooFVGne0WPVTJ5Sz805soh9SuUGGgpTh70EtgpJ1nxSWGtIWUtNc6mSGdZzGtgVTnbDk04J4FrVX3Bu8yetlQbNPPYuxdqaZP1anoKmgtCIhfe+xCkim5YLc+WZXVRGvk6apCLXMnj9ZhRE7fCKQO/F+aNPCONv0gUVncxuWAyiqdRuilqSA7VTEMYTv7pIYSNOjpD5eMIX2wGkYTjEXopGJouUH2nOXlhsGgssmMepSVJhOJKY1Cfq0ND4ydoDd2Mz1Yj+Us9HToqJU6DD1sAIKOV05fBqVsJEJbctI2vpRY/R1nCBySpM4KpzgkCQWwjkjR8h2/nuwDtyMsJe/BdBuDyRwJGnBLNgBUg+tnWl9yePz/ZXVGrAI7gPuc9DIiuKEvEmiO3o9yfRRjsFMWGIk2y/Hfa01V7w== launchpad@labbu" %}
+{%-   set modifiers_bos01 = '{"arm64": "10.43.0.10", "ppc64el": "10.43.0.23", "s390x": "10.43.0.15"}' %}
+{%-   set modifiers_bos02 = '{"arm64": "10.44.0.13", "ppc64el": "10.44.0.19", "s390x": "10.44.0.14"}' %}
+{%-   set name_prefix = "launchpad-buildd-staging" %}
+{%-   set openstack_tenant_name = "vbuilder_staging_project" %}
+{%-   set openstack_tenant_name_bos03 = "launchpad-vbuilder-staging_project" %}
+{%-   set openstack_tenant_name_lcy02 = "launchpad-vbuilder-staging_project" %}
+{%-   set openstack_username = "vbuilder_staging" %}
+{%-   set openstack_username_bos03 = "launchpad-vbuilder-staging" %}
+{%-   set openstack_username_lcy02 = "launchpad-vbuilder-staging" %}
+{%-   set vbuilders_bos01 = {"amd64": {"series": "jammy", "flavor": "vbuilder-gpu", "count": 1}, "arm64": {"series": "jammy", "count": 1, "config_drive": false}, "arm64-gpu": {"arch_base": "arm64", "arch_suffix": "-gpu", "series": "jammy", "flavor": "vbuilder-nvidia-l4", "count": 1, "config_drive": false}, "ppc64el": {"series": "jammy", "count": 1}, "s390x": {"series": "jammy", "count": 1}} %}
+{%-   set vbuilders_bos02 = {"arm64": {"series": "jammy", "count": 1, "config_drive": false}, "ppc64el": {"series": "jammy", "count": 1}, "s390x": {"series": "jammy", "count": 1}} %}
+{%-   set vbuilders_bos03 = {"amd64": {"series": "jammy", "count": 4}} %}
+{%-   set vbuilders_lcy02 = {"amd64": {"series": "jammy", "count": 4}} %}
+{%-   set vbuilder_prefix = "dogfood-" %}
+{%- elif stage_name == "qastaging" %}
+{%-   set clamav_database_url = "http://clamav-database-mirror.staging.lp.internal/" %}
+{%-   set content_id_template = "launchpad-buildd:qastaging" %}
+{%-   set dns_update_host_bos01 = "10.189.0.2" %}
+{%-   set dns_update_host_bos02 = "10.189.128.2" %}
+{%-   set dns_update_host_bos03 = "10.189.128.2" %}
+{%-   set dns_update_host_lcy02 = "10.132.31.11 10.132.31.12 10.132.31.13" %}
+{%-   set dns_update_key_name = "vbuilder-staging-manage" %}
+{%-   set domain_bos01 = "vbuilder.qastaging.bos01.scalingstack" %}
+{%-   set domain_bos02 = "vbuilder.qastaging.bos02.scalingstack" %}
+{%-   set domain_bos03 = "vbuilder.qastaging.bos03.scalingstack" %}
+{%-   set domain_lcy02 = "vbuilder.qastaging.lcy02.scalingstack" %}
+{%-   set extra_constraints = "" %}
+{%-   set gss_series = "focal|jammy" %}
+{%-   set instance_key_name_bos01 = "ppa-manage-test-qastaging" %}
+{%-   set instance_key_name_bos02 = "ppa-manage-test-qastaging" %}
+{%-   set instance_key_name_bos03 = "ppa-manage-test-qastaging" %}
+{%-   set instance_key_name_lcy02 = "ppa-manage-test-qastaging" %}
+{%-   set instance_network_bos01 = "10.189.36.0/24" %}
+{%-   set instance_network_bos02 = "10.189.164.0/23" %}
+{%-   set instance_network_bos03 = "10.144.4.0/23" %}
+{%-   set instance_network_lcy02 = "10.134.4.0/23" %}
+{%-   set instance_router = "vbuilder_staging_router" %}
+{%-   set instance_router_bos03 = "router_launchpad-vbuilder-staging" %}
+{%-   set instance_router_lcy02 = "router_launchpad-vbuilder-staging" %}
+{#-   Output of "openstack image show -c id -f value qemu-riscv64-uboot". #}
+{%-   set kernel_id_bos03_riscv64 = "bcbb013b-9424-4f61-9fda-6374d29d3ee0" %}
+{%-   set launchpad_buildd_repository = "ppa:launchpad/buildd-staging" %}
+{%-   set lp_buildd_managers = "10.132.54.242" %}
+{%-   set lp_environment = "qastaging" %}
+{%-   set lp_sshkey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFrjt0yytzrK9fQuG+6VgE6QStUbDmunlN7+Lv5XhmoL stg-launchpad@launchpad-bastion-ps5" %}
+{%-   set modifiers_bos01 = '{"arm64": "10.43.0.10", "ppc64el": "10.43.0.23", "s390x": "10.43.0.15"}' %}
+{%-   set modifiers_bos02 = '{"arm64": "10.44.0.13", "ppc64el": "10.44.0.19", "s390x": "10.44.0.14"}' %}
+{%-   set modifiers_bos03 = '{"arm64": "10.144.0.127", "riscv64": "10.144.0.114"}' %}
+{%-   set name_prefix = "launchpad-buildd-qastaging" %}
+{%-   set openstack_tenant_name = "vbuilder_staging_project" %}
+{%-   set openstack_tenant_name_bos03 = "launchpad-vbuilder-staging_project" %}
+{%-   set openstack_tenant_name_lcy02 = "launchpad-vbuilder-staging_project" %}
+{%-   set openstack_username = "vbuilder_staging" %}
+{%-   set openstack_username_bos03 = "launchpad-vbuilder-staging" %}
+{%-   set openstack_username_lcy02 = "launchpad-vbuilder-staging" %}
+{%-   set vbuilders_bos01 = {"amd64": {"series": "focal", "flavor": "vbuilder-gpu", "count": 1}, "arm64": {"series": "focal", "count": 1, "config_drive": false}, "arm64-gpu": {"arch_base": "arm64", "arch_suffix": "-gpu", "series": "focal", "flavor": "vbuilder-nvidia-l4", "count": 1, "config_drive": false}, "ppc64el": {"series": "focal", "count": 1}, "s390x": {"series": "focal", "count": 1}} %}
+{%-   set vbuilders_bos02 = {"arm64": {"series": "focal", "count": 1, "config_drive": false}, "ppc64el": {"series": "focal", "count": 1}, "s390x": {"series": "focal", "count": 1}} %}
+{%-   set vbuilders_bos03 = {"amd64": {"series": "focal", "count": 4}, "arm64": {"series": "focal", "count": 1, "config_drive": false, "flavor": "vbuilder-arm64"}, "riscv64": {"series": "jammy", "count": 1, "config_drive": false}} %}
+{%-   set vbuilders_lcy02 = {"amd64": {"series": "focal", "count": 4}} %}
+{%-   set vbuilder_prefix = "qastaging-" %}
+{%- endif %}
+
+{%- macro vbuilder_hostnames(prefix, count) %}
+{%-   set hostname_sep = joiner(", ") -%}
+[{% for i in range(1, count + 1) -%}
+{{ hostname_sep() }}"{{ prefix }}-{{ "%03d" % i }}"
+{%- endfor %}]
+{%- endmacro %}
+
+{%- macro vbuilders(region, arches) %}
+{%-   set arch_sep = joiner(", ") -%}
+[{% for arch, properties in arches|dictsort -%}
+{{ arch_sep() }}{"image_name_prefix": "{{ name_prefix }}{{ properties.get('arch_suffix', '') }}/ubuntu-{{ properties['series'] }}-daily-{{ properties.get('arch_base', arch) }}-", "instance_flavor": "{{ properties.get('flavor', 'vbuilder') }}", "hostnames": {{ vbuilder_hostnames("%s%s-%s" % (vbuilder_prefix, region, arch), properties['count']) }}, "config_drive": {{ properties.get('config_drive', True)|tojson }}}
+{%- endfor %}]
+{%- endmacro -%}
+
+series: "{{ series }}"
+applications:
+  rabbitmq-server:
+    charm: cs:rabbitmq-server
+    constraints: "cores=2 mem=8G root-disk=20G {{ extra_constraints }}"
+    num_units: 2
+  glance-simplestreams-sync-bos01-amd64:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 3, item_filters: ["release~({{ gss_series }})", "arch~(x86_64|amd64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "http://keystone.infra.bos01.scalingstack:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos01
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos01-arm64:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: 'hypervisor_type=kvm hw_firmware_type=uefi'
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 2, item_filters: ["release~({{ gss_series }})", "arch~(arm64|aarch64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "http://keystone.infra.bos01.scalingstack:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos01
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos01-arm64-gpu:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}:gpu"
+      custom_properties: 'hypervisor_type=kvm hw_firmware_type=uefi'
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 2, item_filters: ["release~({{ gss_series }})", "arch~(arm64|aarch64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}-gpu/"
+      openstack-auth-url: "http://keystone.infra.bos01.scalingstack:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos01
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos01-ppc64el:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: 'hypervisor_type=kvm'
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 2, item_filters: ["release~({{ gss_series }})", "arch~(ppc64el)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "http://keystone.infra.bos01.scalingstack:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos01
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos01-s390x:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: 'hypervisor_type=kvm'
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 3, item_filters: ["release~({{ gss_series }})", "arch~(s390x)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "http://keystone.infra.bos01.scalingstack:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos01
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos02-arm64:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: 'hypervisor_type=kvm hw_firmware_type=uefi'
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 2, item_filters: ["release~({{ gss_series }})", "arch~(arm64|aarch64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "http://keystone.infra.bos02.scalingstack:5000/v2.0"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos02
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos02-ppc64el:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: 'hypervisor_type=kvm'
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 2, item_filters: ["release~({{ gss_series }})", "arch~(ppc64el)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "http://keystone.infra.bos02.scalingstack:5000/v2.0"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos02
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos02-s390x:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: 'hypervisor_type=kvm'
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 3, item_filters: ["release~({{ gss_series }})", "arch~(s390x)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "http://keystone.infra.bos02.scalingstack:5000/v2.0"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      region: scalingstack-bos02
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos03-amd64:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 3, item_filters: ["release~({{ gss_series }})", "arch~(x86_64|amd64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "https://keystone.ps6.canonical.com:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name_bos03 }}"
+      openstack-username: "{{ openstack_username_bos03 }}"
+      region: scalingstack-bos03
+      use_swift: false
+      visibility: private
+  glance-simplestreams-sync-bos03-arm64:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: "hypervisor_type=kvm hw_firmware_type=uefi"
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 3, item_filters: ["release~({{ gss_series }})", "arch~(arm64|aarch64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "https://keystone.ps6.canonical.com:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name_bos03 }}"
+      openstack-username: "{{ openstack_username_bos03 }}"
+      region: scalingstack-bos03
+      use_swift: false
+      visibility: private
+{%- if stage_name in ("production", "qastaging") %}
+  glance-simplestreams-sync-bos03-riscv64:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      custom_properties: "hw_emulation_architecture=riscv64 hw_machine_type=virt kernel_id={{ kernel_id_bos03_riscv64 }}"
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 3, item_filters: ["release~({{ gss_series }})", "arch~(riscv64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "https://keystone.ps6.canonical.com:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name_bos03 }}"
+      openstack-username: "{{ openstack_username_bos03 }}"
+      region: scalingstack-bos03
+      use_swift: false
+      visibility: private
+{%- endif %}
+  glance-simplestreams-sync-lcy02-amd64:
+    charm: {{ charm_dir }}/glance-simplestreams-sync
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      content_id_template: "{{ content_id_template }}"
+      mirror_list: |-
+        [{url: "http://cloud-images.ubuntu.com/daily/", name_prefix: "ubuntu:released", path: "streams/v1/index.sjson", max: 3, item_filters: ["release~({{ gss_series }})", "arch~(x86_64|amd64)", "ftype~(disk1.img|disk.img)"]}]
+      name_prefix: "{{ name_prefix }}/"
+      openstack-auth-url: "https://keystone.ps5.canonical.com:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name_lcy02 }}"
+      openstack-username: "{{ openstack_username_lcy02 }}"
+      region: scalingstack-lcy02
+      use_swift: false
+      visibility: private
+  launchpad-buildd-image-modifier-amd64:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+  launchpad-buildd-image-modifier-bos01-amd64:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      extra-keys: "-----BEGIN PGP PUBLIC KEY BLOCK-----\nVersion: GnuPG v2\n\nmI0ESUm55wEEALrxow0PCnGeCAebH9g5+wtZBfXZdx2vZts+XsTTHxDRsMNgMC9b\n0klCgbydvkmF9WCphCjQ61Wp/Bh0C7DSXVCpA/xs55QB5VCUceIMZCbMTPq1h7Ht\ncA1f+o6+OCPUntErG6eGize6kGhdjBNPOT+q4BSIL69rPuwfM9ZyAYcBABEBAAG0\nJkxhdW5jaHBhZCBQUEEgZm9yIExhdW5jaHBhZCBEZXZlbG9wZXJziLYEEwECACAF\nAklJuecCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAtH/tsClF0rxsQA/0Q\nw0Yk+xIA1xibyf+UCF9/4fXzdo/tr76qxPRyFiv0uLbFOmW6t26jzpWBHocCHcCU\n57l7rlcEzIHFMcS9Ol6MughP4lhywf9ceeqg2SD6AXjZ0iFarwkueTcHwff5j0lG\nIzzCUVTYJ+m79f/r0dfctL2DwnX7JnT/41mEuR1qbokBHAQQAQIABgUCTB7s7wAK\nCRDFXO8hUqH8T94pCACxl/Gdo82N01H82HvNBa8zQFixNQIwNJN/VxH3WfRvissW\nOMTJnTnNOQErxUhqHrasvZf3djNoHeKRNToTTBaGiEwoySmEK05i4Toq74jWAOs6\nflD2S8natWbobK5V+B2pXZl5g/4Ay21C3H1sZlUxDCcOH9Jh8/0feAZHoSQ/V1Xa\nrEPb+TGdV0hP3Yp7+nIT91sYkj566kA8fjoxJrY/EvXGn98bhYMbMNbtS1Z0WeGp\nzG2hiL6wLSLBxz4Ae9MShOMwNyC1zmr/d1wlF0Efx1N9HaRtRq2s/zqH+ebB7Sr+\nV+SquObb0qr4eAjtslN5BxWROhf+wZM6WJO0Z6nBiQEcBBABAgAGBQJTHvsiAAoJ\nEIngjfAzAr5Z8y4H/jltxz5OwHIDoiXsyWnpjO1SZUV6I6evKpSD7huYtd7MwFZC\n0CgExsPPqLNQCUxITR+9jlqofi/QsTwP7Qq55VmIrKLrZ9KCK1qBnMa/YEXi6TeK\n65lnyN6lNOdzhcsBm3s1/U9ewWp1vsw4UAclmu6tI8GUko+e32K1QjMtIjeVejQl\nJCYDjuxfHhcFWyRo0TWu24F6VD3YxBHpne/M00yd2mLLpHdQrxw/vbvVhZkRDutQ\nemKRA81ZM2WZ1iqYOXtEs5VrD/PtU0nvSAowgeWBmcOwWn3Om+pVsnSoFo46CDvo\nC6YXOWMOMFIxfVhPWqlBkWQsnXFzgk/Xyo4vlTY=\n=Wq6H\n-----END PGP PUBLIC KEY BLOCK-----"
+      extra-packages: "nvidia-headless-525-grid nvidia-utils-525-grid"
+      # extra-sources must also be set in the secrets file, to add
+      # ppa:launchpad/ubuntu/buildd-gpu; that requires an authentication
+      # token.
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+  launchpad-buildd-image-modifier-bos01-arm64:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false compat_uts_machine=armv7l"
+      remote-modifiers: '{{ modifiers_bos01 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+  launchpad-buildd-image-modifier-bos01-arm64-gpu:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      extra-packages: "nvidia-headless-525-server nvidia-utils-525-server"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false compat_uts_machine=armv7l"
+      remote-modifiers: '{{ modifiers_bos01 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+  launchpad-buildd-image-modifier-bos01-ppc64el:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+      remote-modifiers: '{{ modifiers_bos01 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+  launchpad-buildd-image-modifier-bos01-s390x:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+      remote-modifiers: '{{ modifiers_bos01 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+  launchpad-buildd-image-modifier-bos02-arm64:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false compat_uts_machine=armv7l"
+      remote-modifiers: '{{ modifiers_bos02 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+  launchpad-buildd-image-modifier-bos02-ppc64el:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+      remote-modifiers: '{{ modifiers_bos02 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+  launchpad-buildd-image-modifier-bos02-s390x:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+      remote-modifiers: '{{ modifiers_bos02 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+  launchpad-buildd-image-modifier-bos03-amd64:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+  launchpad-buildd-image-modifier-bos03-arm64:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false compat_uts_machine=armv7l"
+      remote-modifiers: '{{ modifiers_bos03 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+{%- if stage_name in ("production", "qastaging") %}
+  launchpad-buildd-image-modifier-bos03-riscv64:
+    charm: {{ charm_dir }}/launchpad-buildd-image-modifier
+    options:
+      clamav-database-url: "{{ clamav_database_url }}"
+      launchpad-buildd-repository: "{{ launchpad_buildd_repository }}"
+      linux-command-line-extra: "systemd.unified_cgroup_hierarchy=false"
+      # XXX cjwatson 2023-11-15: jammy defaults to 5.0/stable, which has
+      # some race conditions affecting Launchpad builds.  Change this to a
+      # more stable channel (e.g. 5.20/stable) once one exists that contains
+      # https://github.com/canonical/lxd/pull/12530.
+      lxd-channel: "latest/candidate"
+      remote-modifiers: '{{ modifiers_bos03 }}'
+      remote-modifier-private-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder
+      remote-modifier-public-key: include-base64://{{ local_dir }}/id_rsa.imagebuilder.pub
+      sbuild-stalled-package-timeout: 1500
+{%- endif %}
+  vbuilder-manage-bos01:
+    charm: {{ charm_dir }}/vbuilder-manage
+    constraints: "cores=2 mem=8G root-disk=20G {{ extra_constraints }}"
+    expose: true
+    num_units: 1
+    options:
+      amqp-username: vbuilder-manage-bos01
+      amqp-vhost: vbuilder-manage-bos01
+      celery-worker-count: "32"
+      dns-update-host: "{{ dns_update_host_bos01 }}"
+      dns-update-key-name: "{{ dns_update_key_name }}"
+      domain: "{{ domain_bos01 }}"
+      instance-key-name: "{{ instance_key_name_bos01 }}"
+      instance-network: "{{ instance_network_bos01 }}"
+      instance-router: "{{ instance_router }}"
+      log-hosts-allow: "{{ log_hosts_allow }}"
+      lp-buildd-managers: "{{ lp_buildd_managers }}"
+      lp-environment: "{{ lp_environment }}"
+      lp-sshkey: "{{ lp_sshkey }}"
+      openstack-auth-url: http://keystone.infra.bos01.scalingstack:5000/v3
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      vbuilders: '{{ vbuilders("bos01", vbuilders_bos01) }}'
+  vbuilder-manage-bos02:
+    charm: {{ charm_dir }}/vbuilder-manage
+    constraints: "cores=2 mem=8G root-disk=20G {{ extra_constraints }}"
+    expose: true
+    num_units: 1
+    options:
+      amqp-username: vbuilder-manage-bos02
+      amqp-vhost: vbuilder-manage-bos02
+      celery-worker-count: "50"
+      dns-update-host: "{{ dns_update_host_bos02 }}"
+      dns-update-key-name: "{{ dns_update_key_name }}"
+      domain: "{{ domain_bos02 }}"
+      instance-key-name: "{{ instance_key_name_bos02 }}"
+      instance-network: "{{ instance_network_bos02 }}"
+      instance-router: "{{ instance_router }}"
+      log-hosts-allow: "{{ log_hosts_allow }}"
+      lp-buildd-managers: "{{ lp_buildd_managers }}"
+      lp-environment: "{{ lp_environment }}"
+      lp-sshkey: "{{ lp_sshkey }}"
+      openstack-auth-url: http://keystone.infra.bos02.scalingstack:5000/v2.0
+      openstack-tenant-name: "{{ openstack_tenant_name }}"
+      openstack-username: "{{ openstack_username }}"
+      vbuilders: '{{ vbuilders("bos02", vbuilders_bos02) }}'
+  vbuilder-manage-bos03:
+    charm: {{ charm_dir }}/vbuilder-manage
+    constraints: "cores=4 mem=8G root-disk=50G {{ extra_constraints }}"
+    expose: true
+    num_units: 1
+    options:
+      amqp-username: vbuilder-manage-bos03
+      amqp-vhost: vbuilder-manage-bos03
+      celery-worker-count: "50"
+      dns-update-host: "{{ dns_update_host_bos03 }}"
+      dns-update-key-name: "{{ dns_update_key_name }}"
+      domain: "{{ domain_bos03 }}"
+      instance-key-name: "{{ instance_key_name_bos03 }}"
+      instance-network: "{{ instance_network_bos03 }}"
+      instance-router: "{{ instance_router_bos03 }}"
+      log-hosts-allow: "{{ log_hosts_allow }}"
+      lp-buildd-managers: "{{ lp_buildd_managers }}"
+      lp-environment: "{{ lp_environment }}"
+      lp-sshkey: "{{ lp_sshkey }}"
+      openstack-auth-url: "https://keystone.ps6.canonical.com:5000/v3"
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name_bos03 }}"
+      openstack-username: "{{ openstack_username_bos03 }}"
+      vbuilders: '{{ vbuilders("bos03", vbuilders_bos03) }}'
+  vbuilder-manage-lcy02:
+    charm: {{ charm_dir }}/vbuilder-manage
+    constraints: "cores=4 mem=8G root-disk=50G {{ extra_constraints }}"
+    expose: true
+    num_units: 1
+    options:
+      amqp-username: vbuilder-manage-lcy02
+      amqp-vhost: vbuilder-manage-lcy02
+      celery-worker-count: "50"
+      dns-update-host: "{{ dns_update_host_lcy02 }}"
+      dns-update-key-name: "{{ dns_update_key_name }}"
+      domain: "{{ domain_lcy02 }}"
+      instance-key-name: "{{ instance_key_name_lcy02 }}"
+      instance-network: "{{ instance_network_lcy02 }}"
+      instance-router: "{{ instance_router_lcy02 }}"
+      log-hosts-allow: "{{ log_hosts_allow }}"
+      lp-buildd-managers: "{{ lp_buildd_managers }}"
+      lp-environment: "{{ lp_environment }}"
+      lp-sshkey: "{{ lp_sshkey }}"
+      openstack-auth-url: https://keystone.ps5.canonical.com:5000/v3
+      openstack-identity-api-version: "3"
+      openstack-tenant-name: "{{ openstack_tenant_name_lcy02 }}"
+      openstack-username: "{{ openstack_username_lcy02 }}"
+      vbuilders: '{{ vbuilders("lcy02", vbuilders_lcy02) }}'
+  clamav-database-mirror:
+    charm: ch:clamav-database-mirror
+    series: jammy
+    constraints: "{{ extra_constraints }}"
+    num_units: 1
+    options:
+      http-proxy: "http://squid.internal:3128/"
+  ntp:
+    charm: cs:ntp
+    options:
+      source: "ntp1.canonical.com ntp2.canonical.com ntp3.canonical.com ntp4.canonical.com"
+  # We need to configure telegraf explicitly here because we use custom
+  # plugins.  The subordinates spec will set up the relations.
+  telegraf:
+    charm: ch:telegraf
+    channel: candidate
+    expose: true
+    options:
+      extra_plugins: |-
+        [[inputs.procstat]]
+          pattern = "celery.*--app ppareset"
+        [[inputs.procstat]]
+          pattern = "/usr/local/sbin/ppa-reset"
+      install_method: snap
+relations:
+  - ["glance-simplestreams-sync-bos01-amd64:image-modifier", "launchpad-buildd-image-modifier-bos01-amd64:image-modifier"]
+  - ["glance-simplestreams-sync-bos01-arm64:image-modifier", "launchpad-buildd-image-modifier-bos01-arm64:image-modifier"]
+  - ["glance-simplestreams-sync-bos01-arm64-gpu:image-modifier", "launchpad-buildd-image-modifier-bos01-arm64-gpu:image-modifier"]
+  - ["glance-simplestreams-sync-bos01-ppc64el:image-modifier", "launchpad-buildd-image-modifier-bos01-ppc64el:image-modifier"]
+  - ["glance-simplestreams-sync-bos01-s390x:image-modifier", "launchpad-buildd-image-modifier-bos01-s390x:image-modifier"]
+  - ["glance-simplestreams-sync-bos02-arm64:image-modifier", "launchpad-buildd-image-modifier-bos02-arm64:image-modifier"]
+  - ["glance-simplestreams-sync-bos02-ppc64el:image-modifier", "launchpad-buildd-image-modifier-bos02-ppc64el:image-modifier"]
+  - ["glance-simplestreams-sync-bos02-s390x:image-modifier", "launchpad-buildd-image-modifier-bos02-s390x:image-modifier"]
+  - ["glance-simplestreams-sync-bos03-amd64:image-modifier", "launchpad-buildd-image-modifier-bos03-amd64:image-modifier"]
+  - ["glance-simplestreams-sync-bos03-arm64:image-modifier", "launchpad-buildd-image-modifier-bos03-arm64:image-modifier"]
+{%- if stage_name in ("production", "qastaging") %}
+  - ["glance-simplestreams-sync-bos03-riscv64:image-modifier", "launchpad-buildd-image-modifier-bos03-riscv64:image-modifier"]
+{%- endif %}
+  - ["glance-simplestreams-sync-lcy02-amd64:image-modifier", "launchpad-buildd-image-modifier-amd64:image-modifier"]
+  - ["vbuilder-manage-bos01:amqp", "rabbitmq-server:amqp"]
+  - ["vbuilder-manage-bos02:amqp", "rabbitmq-server:amqp"]
+  - ["vbuilder-manage-bos03:amqp", "rabbitmq-server:amqp"]
+  - ["vbuilder-manage-lcy02:amqp", "rabbitmq-server:amqp"]
+  - ["rabbitmq-server", "ntp"]
+  - ["glance-simplestreams-sync-bos01-amd64", "ntp"]
+  - ["glance-simplestreams-sync-bos01-arm64", "ntp"]
+  - ["glance-simplestreams-sync-bos01-arm64-gpu", "ntp"]
+  - ["glance-simplestreams-sync-bos01-ppc64el", "ntp"]
+  - ["glance-simplestreams-sync-bos01-s390x", "ntp"]
+  - ["glance-simplestreams-sync-bos02-arm64", "ntp"]
+  - ["glance-simplestreams-sync-bos02-ppc64el", "ntp"]
+  - ["glance-simplestreams-sync-bos02-s390x", "ntp"]
+  - ["glance-simplestreams-sync-bos03-amd64", "ntp"]
+  - ["glance-simplestreams-sync-bos03-arm64", "ntp"]
+{%- if stage_name in ("production", "qastaging") %}
+  - ["glance-simplestreams-sync-bos03-riscv64", "ntp"]
+{%- endif %}
+  - ["glance-simplestreams-sync-lcy02-amd64", "ntp"]
+  - ["vbuilder-manage-bos01", "ntp"]
+  - ["vbuilder-manage-bos02", "ntp"]
+  - ["vbuilder-manage-bos03", "ntp"]
+  - ["vbuilder-manage-lcy02", "ntp"]
+  - ["clamav-database-mirror", "ntp"]
diff --git a/vbuilder/collect b/vbuilder/collect
new file mode 100644
index 0000000..2deb9b8
--- /dev/null
+++ b/vbuilder/collect
@@ -0,0 +1,4 @@
+rabbitmq-server				cs:rabbitmq-server
+glance-simplestreams-sync		git+lp:~launchpad/charm-glance-simplestreams-sync;revno=scalingstack
+launchpad-buildd-image-modifier		git+lp:charm-launchpad-buildd-image-modifier
+vbuilder-manage				git+lp:launchpad-vbuilder-manage
diff --git a/vbuilder/configs/custom-secgroups-production.yaml b/vbuilder/configs/custom-secgroups-production.yaml
new file mode 100644
index 0000000..95a7565
--- /dev/null
+++ b/vbuilder/configs/custom-secgroups-production.yaml
@@ -0,0 +1,10 @@
+applications:
+    clamav-database-mirror:
+        type: neutron
+        rules:
+            - clamav-database-mirror
+rules:
+    clamav-database-mirror:
+        # Public HTTP.  (Firewalls restrict this to builders, but there's
+        # nothing secret here.)
+        - {"protocol": "tcp", "family": "IPv4", "port": 80, "cidr": "0.0.0.0/0"}
diff --git a/vbuilder/configs/custom-secgroups-staging.yaml b/vbuilder/configs/custom-secgroups-staging.yaml
new file mode 100644
index 0000000..95a7565
--- /dev/null
+++ b/vbuilder/configs/custom-secgroups-staging.yaml
@@ -0,0 +1,10 @@
+applications:
+    clamav-database-mirror:
+        type: neutron
+        rules:
+            - clamav-database-mirror
+rules:
+    clamav-database-mirror:
+        # Public HTTP.  (Firewalls restrict this to builders, but there's
+        # nothing secret here.)
+        - {"protocol": "tcp", "family": "IPv4", "port": 80, "cidr": "0.0.0.0/0"}
diff --git a/vbuilder/expand-bundle b/vbuilder/expand-bundle
new file mode 100755
index 0000000..bbbe52e
--- /dev/null
+++ b/vbuilder/expand-bundle
@@ -0,0 +1,36 @@
+#! /usr/bin/python3
+# The vbuilder bundle is heavily parameterized in order to keep its length
+# under control and reduce repetition.  This provides an easy way to expand
+# the bundle to see the effect of changes.
+
+from argparse import ArgumentParser
+import sys
+
+from jinja2 import (
+    Environment,
+    FileSystemLoader,
+    )
+import yaml
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("stage_name", help="Mojo stage name")
+    args = parser.parse_args()
+
+    template_env = Environment(loader=FileSystemLoader("."))
+    template = template_env.get_template("bundle.yaml")
+    rendered = template.render({
+        "charm_dir": "fake-charm-dir",
+        "stage_name": args.stage_name,
+        })
+    try:
+        yaml.safe_load(rendered)
+    except Exception as e:
+        print(f"Cannot parse rendered template: {e}", file=sys.stderr)
+        sys.exit(1)
+    print(rendered)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/vbuilder/manifest b/vbuilder/manifest
new file mode 100644
index 0000000..8e13254
--- /dev/null
+++ b/vbuilder/manifest
@@ -0,0 +1,5 @@
+collect
+bundle config=bundle.yaml local=secrets
+script config=upgrade-charms
+include config=manifest-secgroups
+juju-check-wait
diff --git a/vbuilder/manifest-rebuild-images b/vbuilder/manifest-rebuild-images
new file mode 100644
index 0000000..dbb5cb1
--- /dev/null
+++ b/vbuilder/manifest-rebuild-images
@@ -0,0 +1 @@
+script config=rebuild-images
diff --git a/vbuilder/manifest-secgroups b/vbuilder/manifest-secgroups
new file mode 100644
index 0000000..8c438ee
--- /dev/null
+++ b/vbuilder/manifest-secgroups
@@ -0,0 +1 @@
+script config=utils/custom-secgroups.py SKIP_STAGES=devel
diff --git a/vbuilder/rebuild-images b/vbuilder/rebuild-images
new file mode 100755
index 0000000..b4f8348
--- /dev/null
+++ b/vbuilder/rebuild-images
@@ -0,0 +1,84 @@
+#! /usr/bin/python3
+
+import os
+
+from utils import utils
+
+
+name_prefix_by_stage = {
+    "qastaging": "launchpad-buildd-qastaging",
+    "staging": "launchpad-buildd-staging",
+    "production": "launchpad-buildd",
+    }
+
+targets_by_stage = {
+    "qastaging": [
+        ("bos01", "amd64", "focal"),
+        ("bos01", "arm64", "focal"),
+        ("bos01", "arm64-gpu", "focal"),
+        ("bos01", "ppc64el", "focal"),
+        ("bos01", "s390x", "focal"),
+        ("bos02", "arm64", "focal"),
+        ("bos02", "ppc64el", "focal"),
+        ("bos02", "s390x", "focal"),
+        ("bos03", "amd64", "focal"),
+        ("bos03", "arm64", "focal"),
+        ("bos03", "riscv64", "jammy"),
+        ("lcy02", "amd64", "focal"),
+        ],
+    "staging": [
+        ("bos01", "amd64", "jammy"),
+        ("bos01", "arm64", "jammy"),
+        ("bos01", "arm64-gpu", "jammy"),
+        ("bos01", "ppc64el", "jammy"),
+        ("bos01", "s390x", "jammy"),
+        ("bos02", "arm64", "jammy"),
+        ("bos02", "ppc64el", "jammy"),
+        ("bos02", "s390x", "jammy"),
+        ("bos03", "amd64", "jammy"),
+        ("lcy02", "amd64", "jammy"),
+        ],
+    "production": [
+        ("bos01", "amd64", "focal"),
+        ("bos01", "arm64", "focal"),
+        ("bos01", "ppc64el", "focal"),
+        ("bos01", "s390x", "focal"),
+        ("bos02", "arm64", "focal"),
+        ("bos02", "ppc64el", "focal"),
+        ("bos02", "s390x", "focal"),
+        ("bos03", "amd64", "focal"),
+        ("bos03", "arm64", "focal"),
+        ("bos03", "riscv64", "jammy"),
+        ("lcy02", "amd64", "focal"),
+        ],
+    }
+
+
+def get_leader_unit(juju_services, application):
+    """Get the current leader unit for an application.
+
+    "juju run-action application/leader" sometimes reports "ERROR could not
+    determine leader for ..." even though a leader exists.  Work around this.
+    """
+    return next(
+        name for name, status in juju_services[application]["units"].items()
+        if status.get("leader", False))
+
+
+def main():
+    stage = os.path.basename(os.environ["MOJO_STAGE"])
+    name_prefix = name_prefix_by_stage[stage]
+    targets = targets_by_stage[stage]
+    juju_services = utils.juju_services()
+    for region, arch, series in targets:
+        application = f"glance-simplestreams-sync-{region}-{arch}"
+        unit = get_leader_unit(juju_services, application)
+        rebuild_cmd = [
+            "juju", "ssh", unit, "sudo", "/usr/local/bin/rebuild-latest-image",
+            f"{name_prefix}/ubuntu-{series}-daily-{arch}-",
+            ]
+        utils.run(None, rebuild_cmd)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/vbuilder/upgrade-charms b/vbuilder/upgrade-charms
new file mode 100755
index 0000000..dcf96c6
--- /dev/null
+++ b/vbuilder/upgrade-charms
@@ -0,0 +1,93 @@
+#! /usr/bin/python3
+
+import utils.check_version  # noqa: F401
+
+import os.path
+import subprocess
+
+
+targets_by_stage = {
+    "qastaging": [
+        ("bos01", "amd64"),
+        ("bos01", "arm64"),
+        ("bos01", "arm64-gpu"),
+        ("bos01", "ppc64el"),
+        ("bos01", "s390x"),
+        ("bos02", "arm64"),
+        ("bos02", "ppc64el"),
+        ("bos02", "s390x"),
+        ("bos03", "amd64"),
+        ("bos03", "arm64"),
+        ("bos03", "riscv64"),
+        ("lcy02", "amd64"),
+    ],
+    "staging": [
+        ("bos01", "amd64"),
+        ("bos01", "arm64"),
+        ("bos01", "arm64-gpu"),
+        ("bos01", "ppc64el"),
+        ("bos01", "s390x"),
+        ("bos02", "arm64"),
+        ("bos02", "ppc64el"),
+        ("bos02", "s390x"),
+        ("bos03", "amd64"),
+        ("lcy02", "amd64"),
+    ],
+    "production": [
+        ("bos01", "amd64"),
+        ("bos01", "arm64"),
+        ("bos01", "ppc64el"),
+        ("bos01", "s390x"),
+        ("bos02", "arm64"),
+        ("bos02", "ppc64el"),
+        ("bos02", "s390x"),
+        ("bos03", "amd64"),
+        ("bos03", "arm64"),
+        ("bos03", "riscv64"),
+        ("lcy02", "amd64"),
+    ],
+}
+
+
+upgraded = set()
+
+
+def upgrade_charm(app_name, charm_name):
+    if app_name in upgraded:
+        return
+    subprocess.check_call(
+        [
+            "juju",
+            "upgrade-charm",
+            app_name,
+            "--path=%s"
+            % os.path.join(
+                os.environ["MOJO_REPO_DIR"],
+                os.environ["MOJO_SERIES"],
+                charm_name,
+            ),
+        ]
+    )
+    upgraded.add(app_name)
+
+
+def main():
+    stage = os.path.basename(os.environ["MOJO_STAGE"])
+    targets = targets_by_stage[stage]
+    for region, arch in targets:
+        upgrade_charm(
+            f"glance-simplestreams-sync-{region}-{arch}",
+            "glance-simplestreams-sync",
+        )
+        # lcy02-amd64 uses an anomalous name for this charm; we won't follow
+        # this pattern for future regions on amd64.
+        if (region, arch) == ("lcy02", "amd64"):
+            lbim_app_name = "launchpad-buildd-image-modifier-amd64"
+        else:
+            lbim_app_name = f"launchpad-buildd-image-modifier-{region}-{arch}"
+        upgrade_charm(lbim_app_name, "launchpad-buildd-image-modifier")
+        upgrade_charm(f"vbuilder-manage-{region}", "vbuilder-manage")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/vbuilder/utils b/vbuilder/utils
new file mode 120000
index 0000000..468ba70
--- /dev/null
+++ b/vbuilder/utils
@@ -0,0 +1 @@
+../utils
\ No newline at end of file