Merge ~smigiel-dariusz/cloud-upgrade-planner:pre_integration into cloud-upgrade-planner:master

[Dariusz Smigiel]

Migration of bootstack-tools to cloud-upgrade-planner

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[James Troup (elmo)]

I'll merge this, but there are somethings that need fixed in followups please:

 1) assumption of /home/ubuntu
 2) reference to Canonical Wiki

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision bf1c391b152991cfca9aa6311e471e48302c7f1d

diff --git a/cloud_upgrade_planner/cli.py b/cloud_upgrade_planner/cli.py
 old mode 100755
 new mode 100644
diff --git a/cloud_upgrade_planner/common.py b/cloud_upgrade_planner/common.py
 new file mode 100644
index 0000000..a31016e
--- /dev/null
+++ b/cloud_upgrade_planner/common.py
@@ -0,0 +1,76 @@
+import re
+import sys
+import yaml
+
+
+def find_units_for_app(app, model_apps):
+    return list(model_apps[app]['units'].keys())
+
+
+def find_leader_for_app(app, model_apps):
+    for unit in model_apps[app]['units'].keys():
+        if 'leader' in model_apps[app]['units'][unit]:
+            return unit
+
+
+def find_charms_apps_in_model(charms, model_apps, svc2charm):
+    apps = []
+    for charm in charms:
+        apps_of_charm = [app for app in model_apps.keys() if svc2charm[app] == charm]
+        apps.extend(apps_of_charm)
+    return apps
+
+
+def get_model_apps(status_file):
+    try:
+        with open(status_file) as myjsfy:
+            return yaml.safe_load(myjsfy)["applications"]
+    except IOError:
+        print("Please run a fresh copy of juju status --format yaml > jsfy")
+        sys.exit(1)
+
+
+def find_apps_from_charm(charm, model_apps):
+    # Match local: or cs: + ~user/charm, series/charm, or charm
+    regex = re.compile("(local:|cs:)(~*.*/)*{}-\d+".format(charm))
+    return [app for app in model_apps.keys()
+            if re.match(regex, model_apps[app]['charm']) is not None]
+
+
+def find_machine_from_unit(app, unit, model_apps):
+    return model_apps[app]['units'][unit]['machine']
+
+
+def render_app_to_charm_dict(model_apps):
+    app2charm = {}
+    # Match local: or cs: + ~user/charm, series/charm, or charm
+    # regex groups are:
+    #   0: whole match
+    #   1: store|local
+    #   2: ~namespace or series
+    #   3: charm name
+    regex = re.compile(r"(local:|cs:)(~*.*/|)(.*)-\d+")
+    for app in model_apps.keys():
+        app2charm[app] = re.match(regex, model_apps[app]['charm']).group(3)
+    return app2charm
+
+
+def available_upgrade(app, model_apps):
+    if 'can-upgrade-to' in model_apps[app]:
+        return True
+    print("      # No charm upgrade available for {}. "
+          "Running: {}".format(app, model_apps[app]["charm"]))
+    return False
+
+
+def find_hacluster_for_unit(app, unit, model_apps):
+    if 'subordinates' not in model_apps[app]['units'][unit]:
+        return None
+    subordinate_units = model_apps[app]['units'][unit]['subordinates'].keys()
+    regex = re.compile("(local:|cs:)(~*.*/)*{}-\d+".format('hacluster'))
+    for subunit in subordinate_units:
+        subunit_app = subunit.split("/")[0]
+        if re.match(regex, model_apps[subunit_app]['charm']) is not None:
+          return subunit
+    return None
+
diff --git a/cloud_upgrade_planner/openstack_managed_upgrade.py b/cloud_upgrade_planner/openstack_managed_upgrade.py
 new file mode 100644
index 0000000..4b4960e
--- /dev/null
+++ b/cloud_upgrade_planner/openstack_managed_upgrade.py
@@ -0,0 +1,278 @@
+#!/usr/bin/python3
+
+# requres file in current directory named "jsfy" created with
+# juju status --format yaml > jsfy
+#  (version of juju must support leadership in yaml output)
+
+# Assumptions
+# we assume you're deploying either cs:charmname or cs:~namespace/charmname
+# or local:<series>/<charm> versions of charms, and does not support
+# differently-named charms.
+
+# set this to the destination openstack-origin of your choice
+# TODO:
+#       include optional unit pauses for pure clean HA
+#       include evacuating and monitoring each nova compute node
+#       include neutron-gateway HA router migrations if pausing
+#       inclue gnocchi/aodh/octavia/placement installs
+
+import argparse
+import re
+import sys
+
+from cloud_upgrade_planner import common
+
+# The phases of the update.  After any of these phases, updates can be paused
+# Each phase is recommended to take a day, with phases 3 and 4 being combined
+PHASE_APPS = {0: ["openstack-service-checks",
+                  "percona-cluster",
+                  "rabbitmq-server"],
+              1: ["keystone",
+                  "ceph-mon"],
+              2: ["ceph-fs",
+                  "ceph-radosgw",
+                  "swift-proxy",
+                  "swift-storage",
+                  "cinder",
+                  "cinder-ceph",
+                  "cinder-backup",
+                  "barbican",
+                  "glance",
+                  "aodh",
+                  "gnocchi",
+                  "ceilometer",
+                  "designate",
+                  "designate-bind",
+                  "neutron-api",
+                  "nova-cloud-controller",
+                  "heat",
+                  "manila",
+                  "manila-generic",
+                  "openstack-dashboard",
+                  "placement",
+                  "octavia",
+                  "easyrsa",
+                  "ovn-central"],
+              3: ["neutron-gateway",
+                  "nova-compute",
+                  "neutron-openvswitch",
+                  "ovn-chassis",
+                  "ceph-osd"]}
+
+ACTION_MANAGED_PHASES = [1, 3]
+DEFERRED_WAIT_PHASES = [2]
+
+# These charms use source: instead of openstack-origin: config flags
+ORIGIN_VAR_IS_SOURCE = ["ceph-osd",
+                        "ceph-mon",
+                        "ceph-radosgw",
+                        "percona-cluster",
+                        "rabbitmq-server"]
+
+# These charms do not have openstack-origin/source config flags
+CHARM_UPGRADES_ONLY = ["neutron-openvswitch",
+                       "ovn-chassis",
+                       "memcached",
+                       "mongodb",
+                       "graylog",
+                       "prometheus",
+                       "openstack-service-checks",
+                       "cinder-ceph",
+                       "cinder-backup",
+                       "easyrsa",
+                       "designate-bind"]
+
+# Enable deferred restarts on these charms as they are upgraded
+DEFERRED_RESTART_CHARMS = [
+    "neutron-gateway",
+    "neutron-openvswitch",
+    "ovn-central",
+    "ovn-chassis",
+    "ovn-dedicated-chassis",
+    "rabbitmq-server",
+]
+
+# TODO: When upgrading to these versions, add step to install these charms
+# between Phase 1 and Phase 2
+NEW_CHARMS_FOR_VERSIONS = {"train": ["placement"],
+                           "queens": ["octavia"],
+                           "ocata": ["aodh", "gnocchi"]}
+
+# This can be used both for required --switch changes for 19.04+ or can be
+# populated with specific version URLs
+# e.g. cinder-backup is no longer at cs:cinder-backup and if upgrading must be
+#      switched to cs:~openstack-charmers/cinder-backup
+# Anything not called out with these paths will be switched to cs:<charmname>
+#       if currently running cs:~*/<charmname> or local:<charmname>
+CHARMSTORE_TILDE_PATHS = {
+    "cinder-backup": "cs:~openstack-charmers/cinder-backup",
+    "easyrsa": "cs:~containers/easyrsa",
+    "policy-routing": "cs:~canonical-bootstack/policy-routing",
+    "infra-node": "cs:~canonical-bootstack/infra-node",
+}
+
+JUJU_WORKING_STATES = "blocked|waiting|maint|error|hook|lost|executing"
+WATCH_COMMAND = "watch \"juju status|egrep '{}'\"".format(JUJU_WORKING_STATES)
+
+PRE_UPGRADE_STEPS = """
+This playbook assumes all applications are running an identical openstack-origin/source of the prior version openstack.
+You can check 'juju config keystone openstack-origin' and 'juju config ceph-osd source' to verify the first and last apps are at the same level.
+
+### IMPORTANT ###
+Be sure to regenerate this playbook each time you complete a section to ensure that the leaders in the plan match the current active leaders
+
+Please read the wiki for openstack upgrades for known issues and read the upstream charm guide release notes and issues.
+https://wiki.canonical.com/CDO/IS/Bootstack/Playbooks/OpenstackReleaseUpgradeActionManaged
+https://docs.openstack.org/project-deploy-guide/charm-deployment-guide/latest/upgrade-issues.html
+
+There may be several things that need to be updated before changing to a new version of openstack, such as
+setting keystone tokens to fernet before upgrade to Rocky,
+ensuring that the ceph minimum_osd_version is set for the current version of ceph before upgrade,
+before upgrading to train, ensure that you've installed octavia and have migrated loadbalancers, as lbaasv2 is deprecated.
+
+See the referenced docs for these steps.
+
+After any upgrades, please check status of services via nagios/thruk to ensure you're not exhibiting new alerts.
+
+This upgrade guide does not cover router or VM migrations to avoid data-plane outages during package upgrades.
+Please be sure to plan accordingly based on customer request.
+
+All charms should be at the latest version before upgrading openstack.
+"""
+
+POST_UPGRADE_STEPS = """
+
+# POST UPGRADE ACTIONS
+Perform these steps after the cloud has completed upgrades
+
+Validate Nagios is clean and any disabled nova-compute services have been re-enabled.
+
+Update ceph minimum_osd_version per docs:
+https://docs.openstack.org/project-deploy-guide/charm-deployment-guide/latest/upgrade-issues.html#ceph-option-require-osd-release
+"""
+
+
+def plan_charm_upgrade(app, model_apps, svc2charm):
+    switch_args, config_filename = "", ""
+    charm = svc2charm[app]
+    current_charm = model_apps[app]['charm']
+    # strip version number (eg: cs:~foo/my-charm-123 becomes cs:~foo/my-charm)
+    current_charm_path = current_charm.rsplit("-", 1)[0]
+    target_tilde = CHARMSTORE_TILDE_PATHS.get(charm, None)
+
+    if "local:" in current_charm_path and not target_tilde:
+        # Provide warning to user that local charm was found and needs sanity check
+        print("""
+          WARNING!!! Application {app} has local charm path {current_charm}
+          Suggesting switch to latest promulgated cs: version.
+          Please ensure updates in local charm are in upstream charm before
+          running upgrade-charm on this application.
+        """.format(app=app, current_charm=current_charm))
+        switch_args += " --switch cs:{}".format(charm)
+    elif target_tilde and current_charm_path != target_tilde:
+        switch_args += " --switch {}".format(target_tilde)
+    elif "cs:~" in current_charm_path or "/" in current_charm_path:
+        print("""
+          WARNING!!! Application {app} from unrecognized path {current_charm}
+          Suggesting switch to latest promulgated cs: version.
+          Please ensure updates in the above branched charm are in the latest
+          promulgated charm before running upgrade-charm on this application.
+        """.format(app=app, current_charm=current_charm))
+        switch_args += " --switch {}".format("cs:{}".format(charm))
+
+    if charm in DEFERRED_RESTART_CHARMS:
+        config_filename = "deferred_restart_config_{}.yaml".format(app)
+        # Format of config file per 'juju deploy --help' doc
+        # application-name:
+        #   option1: value1
+        #   option2: value2
+        with open(config_filename, "w") as conf:
+            print("{}:\n  enable-auto-restarts: False".format(app), file=conf)
+
+    if switch_args or common.available_upgrade(app, model_apps):
+        config_args = ' --config {}'.format(config_filename) if config_filename else ""
+        print("      juju upgrade-charm {}{}{}".format(app, switch_args, config_args))
+        print("      {}\n\n".format(WATCH_COMMAND))
+    elif config_filename:
+        config_args = ' --file {}'.format(config_filename) if config_filename else ""
+        print("      juju config {}{}".format(app, config_args))
+        print("      {}\n\n".format(WATCH_COMMAND))
+
+
+def plan_action_managed_phase_app(app, model_apps, release):
+    leader = common.find_leader_for_app(app, model_apps)
+    units = common.find_units_for_app(app, model_apps)
+    units.remove(leader)
+    unit_list = " ".join(units)
+    upgradeorder = [leader]
+    upgradeorder.extend(units)
+    print("      juju config {} "
+          "action-managed-upgrade=true "
+          "openstack-origin={}".format(app, release))
+    print("      {}".format(WATCH_COMMAND))
+    for unit in upgradeorder:
+        print("      juju run-action --wait {} "
+              "openstack-upgrade".format(unit))
+
+
+def parse_args(argv):
+    """Process CLI arguments."""
+    parser = argparse.ArgumentParser(
+        prog="openstack_managed_upgrade.py",
+        description=(
+            "this program uses the output from juju status --format yaml"
+            "and creates a playbook of juju upgrade actions to upgrade the"
+            "version of Openstack to the target version"
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument("-f", "--status-file", dest="status_file", type=str,
+                        default="jsfy",
+                        help="file containing 'juju status --format yaml' output")
+    parser.add_argument("-c", "--charms-only", dest="charms_only",
+                        action="store_true",
+                        help="Only plan charm upgrades")
+    parser.add_argument("release", type=str,
+                        help="Set release to UCA target for upgrade such as: 'cloud:bionic-ussuri'")
+    return parser.parse_args(argv)
+
+
+def main(argv=None):
+    args = parse_args(argv)
+    model_apps = common.get_model_apps(args.status_file)
+    svc2charm = common.render_app_to_charm_dict(model_apps)
+    print(PRE_UPGRADE_STEPS)
+    for phase in sorted(PHASE_APPS):
+        print("Phase {}\n========".format(phase))
+        this_phase_apps = common.find_charms_apps_in_model(PHASE_APPS[phase],
+                                                    model_apps, svc2charm)
+        for app in this_phase_apps:
+            # skip landscape upgrades
+            if "landscape" in app:
+                continue
+            print("  App: {}".format(app))
+            print("    Charm: {}".format(svc2charm[app]))
+            print("    Commands:")
+            plan_charm_upgrade(app, model_apps, svc2charm)
+            if args.charms_only:
+                continue
+            if svc2charm[app] in CHARM_UPGRADES_ONLY:
+                # no openstack repo upgrade for these charms
+                continue
+            if svc2charm[app] in ORIGIN_VAR_IS_SOURCE:
+                # not possible to be action-managed-upgrade
+                print("      juju config {} source={}".format(app, args.release))
+            elif phase in ACTION_MANAGED_PHASES:
+                plan_action_managed_phase_app(app, model_apps, args.release)
+            else:
+                print("      juju config {} openstack-origin={}"
+                      " action-managed-upgrade=false".format(app, args.release))
+            if phase not in DEFERRED_WAIT_PHASES:
+                print("      {}\n".format(WATCH_COMMAND))
+        if not args.charms_only and phase in DEFERRED_WAIT_PHASES:
+            print("\n  {}\n".format(WATCH_COMMAND))
+
+    print(POST_UPGRADE_STEPS)
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/cloud_upgrade_planner/plan.py b/cloud_upgrade_planner/plan.py
 old mode 100755
 new mode 100644
diff --git a/cloud_upgrade_planner/ubuntu_series_upgrade_for_openstack.py b/cloud_upgrade_planner/ubuntu_series_upgrade_for_openstack.py
 new file mode 100644
index 0000000..8db72c1
--- /dev/null
+++ b/cloud_upgrade_planner/ubuntu_series_upgrade_for_openstack.py
@@ -0,0 +1,328 @@
+#!/usr/bin/python3
+
+# requres file in current directory named "jsfy" created with
+# juju status --format yaml > jsfy
+#  (version of juju must support leadership in yaml output)
+
+# Assumptions
+# we assume you're deploying either cs:charmname or cs:~namespace/charmname
+# or local:<series>/<charm> versions of charms, and does not support
+# differently-named charms.
+
+# set this to the destination openstack-origin of your choice
+# TODO: add in parseargs
+#       include stepped version callouts
+#       include evacuating and monitoring each nova compute node
+#       include neutron-gateway HA router migrations if pausing
+
+import re
+import sys
+import yaml
+
+from cloud_upgrade_planner import common
+
+# Set this to your target platform ### See TODOs
+UCA = "distro"
+series = "bionic"
+svc2charm = {}
+
+# The phases of the update.  After any of these phases, updates can be paused
+# Each phase is recommended to take a day, with phases 3 and 4 being combined
+PHASE_APPS = {0: ["percona-cluster",
+                  "rabbitmq-server",
+                  "ceph-mon",
+                  "ovn-central",
+                  "etcd",
+                  "vault"],
+              1: ["keystone",
+                  "ceph-fs",
+                  "ceph-radosgw",
+                  "swift-proxy",
+                  "swift-storage",
+                  "cinder",
+                  "barbican",
+                  "glance",
+                  "aodh",
+                  "gnocchi",
+                  "ceilometer",
+                  "designate",
+                  "designate-bind",
+                  "neutron-api",
+                  "nova-cloud-controller",
+                  "heat",
+                  "manila",
+                  "manila-generic",
+                  "openstack-dashboard",
+                  "placement",
+                  "octavia",
+                  "memcached",
+                  "easyrsa"],
+              2: ["nova-compute",
+                  "neutron-gateway",
+                  "ceph-osd"],
+}
+
+# These charms use source: instead of openstack-origin: config flags
+ORIGIN_VAR_IS_SOURCE = ["ceph-osd",
+                        "ceph-mon",
+                        "ceph-fs",
+                        "ceph-radosgw",
+                        "percona-cluster",
+                        "rabbitmq-server"]
+
+# These charms only need the OS upgraded, no UCA tracking
+NO_ORIGIN = ["designate-bind", "vault"]
+
+# These charms do not have openstack-origin/source config flags
+CHARM_UPGRADES_ONLY = ["neutron-openvswitch",
+                       "memcached",
+                       "mongodb",
+                       "graylog",
+                       "prometheus",
+                       "openstack-service-checks",
+                       "cinder-ceph",
+                       "cinder-backup"]
+
+NO_PAUSE_APPS = ["ceph-mon",
+                 "ceph-osd",
+                 "neutron-gateway",
+                 "nova-compute"]
+
+POST_APP_UPGRADE_ACTIONS_FOR_LEADERS = {
+    "percona-cluster": "complete-cluster-series-upgrade",
+    "rabbitmq-server": "complete-cluster-series-upgrade",
+    "ceilometer": "ceilometer-upgrade",
+}
+
+POST_APP_UPGRADE_MANUAL_STEPS = {
+    "vault": "You will need to unseal the vault after each unit reboots",
+}
+# This can be used both for required --switch changes for 19.04+ or can be
+# populated with specific version URLs
+# e.g. cinder-backup is no longer at cs:cinder-backup and if upgrading must be
+#      switched to cs:~openstack-charmers/cinder-backup
+CHARMSTORE_PATHS = {"cinder-backup": "cs:~openstack-charmers/cinder-backup"}
+JUJU_WORKING_STATES = "blocked|waiting|maint|error|hook|lost|executing"
+WATCH_COMMAND = "watch \"juju status|egrep '{}'\"".format(JUJU_WORKING_STATES)
+
+PRE_UPGRADE_STEPS = """
+This playbook assumes your current openstack revision is at the highest level of the current ubuntu series:
+  trusty-mitaka
+  xenial-queens
+  bionic-ussuri
+You can check 'juju config keystone openstack-origin' and 'juju config ceph-osd source' to verify.
+
+### IMPORTANT ###
+Be sure to regenerate this playbook each time you complete a section to ensure that the leaders in the plan match the current active leaders
+
+Perform these steps to prepare your environment/directory for the remaining playbook.
+
+# Login to MAAS WebUI and add the new series to MAAS under the Images tab.
+
+# Create "local" file for setting your apt preferences - Required for DRU to be non-interactive
+cat > local << EOF
+DPkg::options { "--force-confdef"; "--force-confnew"; }
+EOF
+
+# update the default-series of the model to the new series
+juju model-config default-series=bionic   # or focal
+
+"""
+
+POST_UPGRADE_STEPS = """
+
+# POST UPGRADE ACTIONS
+Perform these steps after the cloud has completed upgrades
+
+# Remove the "local" file added to apt configs for DRU, as it's not correct for typical upgrades
+juju run --all 'sudo rm /etc/apt/apt.conf.d/local'
+
+N.B. Do not upgrade Nagios from xenial to bionic until all cloud nodes have completed upgrade to bionic.
+
+Upgrade all remaining machines (typically LMA stack) not previously captured:
+# find the units with:
+juju machines|grep <previous release|xenial|bionic|etc>
+
+# Check each of those machines that they're not already upgraded by listing /etc/os-release file
+juju run --machine <comma,separated,list,of,machines,left,at,old,version> 'hostname; grep VERSION_CODENAME /etc/os-release'
+
+# perform steps similiar to above for the remaining machines/applications.
+"""
+
+
+def plan_parallel(model_apps, parallel_apps, machines_planned):
+    cluster_pause_tasks = []
+    pause_tasks = []
+    leader_prepare_tasks = []
+    non_leader_prepare_tasks = []
+    upgrade_machines = []
+    post_upgrade_tasks = []
+    leader_complete_tasks = []
+    non_leader_complete_tasks = []
+    cleanup_tasks = []
+    for app in parallel_apps:
+        print("      juju set-series {} {}".format(app, series))
+        leader = common.find_leader_for_app(app, model_apps)
+        units = common.find_units_for_app(app, model_apps)
+        units.remove(leader)
+
+        post_leader_action = POST_APP_UPGRADE_ACTIONS_FOR_LEADERS.get(svc2charm[app], None)
+        manual_steps = POST_APP_UPGRADE_MANUAL_STEPS.get(svc2charm[app], None)
+        if post_leader_action:
+            cleanup_tasks.append("      juju run-action --wait {} {}".format(leader, post_leader_action))
+        if manual_steps:
+            cleanup_tasks.append("      ## Manual step: {}".format(manual_steps))
+
+        for unit in units:
+            unit_machine = common.find_machine_from_unit(app, unit, model_apps)
+            if unit_machine in machines_planned:
+                # CONSIDER: Note that we decided to skip dist-upgrade since it was already done?
+                continue
+            hacluster_unit = common.find_hacluster_for_unit(app, unit, model_apps)
+            if hacluster_unit is not None:
+                cluster_pause_tasks.append("      juju run-action --wait {} pause".format(hacluster_unit))
+            if svc2charm[app] not in NO_PAUSE_APPS:
+                pause_tasks.append("      juju run-action --wait {} pause".format(unit))
+            non_leader_prepare_tasks.append("      juju upgrade-series {} prepare {} -y  # {}".format(unit_machine, series, unit))
+            non_leader_complete_tasks.append("      juju upgrade-series {} complete  # {}".format(unit_machine, unit))
+            upgrade_machines.append(unit_machine)
+            machines_planned.append(unit_machine)
+
+        leader_machine = common.find_machine_from_unit(app, leader, model_apps)
+        if leader_machine in machines_planned:
+            continue
+        # upgrade leader unit's machine
+        leader_prepare_tasks.append("      juju upgrade-series {} prepare {} -y  # {}".format(leader_machine, series, leader))
+        upgrade_machines.append(leader_machine)
+        leader_complete_tasks.append("      echo 'Check status of workload for {}'".format(leader))
+        if svc2charm[app] in ORIGIN_VAR_IS_SOURCE:
+            post_upgrade_tasks.append("      juju config {} source={}".format(app, UCA))
+        elif svc2charm[app] not in NO_ORIGIN:
+            post_upgrade_tasks.append("      juju config {} openstack-origin={}".format(app,UCA))
+        leader_complete_tasks.append("      juju upgrade-series {} complete  # {}".format(leader_machine, leader))
+        machines_planned.append(leader_machine)
+    print("    # Pause non-leaders")
+    print("\n".join(cluster_pause_tasks))
+    print("\n".join(pause_tasks))
+    print("    # Prepare leaders - These can be done in parallel before non-leaders")
+    print("\n".join(leader_prepare_tasks))
+    print("    # Prepare non-leaders - These can be done in parallel after leaders")
+    print("\n".join(non_leader_prepare_tasks))
+    print("    for i in {}; do juju scp local $i: ; done".format(" ".join(upgrade_machines)))
+    machine_spec = ",".join(upgrade_machines)
+    print("    # Update all machines to latest patches")
+    print(" ### IF YOU HAVE A VERSION OF JUJU THAT SHOWS 'action terminated' on some units when running against multiple units, upgrade juju agents before this step")
+    print("      juju run --machine {} --timeout=20m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'".format(machine_spec))
+    print("    # Perform do-release-upgrade")
+    print("      juju run --machine {} --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'".format(machine_spec))
+    print(" #### Check above and reboot and re-run d-r-u for any machines that did not upgrade due to needing reboot")
+    print(" #### Check above and reboot and re-run d-r-u for any machines that did not upgrade due to needing reboot")
+    print(" #### Check above and reboot and re-run d-r-u for any machines that did not upgrade due to needing reboot")
+    print("    # Reboot all API machines into {}".format(series))
+    print("      juju run --machine {} --timeout=10m 'sudo init 6'".format(machine_spec))
+    print("    # Post Upgrade tasks")
+    print("\n".join(post_upgrade_tasks))
+    print("    # Complete leaders - These can all be done in parallel before non-leaders")
+    print("\n".join(leader_complete_tasks))
+    print("    # Complete non-leaders - These can all be done in parallel after leaders complete")
+    print("\n".join(non_leader_complete_tasks))
+    print("    # Post App Upgrade cleanup tasks")
+    print("\n".join(cleanup_tasks))
+    print("\n")
+
+
+def main():
+    model_apps = common.get_model_apps("jsfy")
+    machines_planned = []
+
+    print(PRE_UPGRADE_STEPS)
+
+    for phase in sorted(PHASE_APPS):
+        print("Phase {}\n========".format(phase))
+        this_phase_apps = []
+        for app in PHASE_APPS[phase]:
+            if 'landscape' in app:
+                continue
+            services = common.find_apps_from_charm(app, model_apps)
+            this_phase_apps.extend(services)
+            for service in services:
+                svc2charm[service] = app
+        if phase == 1:
+            plan_parallel(model_apps, this_phase_apps, machines_planned)
+            continue
+        for app in this_phase_apps:
+            plan_sequential(model_apps, app, machines_planned)
+
+    print(POST_UPGRADE_STEPS)
+
+
+def plan_sequential(model_apps, app, machines_planned):
+    post_leader_action = POST_APP_UPGRADE_ACTIONS_FOR_LEADERS.get(svc2charm[app], None)
+    manual_steps = POST_APP_UPGRADE_MANUAL_STEPS.get(svc2charm[app], None)
+    findlandscape = re.compile("landscape.*")
+    if re.match(findlandscape, app) is not None:
+        return
+    print("  App: {}".format(app))
+    leader = common.find_leader_for_app(app, model_apps)
+    units = common.find_units_for_app(app, model_apps)
+    units.remove(leader)
+    unit_list = " ".join(units)
+    print("    Units {} {}".format(leader, unit_list))
+    print("      {}".format(WATCH_COMMAND))
+    print("      juju set-series {} {}".format(app, series))
+    for unit in units:
+        hacluster_unit = common.find_hacluster_for_unit(app, unit, model_apps)
+        if hacluster_unit is not None:
+            print("      juju run-action --wait {} pause".format(hacluster_unit))
+    for unit in units:
+        if svc2charm[app] not in NO_PAUSE_APPS:
+            print("      juju run-action --wait {} pause".format(unit))
+    # gate for machine already done
+    leader_machine = common.find_machine_from_unit(app, leader, model_apps)
+    if leader_machine in machines_planned:
+        print("      # Skipping dist-upgrade as already done prior for {}".format(leader_machine))
+    else:
+        # upgrade leader unit's machine
+        print("      juju upgrade-series {} prepare {} -y".format(leader_machine, series))
+        print("      juju scp local {}:".format(leader_machine))
+        print(
+            "      juju run --machine {} --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'".format(
+                leader_machine))
+        print("      juju run --machine {} --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'".format(leader_machine))
+        print("      juju run --machine {} --timeout=5m 'sudo init 6'".format(leader_machine))
+        print("      sleep 120; jw; echo 'Check status of workload for {}'".format(leader))
+    if svc2charm[app] in ORIGIN_VAR_IS_SOURCE:
+        print("      juju config {} source={}".format(app, UCA))
+    elif svc2charm[app] not in NO_ORIGIN:
+        print("      juju config {} openstack-origin={}".format(app, UCA))
+    print("      sleep 120; jw")
+    if leader_machine in machines_planned:
+        print("      # Skipping dist-upgrade as already done prior for {}".format(leader_machine))
+    else:
+        print("      juju upgrade-series {} complete".format(leader_machine))
+        machines_planned.append(leader_machine)
+    for unit in units:
+        non_leader_machine = common.find_machine_from_unit(app, unit, model_apps)
+        if non_leader_machine in machines_planned:
+            print("      # Skipping dist-upgrade as already done prior for {}".format(non_leader_machine))
+        else:
+            print("      juju upgrade-series {} prepare {} -y".format(non_leader_machine, series))
+            print("      juju scp local {}:".format(non_leader_machine))
+            print(
+                "      juju run --machine {} --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'".format(
+                    non_leader_machine))
+            print("      juju run --machine {} --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'".format(
+                non_leader_machine))
+            print("      juju run  --machine {} --timeout=5m 'sudo init 6'".format(non_leader_machine))
+            print("      sleep 120; jw; echo 'Check status of workload for {}'".format(unit))
+            print("      juju upgrade-series {} complete".format(non_leader_machine))
+            machines_planned.append(non_leader_machine)
+    if post_leader_action:
+        print("      juju run-action --wait {} {}".format(leader, post_leader_action))
+    if manual_steps:
+        print("      ## Manual step: {}".format(manual_steps))
+    print('\n')
+
+
+if __name__ == "__main__":
+    main()
diff --git a/setup.py b/setup.py
index 8c310d4..a9ecfa7 100644
--- a/setup.py
+++ b/setup.py
@@ -48,6 +48,8 @@ setuptools.setup(
         "console_scripts": [
             "cloud-upgrade-planner=cloud_upgrade_planner.cli:main",
             "update-charm-revisions=cloud_upgrade_planner.cli:refresh",
+            "ubuntu-series-upgrade-for-openstack=cloud_upgrade_planner.ubuntu_series_upgrade_for_openstack:main",
+            "openstack-managed-upgrade=cloud_upgrade_planner.openstack_managed_upgrade:main",
         ]
     },
     setup_requires=["setuptools_scm"],
diff --git a/tests/test_cli.py b/tests/test_cli.py
 old mode 100755
 new mode 100644
diff --git a/tests/test_ubuntu_series_upgrade_for_openstack.py b/tests/test_ubuntu_series_upgrade_for_openstack.py
 new file mode 100644
index 0000000..8d80437
--- /dev/null
+++ b/tests/test_ubuntu_series_upgrade_for_openstack.py
@@ -0,0 +1,728 @@
+import pytest
+from unittest.mock import patch, mock_open, call
+import yaml
+
+from cloud_upgrade_planner import common as c
+from cloud_upgrade_planner import openstack_managed_upgrade as o
+from cloud_upgrade_planner import ubuntu_series_upgrade_for_openstack as u
+
+
+JUJU_FILE_MOCK = """
+applications:
+  aodh-mysql-router:
+    can-upgrade-to: cs:mysql-router-11
+    charm: cs:mysql-router-6
+  easyrsa:
+    can-upgrade-to: cs:~containers/easyrsa-408
+    charm: cs:~containers/easyrsa-345
+    units:
+      easyrsa/0:
+        leader: true
+        machine: 7/lxd/3
+  hacluster-neutron:
+    can-upgrade-to: cs:hacluster-78
+    charm: cs:hacluster-74
+  hacluster-nova:
+    can-upgrade-to: cs:hacluster-78
+    charm: cs:hacluster-74
+  hacluster-vault:
+    can-upgrade-to: cs:hacluster-78
+    charm: cs:hacluster-74
+  landscape-client:
+    charm: cs:landscape-client-35
+  juju-lint:
+    charm: local:focal/juju-lint-1
+    units:
+      juju-lint/0:
+        leader: true
+        machine: 8/lxd/15
+  memcached:
+    charm: cs:memcached-34
+    units:
+      memcached/1:
+        leader: true
+        machine: 5/lxd/4
+  mysql-innodb-cluster:
+    can-upgrade-to: cs:mysql-innodb-cluster-11
+    charm: cs:mysql-innodb-cluster-5
+    units:
+      mysql-innodb-cluster/0:
+        machine: 3/lxd/8
+        subordinates:
+          landscape-client/65: true
+      mysql-innodb-cluster/1:
+        leader: true
+        machine: 4/lxd/8
+        subordinates:
+          landscape-client/66: true
+      mysql-innodb-cluster/2:
+        machine: 5/lxd/8
+        subordinates:
+          landscape-client/67: true
+  neutron-api:
+    can-upgrade-to: cs:neutron-api-299
+    charm: cs:neutron-api-292
+    units:
+      neutron-api/0:
+        machine: 6/lxd/5
+        subordinates:
+          hacluster-neutron/0: true
+          landscape-client/69: true
+          neutron-api-mysql-router/0: true
+      neutron-api/1:
+        machine: 7/lxd/6
+        subordinates:
+          hacluster-neutron/1: true
+          landscape-client/68: true
+          neutron-api-mysql-router/1: true
+      neutron-api/2:
+        leader: true
+        machine: 8/lxd/5
+        subordinates:
+          hacluster-neutron/2: true
+          landscape-client/70: true
+          neutron-api-mysql-router/2: true
+  nova-cloud-controller:
+    can-upgrade-to: cs:nova-cloud-controller-358
+    charm: cs:nova-cloud-controller-353
+    units:
+      nova-cloud-controller/0:
+        machine: 6/lxd/6
+        subordinates:
+          hacluster-nova/0: true
+          landscape-client/72: true
+          nova-cloud-controller-mysql-router/0: true
+      nova-cloud-controller/1:
+        machine: 7/lxd/7
+        subordinates:
+          hacluster-nova/2: true
+          landscape-client/73: true
+          nova-cloud-controller-mysql-router/2: true
+      nova-cloud-controller/2:
+        leader: true
+        machine: 8/lxd/6
+        subordinates:
+          hacluster-nova/1: true
+          landscape-client/71: true
+          nova-cloud-controller-mysql-router/1: true
+  nova-cloud-controller-mysql-router:
+    can-upgrade-to: cs:mysql-router-11
+    charm: cs:mysql-router-6
+  nova-compute-kvm:
+    can-upgrade-to: cs:nova-compute-334
+    charm: cs:nova-compute-325
+    units:
+      nova-compute-kvm/1:
+        machine: '10'
+        subordinates:
+          landscape-client/19: true
+      nova-compute-kvm/10:
+        machine: '19'
+        subordinates:
+          landscape-client/10: true
+      nova-compute-kvm/11:
+        machine: '20'
+        subordinates:
+          landscape-client/20: true
+      nova-compute-kvm/12:
+        machine: '21'
+        subordinates:
+          landscape-client/11: true
+      nova-compute-kvm/13:
+        machine: '22'
+        subordinates:
+          landscape-client/4: true
+      nova-compute-kvm/7:
+        leader: true
+        machine: '16'
+        subordinates:
+          landscape-client/7: true
+      nova-compute-kvm/9:
+        machine: '18'
+        subordinates:
+          landscape-client/16: true
+  rabbitmq-server:
+    can-upgrade-to: cs:~llama-charmers-next/rabbitmq-server-6
+    charm: cs:~llama-charmers-next/rabbitmq-server-5
+    units:
+      rabbitmq-server/4:
+        machine: 6/lxd/12
+      rabbitmq-server/5:
+        leader: true
+        machine: 7/lxd/12
+      rabbitmq-server/6:
+        machine: 8/lxd/11
+  vault:
+    can-upgrade-to: cs:vault-50
+    charm: cs:vault-44
+    units:
+      vault/0:
+        machine: '0'
+        subordinates:
+          hacluster-vault/1: true
+          landscape-client/24: true
+      vault/1:
+        leader: true
+        machine: '1'
+        subordinates:
+          hacluster-vault/2: true
+          landscape-client/23: true
+      vault/2:
+        machine: '2'
+        subordinates:
+          hacluster-vault/0: true
+          landscape-client/22: true
+"""
+JUJU_FILE_DICT = yaml.safe_load(JUJU_FILE_MOCK)["applications"]
+
+SVC2CHARM = {
+    "aodh-mysql-router": "mysql-router",
+    "easyrsa": "easyrsa",
+    "hacluster-neutron": "hacluster",
+    "hacluster-nova": "hacluster",
+    "hacluster-vault": "hacluster",
+    "juju-lint": "juju-lint",
+    "landscape-client": "landscape-client",
+    "memcached": "memcached",
+    "mysql-innodb-cluster": "mysql-innodb-cluster",
+    "neutron-api": "neutron-api",
+    "nova-cloud-controller": "nova-cloud-controller",
+    "nova-cloud-controller-mysql-router": "mysql-router",
+    "nova-compute-kvm": "nova-compute",
+    "rabbitmq-server": "rabbitmq-server",
+    "vault": "vault",
+}
+
+
+class TestCommon:
+    @pytest.mark.parametrize(
+        "app,expected",
+        [
+            ("memcached", "memcached/1"),
+            ("mysql-innodb-cluster", "mysql-innodb-cluster/1"),
+            ("neutron-api", "neutron-api/2"),
+        ],
+    )
+    def test_find_leader_for_app(self, app, expected):
+        output = c.find_leader_for_app(app, JUJU_FILE_DICT)
+        assert output == expected
+
+    def test_get_model_apps_missing_file(self):
+        with pytest.raises(SystemExit):
+            c.get_model_apps("")
+
+    @patch("builtins.open", new_callable=mock_open, read_data=JUJU_FILE_MOCK)
+    def test_get_model_apps_provided_file(self, juju_file):
+        output = c.get_model_apps("test")
+        assert output == JUJU_FILE_DICT
+
+    @pytest.mark.parametrize(
+        "charms,expected",
+        [
+            (
+                ["mysql-router", "nova-compute", "hacluster", "vault"],
+                [
+                    "aodh-mysql-router",
+                    "nova-cloud-controller-mysql-router",
+                    "nova-compute-kvm",
+                    "hacluster-neutron",
+                    "hacluster-nova",
+                    "hacluster-vault",
+                    "vault",
+                ],
+            ),
+            (["non-existing-charm"], []),
+        ],
+    )
+    def test_find_charms_apps_in_model(self, charms, expected):
+        output = c.find_charms_apps_in_model(charms, JUJU_FILE_DICT, SVC2CHARM)
+        assert output == expected
+
+    @pytest.mark.parametrize(
+        "app,expected",
+        [
+            (
+                "nova-compute-kvm",
+                [
+                    "nova-compute-kvm/1",
+                    "nova-compute-kvm/10",
+                    "nova-compute-kvm/11",
+                    "nova-compute-kvm/12",
+                    "nova-compute-kvm/13",
+                    "nova-compute-kvm/7",
+                    "nova-compute-kvm/9",
+                ],
+            ),
+            (
+                "mysql-innodb-cluster",
+                [
+                    "mysql-innodb-cluster/0",
+                    "mysql-innodb-cluster/1",
+                    "mysql-innodb-cluster/2",
+                ],
+            ),
+        ],
+    )
+    def test_find_units_for_app(self, app, expected):
+        output = c.find_units_for_app(app, JUJU_FILE_DICT)
+        assert output == expected
+
+    @pytest.mark.parametrize(
+        "charm,expected",
+        [
+            ("neutron-api", ["neutron-api"]),
+            ("nova-compute", ["nova-compute-kvm"]),
+            (
+                "mysql-router",
+                ["aodh-mysql-router", "nova-cloud-controller-mysql-router"],
+            ),
+        ],
+    )
+    def test_find_apps_from_charm(self, charm, expected):
+        output = c.find_apps_from_charm(charm, JUJU_FILE_DICT)
+        assert output == expected
+
+    def test_find_units_for_app_not_in_phase(self):
+        with pytest.raises(KeyError):
+            c.find_units_for_app("nova-cloud-controller-mysql-router", JUJU_FILE_DICT)
+
+    def test_find_units_for_app_missing(self):
+        with pytest.raises(KeyError):
+            c.find_units_for_app("appx", JUJU_FILE_DICT)
+
+    @pytest.mark.parametrize(
+        "app,unit,expected",
+        [
+            ("nova-compute-kvm", "nova-compute-kvm/9", "18"),
+            ("memcached", "memcached/1", "5/lxd/4"),
+        ],
+    )
+    def test_find_machine_from_unit(self, app, unit, expected):
+        output = c.find_machine_from_unit(app, unit, JUJU_FILE_DICT)
+        assert output == expected
+
+    def test_render_app_to_charm_dict(self):
+        output = c.render_app_to_charm_dict(JUJU_FILE_DICT)
+        assert output == SVC2CHARM
+
+    @pytest.mark.parametrize(
+        "app,expected",
+        [
+            ("aodh-mysql-router", True),
+            ("memcached", False),
+        ],
+    )
+    def test_available_upgrade(self, app, expected):
+        output = c.available_upgrade(app, JUJU_FILE_DICT)
+        assert output == expected
+
+    @pytest.mark.parametrize(
+        "app,unit,expected",
+        [
+            ("nova-cloud-controller", "nova-cloud-controller/1", "hacluster-nova/2"),
+            ("neutron-api", "neutron-api/2", "hacluster-neutron/2"),
+            ("nova-compute-kvm", "nova-compute-kvm/1", None),  # missing HA
+            ("memcached", "memcached/1", None),  # missing subordinates
+        ],
+    )
+    def test_find_hacluster_for_unit(self, app, unit, expected):
+        output = c.find_hacluster_for_unit(app, unit, JUJU_FILE_DICT)
+        assert output == expected
+
+
+class TestOpenstackManagedUpgrade:
+    @patch("builtins.print")
+    @patch(
+        "cloud_upgrade_planner.common.find_units_for_app",
+        return_value=["neutron-api/0", "neutron-api/1", "neutron-api/2"],
+    )
+    @patch(
+        "cloud_upgrade_planner.common.find_leader_for_app", return_value="neutron-api/2"
+    )
+    def test_plan_action_managed_phase_app(self, mock_leader, mock_units, mock_stdout):
+        o.plan_action_managed_phase_app("test-app", JUJU_FILE_DICT, "test-release")
+        calls = [
+            call(
+                "      juju config test-app action-managed-upgrade=true openstack-origin=test-release"
+            ),
+            call("      {}".format(o.WATCH_COMMAND)),
+            call("      juju run-action --wait neutron-api/2 openstack-upgrade"),
+            call("      juju run-action --wait neutron-api/0 openstack-upgrade"),
+            call("      juju run-action --wait neutron-api/1 openstack-upgrade"),
+        ]
+        assert mock_stdout.mock_calls == calls
+
+    @patch("builtins.print")
+    def test_plan_charm_upgrade_local_path(self, mock_stdout):
+        output = o.plan_charm_upgrade("juju-lint", JUJU_FILE_DICT, SVC2CHARM)
+        calls = [
+            call(
+                "\n          WARNING!!! Application juju-lint has local charm path local:focal/juju-lint-1\n          Suggesting switch to latest promulgated cs: version.\n          Please ensure updates in local charm are in upstream charm before\n          running upgrade-charm on this application.\n        "
+            ),
+            call("      juju upgrade-charm juju-lint --switch cs:juju-lint"),
+            call(
+                "      watch \"juju status|egrep 'blocked|waiting|maint|error|hook|lost|executing'\"\n\n"
+            ),
+        ]
+        assert mock_stdout.mock_calls == calls
+
+    @patch("builtins.print")
+    def test_plan_charm_upgrade_cs_path(self, mock_stdout):
+        output = o.plan_charm_upgrade("neutron-api", JUJU_FILE_DICT, SVC2CHARM)
+        calls = [
+            call("      juju upgrade-charm neutron-api"),
+            call(
+                "      watch \"juju status|egrep 'blocked|waiting|maint|error|hook|lost|executing'\"\n\n"
+            ),
+        ]
+        assert mock_stdout.mock_calls == calls
+
+    @patch("builtins.print")
+    def test_plan_charm_upgrade_tilde(self, mock_stdout):
+        output = o.plan_charm_upgrade("easyrsa", JUJU_FILE_DICT, SVC2CHARM)
+        calls = [
+            call(
+                "\n          WARNING!!! Application easyrsa from unrecognized path cs:~containers/easyrsa-345\n          Suggesting switch to latest promulgated cs: version.\n          Please ensure updates in the above branched charm are in the latest\n          promulgated charm before running upgrade-charm on this application.\n        "
+            ),
+            call("      juju upgrade-charm easyrsa --switch cs:easyrsa"),
+            call(
+                "      watch \"juju status|egrep 'blocked|waiting|maint|error|hook|lost|executing'\"\n\n"
+            ),
+        ]
+        assert mock_stdout.mock_calls == calls
+
+    @patch("builtins.print")
+    @patch("builtins.open", new_callable=mock_open, read_data=JUJU_FILE_MOCK)
+    def test_plan_charm_upgrade_deferred(self, defer_file, mock_stdout):
+        output = o.plan_charm_upgrade("rabbitmq-server", JUJU_FILE_DICT, SVC2CHARM)
+        calls = [
+            call(
+                "\n          WARNING!!! Application rabbitmq-server from unrecognized path cs:~llama-charmers-next/rabbitmq-server-5\n          Suggesting switch to latest promulgated cs: version.\n          Please ensure updates in the above branched charm are in the latest\n          promulgated charm before running upgrade-charm on this application.\n        "
+            ),
+            call("rabbitmq-server:\n  enable-auto-restarts: False", file=defer_file()),
+            call(
+                "      juju upgrade-charm rabbitmq-server --switch cs:rabbitmq-server --config deferred_restart_config_rabbitmq-server.yaml"
+            ),
+            call(
+                "      watch \"juju status|egrep 'blocked|waiting|maint|error|hook|lost|executing'\"\n\n"
+            ),
+        ]
+        assert mock_stdout.mock_calls == calls
+
+
+class TestUbuntuSeriesUpgrade:
+    def test_main_missing_juju_file(self):
+        with pytest.raises(SystemExit):
+            u.main()
+
+    @patch("cloud_upgrade_planner.ubuntu_series_upgrade_for_openstack.plan_sequential")
+    @patch("cloud_upgrade_planner.ubuntu_series_upgrade_for_openstack.plan_parallel")
+    @patch(
+        "cloud_upgrade_planner.common.find_apps_from_charm", return_value=["app"]
+    )
+    @patch("builtins.open", new_callable=mock_open, read_data=JUJU_FILE_MOCK)
+    def test_main_with_juju_file(
+        self, juju_file, mock_apps_from_charm, mock_parallel, mock_sequential
+    ):
+        u.main()
+
+        charm_calls = []
+        for phase in range(2):
+            for app in u.PHASE_APPS[phase]:
+                charm_calls.append(call(app, JUJU_FILE_DICT))
+        mock_apps_from_charm.assert_has_calls(charm_calls)
+
+        parallel_apps_no = len(u.PHASE_APPS[1])
+        mock_parallel.assert_called_once_with(
+            JUJU_FILE_DICT, ["app"] * parallel_apps_no, []
+        )
+
+        sequential_apps_no = len(u.PHASE_APPS[0]) + len(u.PHASE_APPS[2])
+        seq_calls = [call(JUJU_FILE_DICT, "app", []) for _ in range(sequential_apps_no)]
+        mock_sequential.assert_has_calls(seq_calls)
+
+    @patch("builtins.print")
+    @patch("builtins.open", new_callable=mock_open, read_data=JUJU_FILE_MOCK)
+    def test_main_output(self, juju_file, mock_stdout):
+        u.main()
+        calls = [
+            call(u.PRE_UPGRADE_STEPS),
+            call("Phase 0\n========"),
+            call("  App: rabbitmq-server"),
+            call("    Units rabbitmq-server/5 rabbitmq-server/4 rabbitmq-server/6"),
+            call(
+                "      watch \"juju status|egrep 'blocked|waiting|maint|error|hook|lost|executing'\""
+            ),
+            call("      juju set-series rabbitmq-server bionic"),
+            call("      juju run-action --wait rabbitmq-server/4 pause"),
+            call("      juju run-action --wait rabbitmq-server/6 pause"),
+            call("      juju upgrade-series 7/lxd/12 prepare bionic -y"),
+            call("      juju scp local 7/lxd/12:"),
+            call(
+                "      juju run --machine 7/lxd/12 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 7/lxd/12 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run --machine 7/lxd/12 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for rabbitmq-server/5'"
+            ),
+            call("      juju config rabbitmq-server source=distro"),
+            call("      sleep 120; jw"),
+            call("      juju upgrade-series 7/lxd/12 complete"),
+            call("      juju upgrade-series 6/lxd/12 prepare bionic -y"),
+            call("      juju scp local 6/lxd/12:"),
+            call(
+                "      juju run --machine 6/lxd/12 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 6/lxd/12 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 6/lxd/12 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for rabbitmq-server/4'"
+            ),
+            call("      juju upgrade-series 6/lxd/12 complete"),
+            call("      juju upgrade-series 8/lxd/11 prepare bionic -y"),
+            call("      juju scp local 8/lxd/11:"),
+            call(
+                "      juju run --machine 8/lxd/11 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 8/lxd/11 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 8/lxd/11 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for rabbitmq-server/6'"
+            ),
+            call("      juju upgrade-series 8/lxd/11 complete"),
+            call(
+                "      juju run-action --wait rabbitmq-server/5 complete-cluster-series-upgrade"
+            ),
+            call("\n"),
+            call("  App: vault"),
+            call("    Units vault/1 vault/0 vault/2"),
+            call(
+                "      watch \"juju status|egrep 'blocked|waiting|maint|error|hook|lost|executing'\""
+            ),
+            call("      juju set-series vault bionic"),
+            call("      juju run-action --wait hacluster-vault/1 pause"),
+            call("      juju run-action --wait hacluster-vault/0 pause"),
+            call("      juju run-action --wait vault/0 pause"),
+            call("      juju run-action --wait vault/2 pause"),
+            call("      juju upgrade-series 1 prepare bionic -y"),
+            call("      juju scp local 1:"),
+            call(
+                "      juju run --machine 1 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 1 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run --machine 1 --timeout=5m 'sudo init 6'"),
+            call("      sleep 120; jw; echo 'Check status of workload for vault/1'"),
+            call("      sleep 120; jw"),
+            call("      juju upgrade-series 1 complete"),
+            call("      juju upgrade-series 0 prepare bionic -y"),
+            call("      juju scp local 0:"),
+            call(
+                "      juju run --machine 0 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 0 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 0 --timeout=5m 'sudo init 6'"),
+            call("      sleep 120; jw; echo 'Check status of workload for vault/0'"),
+            call("      juju upgrade-series 0 complete"),
+            call("      juju upgrade-series 2 prepare bionic -y"),
+            call("      juju scp local 2:"),
+            call(
+                "      juju run --machine 2 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 2 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 2 --timeout=5m 'sudo init 6'"),
+            call("      sleep 120; jw; echo 'Check status of workload for vault/2'"),
+            call("      juju upgrade-series 2 complete"),
+            call(
+                "      ## Manual step: You will need to unseal the vault after each unit reboots"
+            ),
+            call("\n"),
+            call("Phase 1\n========"),
+            call("      juju set-series neutron-api bionic"),
+            call("      juju set-series nova-cloud-controller bionic"),
+            call("      juju set-series memcached bionic"),
+            call("      juju set-series easyrsa bionic"),
+            call("    # Pause non-leaders"),
+            call(
+                "      juju run-action --wait hacluster-neutron/0 pause\n      juju run-action --wait hacluster-neutron/1 pause\n      juju run-action --wait hacluster-nova/0 pause\n      juju run-action --wait hacluster-nova/2 pause"
+            ),
+            call(
+                "      juju run-action --wait neutron-api/0 pause\n      juju run-action --wait neutron-api/1 pause\n      juju run-action --wait nova-cloud-controller/0 pause\n      juju run-action --wait nova-cloud-controller/1 pause"
+            ),
+            call(
+                "    # Prepare leaders - These can be done in parallel before non-leaders"
+            ),
+            call(
+                "      juju upgrade-series 8/lxd/5 prepare bionic -y  # neutron-api/2\n      juju upgrade-series 8/lxd/6 prepare bionic -y  # nova-cloud-controller/2\n      juju upgrade-series 5/lxd/4 prepare bionic -y  # memcached/1\n      juju upgrade-series 7/lxd/3 prepare bionic -y  # easyrsa/0"
+            ),
+            call(
+                "    # Prepare non-leaders - These can be done in parallel after leaders"
+            ),
+            call(
+                "      juju upgrade-series 6/lxd/5 prepare bionic -y  # neutron-api/0\n      juju upgrade-series 7/lxd/6 prepare bionic -y  # neutron-api/1\n      juju upgrade-series 6/lxd/6 prepare bionic -y  # nova-cloud-controller/0\n      juju upgrade-series 7/lxd/7 prepare bionic -y  # nova-cloud-controller/1"
+            ),
+            call(
+                "    for i in 6/lxd/5 7/lxd/6 8/lxd/5 6/lxd/6 7/lxd/7 8/lxd/6 5/lxd/4 7/lxd/3; do juju scp local $i: ; done"
+            ),
+            call("    # Update all machines to latest patches"),
+            call(
+                " ### IF YOU HAVE A VERSION OF JUJU THAT SHOWS 'action terminated' on some units when running against multiple units, upgrade juju agents before this step"
+            ),
+            call(
+                "      juju run --machine 6/lxd/5,7/lxd/6,8/lxd/5,6/lxd/6,7/lxd/7,8/lxd/6,5/lxd/4,7/lxd/3 --timeout=20m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call("    # Perform do-release-upgrade"),
+            call(
+                "      juju run --machine 6/lxd/5,7/lxd/6,8/lxd/5,6/lxd/6,7/lxd/7,8/lxd/6,5/lxd/4,7/lxd/3 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call(
+                " #### Check above and reboot and re-run d-r-u for any machines that did not upgrade due to needing reboot"
+            ),
+            call(
+                " #### Check above and reboot and re-run d-r-u for any machines that did not upgrade due to needing reboot"
+            ),
+            call(
+                " #### Check above and reboot and re-run d-r-u for any machines that did not upgrade due to needing reboot"
+            ),
+            call("    # Reboot all API machines into bionic"),
+            call(
+                "      juju run --machine 6/lxd/5,7/lxd/6,8/lxd/5,6/lxd/6,7/lxd/7,8/lxd/6,5/lxd/4,7/lxd/3 --timeout=10m 'sudo init 6'"
+            ),
+            call("    # Post Upgrade tasks"),
+            call(
+                "      juju config neutron-api openstack-origin=distro\n      juju config nova-cloud-controller openstack-origin=distro\n      juju config memcached openstack-origin=distro\n      juju config easyrsa openstack-origin=distro"
+            ),
+            call(
+                "    # Complete leaders - These can all be done in parallel before non-leaders"
+            ),
+            call(
+                "      echo 'Check status of workload for neutron-api/2'\n      juju upgrade-series 8/lxd/5 complete  # neutron-api/2\n      echo 'Check status of workload for nova-cloud-controller/2'\n      juju upgrade-series 8/lxd/6 complete  # nova-cloud-controller/2\n      echo 'Check status of workload for memcached/1'\n      juju upgrade-series 5/lxd/4 complete  # memcached/1\n      echo 'Check status of workload for easyrsa/0'\n      juju upgrade-series 7/lxd/3 complete  # easyrsa/0"
+            ),
+            call(
+                "    # Complete non-leaders - These can all be done in parallel after leaders complete"
+            ),
+            call(
+                "      juju upgrade-series 6/lxd/5 complete  # neutron-api/0\n      juju upgrade-series 7/lxd/6 complete  # neutron-api/1\n      juju upgrade-series 6/lxd/6 complete  # nova-cloud-controller/0\n      juju upgrade-series 7/lxd/7 complete  # nova-cloud-controller/1"
+            ),
+            call("    # Post App Upgrade cleanup tasks"),
+            call(""),
+            call("\n"),
+            call("Phase 2\n========"),
+            call("  App: nova-compute-kvm"),
+            call(
+                "    Units nova-compute-kvm/7 nova-compute-kvm/1 nova-compute-kvm/10 nova-compute-kvm/11 nova-compute-kvm/12 nova-compute-kvm/13 nova-compute-kvm/9"
+            ),
+            call(
+                "      watch \"juju status|egrep 'blocked|waiting|maint|error|hook|lost|executing'\""
+            ),
+            call("      juju set-series nova-compute-kvm bionic"),
+            call("      juju upgrade-series 16 prepare bionic -y"),
+            call("      juju scp local 16:"),
+            call(
+                "      juju run --machine 16 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 16 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run --machine 16 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for nova-compute-kvm/7'"
+            ),
+            call("      juju config nova-compute-kvm openstack-origin=distro"),
+            call("      sleep 120; jw"),
+            call("      juju upgrade-series 16 complete"),
+            call("      juju upgrade-series 10 prepare bionic -y"),
+            call("      juju scp local 10:"),
+            call(
+                "      juju run --machine 10 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 10 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 10 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for nova-compute-kvm/1'"
+            ),
+            call("      juju upgrade-series 10 complete"),
+            call("      juju upgrade-series 19 prepare bionic -y"),
+            call("      juju scp local 19:"),
+            call(
+                "      juju run --machine 19 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 19 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 19 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for nova-compute-kvm/10'"
+            ),
+            call("      juju upgrade-series 19 complete"),
+            call("      juju upgrade-series 20 prepare bionic -y"),
+            call("      juju scp local 20:"),
+            call(
+                "      juju run --machine 20 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 20 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 20 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for nova-compute-kvm/11'"
+            ),
+            call("      juju upgrade-series 20 complete"),
+            call("      juju upgrade-series 21 prepare bionic -y"),
+            call("      juju scp local 21:"),
+            call(
+                "      juju run --machine 21 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 21 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 21 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for nova-compute-kvm/12'"
+            ),
+            call("      juju upgrade-series 21 complete"),
+            call("      juju upgrade-series 22 prepare bionic -y"),
+            call("      juju scp local 22:"),
+            call(
+                "      juju run --machine 22 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 22 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 22 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for nova-compute-kvm/13'"
+            ),
+            call("      juju upgrade-series 22 complete"),
+            call("      juju upgrade-series 18 prepare bionic -y"),
+            call("      juju scp local 18:"),
+            call(
+                "      juju run --machine 18 --timeout=60m 'sudo service cron stop; sudo service landscape-client stop; sudo cp /home/ubuntu/local /etc/apt/apt.conf.d/local; sudo apt-get update; sudo apt-get dist-upgrade -y; sudo apt-get autoremove -y; sudo rm /var/run/reboot-required /var/run/reboot-required.pkgs'"
+            ),
+            call(
+                "      juju run --machine 18 --timeout=60m 'sudo do-release-upgrade -f DistUpgradeViewNonInteractive'"
+            ),
+            call("      juju run  --machine 18 --timeout=5m 'sudo init 6'"),
+            call(
+                "      sleep 120; jw; echo 'Check status of workload for nova-compute-kvm/9'"
+            ),
+            call("      juju upgrade-series 18 complete"),
+            call("\n"),
+            call(u.POST_UPGRADE_STEPS),
+        ]
+
+        assert mock_stdout.mock_calls == calls