Merge ~cjwatson/launchpad-layers:payload-layer into launchpad-layers:main

[Colin Watson]

Move part of launchpad-base into a new launchpad-payload layer

Some charms - at least the prospective `launchpad-assets`, which publishes CSS/JS etc. assets for use by frontends - only really need an unpacked Launchpad payload, and don't need the rest of `launchpad-base`. Extract that part into a separate layer.

be89b3b... by Colin Watson on 2023-04-28

Add charms.launchpad.payload.get_service_config

This is useful in both the prospective `launchpad-assets` and
`launchpad-loggerhead` charms, so let's have it in common code for
charms that don't need database integration.

8a69d5d... by Colin Watson on 2023-04-28

Move LAZR config file handling to payload layer

It's useful for nearly any Launchpad deployment, and doesn't have
onerous dependencies.

b80ee57... by Colin Watson on 2023-04-28

Move crontab handling to payload layer

It's useful for nearly any Launchpad deployment, and doesn't have
onerous dependencies.

[Guruprasad (lgp171188)]

LGTM 👍🏼

review: Approve

[Clinton Fung (clinton-fung)]

Makes sense

review: Approve

[Colin Watson (cjwatson)]

Tweaked copyright years; landing.

diff --git a/launchpad-base/config.yaml b/launchpad-base/config.yaml
index 14feeca..b804392 100644
--- a/launchpad-base/config.yaml
+++ b/launchpad-base/config.yaml
@@ -43,10 +43,6 @@ options:
     type: string
     description: URL of file used to control whether cron scripts run.
     default: "file:cronscripts.ini"
-  cron_mailto:
-    type: string
-    description: Email address for output from cron jobs.
-    default: "[e-mail]"
   databases:
     type: string
     description: >
@@ -274,10 +270,3 @@ options:
     type: string
     description: Outbound webhook request proxy URL.
     default:
-  # layer-apt
-  install_sources:
-    default: |
-      - ppa:launchpad/ppa
-  install_keys:
-    default: |
-      - null  # PPA keys securely added via Launchpad.
diff --git a/launchpad-base/layer.yaml b/launchpad-base/layer.yaml
index 32c0987..5abc6ac 100644
--- a/launchpad-base/layer.yaml
+++ b/launchpad-base/layer.yaml
@@ -2,15 +2,5 @@ includes:
   - layer:basic
   - layer:ols-pg
   - interface:rabbitmq
-options:
-  apt:
-    packages:
-      - launchpad-dependencies
-  ols:
-    service_name: launchpad
-    config_filename: service.conf
-    user: launchpad
-    tarball_payload: true
-    symlink_switch_payload: true
-    python_bin: /usr/bin/python3
+  - layer:launchpad-payload
 repo: https://git.launchpad.net/launchpad-layers
diff --git a/launchpad-base/lib/charms/launchpad/base.py b/launchpad-base/lib/charms/launchpad/base.py
index 7768c9b..2d441de 100644
--- a/launchpad-base/lib/charms/launchpad/base.py
+++ b/launchpad-base/lib/charms/launchpad/base.py
@@ -11,15 +11,14 @@ from email.utils import parseaddr
 from urllib.parse import urlparse
 
 from charmhelpers.core import hookenv, host, templating
+from charms.launchpad.payload import config_file_path
+from charms.launchpad.payload import get_service_config as get_payload_config
+from charms.launchpad.payload import home_dir
 from charms.reactive import endpoint_from_flag
 from ols import base
 from psycopg2.extensions import make_dsn, parse_dsn
 
 
-def home_dir():
-    return os.path.join("/home", base.user())
-
-
 def oopses_dir():
     return os.path.join(base.base_dir(), "oopses")
 
@@ -51,7 +50,6 @@ def ensure_lp_directories():
     for dirpath in oopses_dir(), var_dir():
         host.mkdir(dirpath, group=base.user(), perms=0o775)
     host.mkdir(secrets_dir(), group=base.user(), perms=0o750)
-    host.mkdir(home_dir(), owner=base.user(), group=base.user(), perms=0o755)
 
 
 def _get_first_rabbitmq_hostname(rabbitmq):
@@ -64,17 +62,13 @@ def _get_first_rabbitmq_hostname(rabbitmq):
 
 
 def get_service_config():
-    config = dict(hookenv.config())
+    config = get_payload_config()
     config.update(
         {
-            "base_dir": base.base_dir(),
-            "code_dir": base.code_dir(),
-            "logs_dir": base.logs_dir(),
             "oopses_dir": oopses_dir(),
             # Used by some templates.
             "parseaddr": parseaddr,
             "secrets_dir": secrets_dir(),
-            "user": base.user(),
             "var_dir": var_dir(),
         }
     )
@@ -109,26 +103,6 @@ def get_service_config():
     return config
 
 
-def config_file_path(name, secret=False):
-    if secret:
-        config_dir = os.path.join(base.base_dir(), "secrets")
-    else:
-        config_dir = os.path.join(base.code_dir(), "production-configs")
-    return os.path.join(config_dir, name)
-
-
-def configure_lazr(config, template, name, secret=False):
-    hookenv.log("Writing service configuration.")
-    templating.render(
-        template,
-        config_file_path(name, secret=secret),
-        config,
-        owner="root",
-        group=base.user(),
-        perms=0o440 if secret else 0o444,
-    )
-
-
 def lazr_config_files():
     return [
         config_file_path("launchpad-base-lazr.conf"),
@@ -266,19 +240,3 @@ def strip_dsn_authentication(dsn):
     parsed_dsn = parse_dsn(dsn)
     parsed_dsn.pop("password", None)
     return make_dsn(**parsed_dsn)
-
-
-def configure_cron(config, template):
-    hookenv.log("Writing crontab.")
-    crontab_path = os.path.join(home_dir(), "crontab")
-    templating.render(
-        template,
-        crontab_path,
-        config,
-        owner=base.user(),
-        group=base.user(),
-        perms=0o600,
-    )
-    subprocess.run(
-        ["sudo", "-H", "-u", base.user(), "crontab", crontab_path], check=True
-    )
diff --git a/launchpad-base/reactive/launchpad-base.py b/launchpad-base/reactive/launchpad-base.py
index adc40d7..186e7df 100644
--- a/launchpad-base/reactive/launchpad-base.py
+++ b/launchpad-base/reactive/launchpad-base.py
@@ -1,18 +1,16 @@
 # Copyright 2022 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-import subprocess
-
 from charmhelpers.core import hookenv
 from charms.launchpad.base import (
     change_shell,
-    configure_lazr,
     configure_rsync,
     ensure_lp_directories,
     get_service_config,
     strip_dsn_authentication,
     update_pgpass,
 )
+from charms.launchpad.payload import configure_lazr
 from charms.reactive import (
     clear_flag,
     endpoint_from_flag,
@@ -27,18 +25,6 @@ from charms.reactive import (
 from ols import base, postgres
 
 
-# Monkey-patch layer:ols.
-def create_virtualenv(wheels_dir, codedir, python_exe):
-    subprocess.run(
-        ["make", "compile", f"PYTHON={python_exe}"],
-        cwd=codedir,
-        check=True,
-    )
-
-
-base.create_virtualenv = create_virtualenv
-
-
 @when("rabbitmq.connected")
 def prepare_rabbitmq():
     rabbitmq = endpoint_from_flag("rabbitmq.connected")
@@ -73,7 +59,11 @@ def rabbitmq_unavailable():
     clear_flag("launchpad.rabbitmq.available")
 
 
-@when("ols.configured", "db.master.available", "launchpad.rabbitmq.available")
+@when(
+    "launchpad.payload.configured",
+    "db.master.available",
+    "launchpad.rabbitmq.available",
+)
 @when_not("launchpad.base.configured")
 def configure():
     db = endpoint_from_flag("db.master.available")
@@ -113,30 +103,15 @@ def configure():
 
 @when("launchpad.base.configured")
 @when_not_all(
-    "ols.configured", "db.master.available", "launchpad.rabbitmq.available"
+    "launchpad.payload.configured",
+    "db.master.available",
+    "launchpad.rabbitmq.available",
 )
 def deconfigure():
     clear_flag("launchpad.base.configured")
     clear_flag("service.configured")
 
 
-@hook("upgrade-charm")
-def upgrade_charm():
-    # The ols layer takes care of removing the ols.service.installed,
-    # ols.configured, and service.configured states.  Remove
-    # launchpad.base.configured as well so that we have an opportunity to
-    # rewrite base configuration files.
-    clear_flag("launchpad.base.configured")
-
-
-@when("config.changed.build_label")
-def build_label_changed():
-    clear_flag("ols.service.installed")
-    clear_flag("ols.configured")
-    clear_flag("launchpad.base.configured")
-    clear_flag("service.configured")
-
-
 @when("config.changed")
 def config_changed():
     clear_flag("launchpad.base.configured")
diff --git a/launchpad-payload/config.yaml b/launchpad-payload/config.yaml
new file mode 100644
index 0000000..bdb5437
--- /dev/null
+++ b/launchpad-payload/config.yaml
@@ -0,0 +1,12 @@
+options:
+  cron_mailto:
+    type: string
+    description: Email address for output from cron jobs.
+    default: "[e-mail]"
+  # layer-apt
+  install_sources:
+    default: |
+      - ppa:launchpad/ppa
+  install_keys:
+    default: |
+      - null  # PPA keys securely added via Launchpad.
diff --git a/launchpad-base/icon.svg b/launchpad-payload/icon.svg
similarity index 100%
rename from launchpad-base/icon.svg
rename to launchpad-payload/icon.svg
diff --git a/launchpad-payload/layer.yaml b/launchpad-payload/layer.yaml
new file mode 100644
index 0000000..970a981
--- /dev/null
+++ b/launchpad-payload/layer.yaml
@@ -0,0 +1,17 @@
+includes:
+  - layer:basic
+  - layer:ols
+options:
+  apt:
+    packages:
+      - launchpad-dependencies
+  ols:
+    service_name: launchpad
+    config_filename: service.conf
+    user: launchpad
+    tarball_payload: true
+    symlink_switch_payload: true
+    python_bin: /usr/bin/python3
+  launchpad-payload:
+    build_target: compile
+repo: https://git.launchpad.net/launchpad-layers
diff --git a/launchpad-payload/lib/charms/launchpad/payload.py b/launchpad-payload/lib/charms/launchpad/payload.py
new file mode 100644
index 0000000..ba87457
--- /dev/null
+++ b/launchpad-payload/lib/charms/launchpad/payload.py
@@ -0,0 +1,66 @@
+# Copyright 2022-2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import os.path
+import subprocess
+
+from charmhelpers.core import hookenv, host, templating
+from ols import base
+
+
+def home_dir():
+    return os.path.join("/home", base.user())
+
+
+def ensure_home_dir():
+    host.mkdir(home_dir(), owner=base.user(), group=base.user(), perms=0o755)
+
+
+def get_service_config():
+    config = dict(hookenv.config())
+    config.update(
+        {
+            "base_dir": base.base_dir(),
+            "code_dir": base.code_dir(),
+            "logs_dir": base.logs_dir(),
+            "payloads_dir": base.payloads_dir(),
+            "user": base.user(),
+        }
+    )
+    return config
+
+
+def config_file_path(name, secret=False):
+    if secret:
+        config_dir = os.path.join(base.base_dir(), "secrets")
+    else:
+        config_dir = os.path.join(base.code_dir(), "production-configs")
+    return os.path.join(config_dir, name)
+
+
+def configure_lazr(config, template, name, secret=False):
+    hookenv.log("Writing service configuration.")
+    templating.render(
+        template,
+        config_file_path(name, secret=secret),
+        config,
+        owner="root",
+        group=base.user(),
+        perms=0o440 if secret else 0o444,
+    )
+
+
+def configure_cron(config, template):
+    hookenv.log("Writing crontab.")
+    crontab_path = os.path.join(home_dir(), "crontab")
+    templating.render(
+        template,
+        crontab_path,
+        config,
+        owner=base.user(),
+        group=base.user(),
+        perms=0o600,
+    )
+    subprocess.run(
+        ["sudo", "-H", "-u", base.user(), "crontab", crontab_path], check=True
+    )
diff --git a/launchpad-payload/reactive/launchpad-payload.py b/launchpad-payload/reactive/launchpad-payload.py
new file mode 100644
index 0000000..0a77b45
--- /dev/null
+++ b/launchpad-payload/reactive/launchpad-payload.py
@@ -0,0 +1,43 @@
+# Copyright 2022-2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import subprocess
+
+from charms.launchpad.payload import ensure_home_dir
+from charms.layer import options
+from charms.reactive import clear_flag, set_flag, when, when_not
+from ols import base
+
+
+# Monkey-patch layer:ols.
+def create_virtualenv(wheels_dir, codedir, python_exe):
+    build_target = (
+        options().get("launchpad-payload", {}).get("build_target", "compile")
+    )
+    subprocess.run(
+        ["make", build_target, f"PYTHON={python_exe}"],
+        cwd=codedir,
+        check=True,
+    )
+
+
+base.create_virtualenv = create_virtualenv
+
+
+@when("ols.configured")
+@when_not("launchpad.payload.configured")
+def configure():
+    ensure_home_dir()
+    set_flag("launchpad.payload.configured")
+
+
+@when("launchpad.payload.configured")
+@when_not("ols.configured")
+def deconfigure():
+    clear_flag("launchpad.payload.configured")
+
+
+@when("config.changed.build_label")
+def build_label_changed():
+    clear_flag("ols.service.installed")
+    clear_flag("ols.configured")
diff --git a/launchpad-base/templates/macros.j2 b/launchpad-payload/templates/macros.j2
similarity index 100%
rename from launchpad-base/templates/macros.j2
rename to launchpad-payload/templates/macros.j2