Merge ~cjwatson/lp-codeimport:charm-codeimport-storage into lp-codeimport:master

[Colin Watson]

charm: Add an lp-codeimport-storage charm

This can be used to provide import data storage instead of the older approach of setting `bazaar_branch_store` and `foreign_tree_store` on `lp-codeimport`. It isn't much more than a vanilla system with SSH key authorization and a couple of pre-created directories, but the new `codeimport-storage` relation makes it a little easier to set things up in a Mojo spec.

Data migration from any previous storage systems needs to be handled manually.

[Guruprasad (lgp171188)]

LGTM 👍🏼 Don't we have/need tests for this charm?

review: Approve

[Colin Watson (cjwatson)]

I haven't figured out how to get integration tests set up for our charms yet :-( Definitely something we need though!

a4ad063... by Colin Watson on 2023-03-27

Add icon

diff --git a/charm/lp-codeimport-storage/README.md b/charm/lp-codeimport-storage/README.md
new file mode 100644
index 0000000..cad1a3b
--- /dev/null
+++ b/charm/lp-codeimport-storage/README.md
@@ -0,0 +1,3 @@
+# Launchpad code import storage
+
+This charm provides storage for the `lp-codeimport` charm.
diff --git a/charm/lp-codeimport-storage/charmcraft.yaml b/charm/lp-codeimport-storage/charmcraft.yaml
new file mode 100644
index 0000000..ac76470
--- /dev/null
+++ b/charm/lp-codeimport-storage/charmcraft.yaml
@@ -0,0 +1,44 @@
+type: charm
+bases:
+  - build-on:
+    - name: ubuntu
+      channel: "22.04"
+      architectures: [amd64]
+    run-on:
+    - name: ubuntu
+      channel: "22.04"
+      architectures: [amd64]
+parts:
+  charm-wheels:
+    source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels
+    source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      "*": charm-wheels/
+    prime:
+      - "-charm-wheels"
+  ols-layers:
+    source: https://git.launchpad.net/ols-charm-deps
+    source-commit: "1ca8acbef7eb49b8a2cc81e5e13479b4f226a48b"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      "*": layers/
+    stage:
+      - layers
+    prime:
+      - "-layers"
+  lp-codeimport-storage:
+    after:
+      - charm-wheels
+      - ols-layers
+    source: .
+    plugin: reactive
+    build-snaps: [charm/2.x/stable]
+    build-environment:
+      - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer
+      - PIP_NO_INDEX: "true"
+      - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels
diff --git a/charm/lp-codeimport-storage/config.yaml b/charm/lp-codeimport-storage/config.yaml
new file mode 100644
index 0000000..4f9deac
--- /dev/null
+++ b/charm/lp-codeimport-storage/config.yaml
@@ -0,0 +1,5 @@
+options:
+  public_ssh_key:
+    type: string
+    default: ""
+    description: Base64-encoded public SSH key of the code import workers.
diff --git a/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/__init__.py
diff --git a/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py
new file mode 100644
index 0000000..c6b06a0
--- /dev/null
+++ b/charm/lp-codeimport-storage/hooks/relations/codeimport-storage/provides.py
@@ -0,0 +1,39 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from charmhelpers.core import hookenv
+from charms.reactive import (
+    Endpoint,
+    remove_state,
+    set_state,
+    when,
+    when_not,
+    )
+
+
+class CodeImportStorageProvides(Endpoint):
+    @when("endpoint.{endpoint_name}.joined")
+    @when_not("endpoint.{endpoint_name}.available")
+    def joined(self):
+        for relation in self.relations:
+            ingress_address = hookenv.network_get(
+                relation.endpoint_name, relation_id=relation.relation_id
+            )["ingress-addresses"][0]
+            relation.to_publish[
+                "bazaar_branch_store"
+            ] = f"sftp://importd@{ingress_address}/srv/importd/www/"
+            relation.to_publish[
+                "foreign_tree_store"
+            ] = f"sftp://importd@{ingress_address}/srv/importd/sources/"
+        set_state(self.expand_name("endpoint.{endpoint_name}.available"))
+
+    @when("endpoint.{endpoint_name}.available")
+    @when_not("endpoint.{endpoint_name}.joined")
+    def departed(self):
+        remove_state(self.expand_name("endpoint.{endpoint_name}.available"))
+
+    def get_codeimport_subnets(self):
+        subnets = set()
+        for unit in self.all_joined_units:
+            subnets.update(unit.received_raw["egress-subnets"].split(","))
+        return sorted(subnets)
diff --git a/charm/lp-codeimport-storage/icon.svg b/charm/lp-codeimport-storage/icon.svg
new file mode 100644
index 0000000..b2889cc
--- /dev/null
+++ b/charm/lp-codeimport-storage/icon.svg
@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 165.39062 165.39062"><defs><style>.cls-1{fill:#e9500e;}.cls-2{fill:#fff;}</style></defs><rect class="cls-1" width="165.39062" height="165.39062"/><path class="cls-2" d="M29.63876,57.97189C43.189,67.692,61.13456,69.25577,77.65457,62.15038c16.25576-6.87157,27.74036-21.43444,29.97828-38.0075.04663-.34331.11016-.81367-1.59861-1.24044l-10.10934-2.197c-.3254-.04494-.79136-.04967-1.15258,1.22455C91.37844,36.07384,84.34062,45.04243,72.6347,50.1123c-11.77316,5.10029-23.18748,4.05279-35.91893-3.29386-.58119-.27843-.91909-.26086-1.45568.52577l-5.77947,8.65163A1.34512,1.34512,0,0,0,29.63876,57.97189Z" transform="translate(0.39062 0.39062)"/><path class="cls-2" d="M79.86106,139.66026l10.3631.565c1.74155.03446,1.79122-.42981,1.83717-.77312,2.23826-16.5734-4.97222-33.66107-18.81739-44.59422C59.196,83.62132,41.47815,80.36935,25.83365,86.14747a1.33956,1.33956,0,0,0-.67918,1.85373l3.28,9.88226c.30952.90153.62816,1.011,1.26443.89409,14.22464-3.70543,25.50717-1.68748,35.50635,6.3512,9.94174,7.9934,14.34865,18.50754,13.86883,33.08867C79.08524,139.50144,79.53735,139.615,79.86106,139.66026Z" transform="translate(0.39062 0.39062)"/><path class="cls-2" d="M86.50488,70.59048a10.50817,10.50817,0,0,0-1.39587-.09461A9.35237,9.35237,0,0,0,79.39915,72.382a9.61981,9.61981,0,1,0,7.10573-1.79156Z" transform="translate(0.39062 0.39062)"/><path class="cls-2" d="M138.26869,53.18923,133.457,43.97736c-.68628-1.51583-1.22793-1.36985-1.79594-1.17657-15.382,6.63165-25.99848,21.22156-28.40434,39.03776-2.40755,17.82971,3.97169,34.72681,17.0647,45.19906a1.177,1.177,0,0,0,.90794.32844,1.48362,1.48362,0,0,0,.99546-.54l6.76175-8.11166c.62342-.78393.35783-1.18333.0321-1.52461-10.60639-10.44454-14.5764-20.81677-12.84905-33.60769,1.73682-12.86121,8.51918-22.08254,21.34457-29.019C138.52854,53.95289,138.36533,53.421,138.26869,53.18923Z" transform="translate(0.39062 0.39062)"/></svg>
\ No newline at end of file
diff --git a/charm/lp-codeimport-storage/layer.yaml b/charm/lp-codeimport-storage/layer.yaml
new file mode 100644
index 0000000..590c59f
--- /dev/null
+++ b/charm/lp-codeimport-storage/layer.yaml
@@ -0,0 +1,2 @@
+includes:
+    - layer:basic
diff --git a/charm/lp-codeimport-storage/metadata.yaml b/charm/lp-codeimport-storage/metadata.yaml
new file mode 100644
index 0000000..95de7f8
--- /dev/null
+++ b/charm/lp-codeimport-storage/metadata.yaml
@@ -0,0 +1,14 @@
+name: lp-codeimport-storage
+display-name: lp-codeimport-storage
+summary: Launchpad code import storage
+maintainer: Colin Watson <[e-mail]>
+description: Storage for use with lp-codeimport workers.
+tags:
+  # https://juju.is/docs/charm-metadata#heading--charm-store-fields
+  - network
+series:
+  - jammy
+subordinate: false
+provides:
+  codeimport-storage:
+    interface: codeimport-storage
diff --git a/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py b/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py
new file mode 100644
index 0000000..5be4349
--- /dev/null
+++ b/charm/lp-codeimport-storage/reactive/lp-codeimport-storage.py
@@ -0,0 +1,74 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import base64
+import os.path
+
+from charmhelpers.core import (
+    hookenv,
+    host,
+    templating,
+    )
+from charms.reactive import (
+    endpoint_from_flag,
+    hook,
+    remove_state,
+    set_state,
+    when,
+    when_not,
+    )
+
+
+@hook("upgrade-charm")
+def upgrade_charm():
+    remove_state("service.configured")
+
+
+@hook("config.changed")
+def config_changed():
+    remove_state("service.configured")
+
+
+@hook(
+    "{provides:codeimport-storage}-relation-{joined,changed,broken,departed}"
+)
+def codeimport_storage_changed(*args):
+    remove_state("service.configured")
+
+
+@when("config.set.public_ssh_key", "endpoint.codeimport-storage.available")
+@when_not("service.configured")
+def configure():
+    codeimport_storage = endpoint_from_flag(
+        "endpoint.codeimport-storage.available"
+    )
+    hookenv.log("Creating service user")
+    host.add_group("importd")
+    host.adduser("importd", primary_group="importd")
+    for directory in ("/srv/importd/sources", "/srv/importd/www"):
+        if not os.path.exists(directory):
+            host.mkdir(
+                directory, owner="importd", group="importd", perms=0o755
+            )
+    ssh_dir = "/home/importd/.ssh"
+    if not os.path.exists(ssh_dir):
+        host.mkdir(ssh_dir, owner="importd", group="importd", perms=0o700)
+    config = dict(hookenv.config())
+    config["codeimport_subnets"] = codeimport_storage.get_codeimport_subnets()
+    config["public_ssh_key"] = base64.b64decode(
+        config["public_ssh_key"].encode("ASCII")
+    ).decode("ASCII")
+    templating.render(
+        "authorized_keys.j2",
+        os.path.join(ssh_dir, "authorized_keys"),
+        config,
+        owner="importd",
+        group="importd",
+        perms=0o600,
+    )
+    set_state("service.configured")
+
+
+@when("service.configured")
+def check_is_running():
+    hookenv.status_set("active", "Ready")
diff --git a/charm/lp-codeimport-storage/templates/authorized_keys.j2 b/charm/lp-codeimport-storage/templates/authorized_keys.j2
new file mode 100644
index 0000000..77d398e
--- /dev/null
+++ b/charm/lp-codeimport-storage/templates/authorized_keys.j2
@@ -0,0 +1,2 @@
+restrict,from="{{ codeimport_subnets|join(",") }}" {{ public_ssh_key }}
+
diff --git a/charm/lp-codeimport/config.yaml b/charm/lp-codeimport/config.yaml
index cac88ce..37f990c 100644
--- a/charm/lp-codeimport/config.yaml
+++ b/charm/lp-codeimport/config.yaml
@@ -11,13 +11,16 @@ options:
       default is to use the global CA infrastructure.
   bazaar_branch_store:
     type: string
-    default: sftp://[e-mail]/srv/importd/www/
-    description: Where the Bazaar imports are stored.
+    default: ""
+    description: >
+      Where the Bazaar imports are stored.  If unset, rely on the
+      codeimport-storage relation instead.
   foreign_tree_store:
     type: string
-    default: sftp://[e-mail]/srv/importd/sources/
+    default: ""
     description: >
-      Where the tarballs of foreign branches are uploaded for storage.
+      Where the tarballs of foreign branches are uploaded for storage.  If
+      unset, rely on the codeimport-storage relation instead.
   private_ssh_key:
     type: string
     default: ""
diff --git a/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py b/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/charm/lp-codeimport/hooks/relations/codeimport-storage/__init__.py
diff --git a/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py b/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py
new file mode 100644
index 0000000..16e31d4
--- /dev/null
+++ b/charm/lp-codeimport/hooks/relations/codeimport-storage/requires.py
@@ -0,0 +1,32 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from charms.reactive import (
+    clear_flag,
+    Endpoint,
+    toggle_flag,
+    when,
+    when_not,
+    )
+
+
+class CodeImportStorageRequires(Endpoint):
+    @when("endpoint.{endpoint_name}.changed")
+    def joined(self):
+        toggle_flag(
+            self.expand_name("endpoint.{endpoint_name}.available"),
+            bool(self.bazaar_branch_store) and bool(self.foreign_tree_store),
+        )
+
+    @when("endpoint.{endpoint_name}.available")
+    @when_not("endpoint.{endpoint_name}.joined")
+    def departed(self):
+        clear_flag(self.expand_name("endpoint.{endpoint_name}.available"))
+
+    @property
+    def bazaar_branch_store(self):
+        return self.all_joined_units.received["bazaar_branch_store"]
+
+    @property
+    def foreign_tree_store(self):
+        return self.all_joined_units.received["foreign_tree_store"]
diff --git a/charm/lp-codeimport/metadata.yaml b/charm/lp-codeimport/metadata.yaml
index 0c4ee40..f688508 100644
--- a/charm/lp-codeimport/metadata.yaml
+++ b/charm/lp-codeimport/metadata.yaml
@@ -9,3 +9,6 @@ tags:
 series:
   - bionic
 subordinate: false
+requires:
+  codeimport-storage:
+    interface: codeimport-storage
diff --git a/charm/lp-codeimport/reactive/lp-codeimport.py b/charm/lp-codeimport/reactive/lp-codeimport.py
index 04df2a6..9c7c9e7 100644
--- a/charm/lp-codeimport/reactive/lp-codeimport.py
+++ b/charm/lp-codeimport/reactive/lp-codeimport.py
@@ -19,9 +19,12 @@ from charmhelpers.core import (
     templating,
     )
 from charms.reactive import (
+    endpoint_from_flag,
     remove_state,
     set_state,
     when,
+    when_any,
+    when_none,
     when_not,
     )
 from ols import base
@@ -166,9 +169,51 @@ def configure_rsync(config):
         raise RuntimeError('Failed to restart rsync')
 
 
-@when('ols.configured')
+@when_any(
+    "endpoint.codeimport-storage.available",
+    "config.set.bazaar_branch_store",
+)
+def bazaar_branch_store_available():
+    set_state("service.bazaar_branch_store.available")
+
+
+@when_none(
+    "endpoint.codeimport-storage.available",
+    "config.set.bazaar_branch_store",
+)
+def bazaar_branch_store_unavailable():
+    remove_state("service.bazaar_branch_store.available")
+    remove_state("service.configured")
+
+
+@when_any(
+    "endpoint.codeimport-storage.available",
+    "config.set.foreign_tree_store",
+)
+def foreign_tree_store_available():
+    set_state("service.foreign_tree_store.available")
+
+
+@when_none(
+    "endpoint.codeimport-storage.available",
+    "config.set.foreign_tree_store",
+)
+def foreign_tree_store_unavailable():
+    remove_state("service.foreign_tree_store.available")
+    remove_state("service.configured")
+
+
+@when(
+    "ols.configured",
+    "service.bazaar_branch_store.available",
+    "service.foreign_tree_store.available",
+)
 @when_not('service.configured')
 def configure():
+    codeimport_storage = endpoint_from_flag(
+        "endpoint.codeimport-storage.available"
+    )
+
     ensure_lp_directories()
 
     system_packages = os.path.join(base.code_dir(), 'system-packages.txt')
@@ -188,6 +233,14 @@ def configure():
     config_path = os.path.join(
         base.code_dir(), 'production-configs', 'charm', 'codeimport-lazr.conf')
     svc_config = dict(config)
+    if not svc_config["bazaar_branch_store"]:
+        svc_config["bazaar_branch_store"] = (
+            codeimport_storage.bazaar_branch_store
+        )
+    if not svc_config["foreign_tree_store"]:
+        svc_config["foreign_tree_store"] = (
+            codeimport_storage.foreign_tree_store
+        )
     svc_config.update({
         'base_dir': base.base_dir(),
         'code_dir': base.code_dir(),
@@ -199,7 +252,7 @@ def configure():
         'home_dir': home_dir(),
         'user': base.user(),
         'code_import_storage_host': (
-            urlparse(config['bazaar_branch_store']).hostname),
+            urlparse(svc_config['bazaar_branch_store']).hostname),
         # Chosen to allow distributing dispatch start time over a 30-second
         # interval.
         'dispatch_offset': host.modulo_distribution(modulo=6, wait=5),