Merge ~cjwatson/launchpad:charm-debian-importer into launchpad:master

[Colin Watson]

charm: Add a launchpad-debian-importer charm

[Simone Pelosi (pelpsi)]

LGTM!

[Simone Pelosi (pelpsi)]

review: Approve

diff --git a/charm/Makefile b/charm/Makefile
index d1e4611..857f6b2 100644
--- a/charm/Makefile
+++ b/charm/Makefile
@@ -17,6 +17,7 @@ CHARMS := \
 	launchpad-admin \
 	launchpad-appserver \
 	launchpad-assets \
+	launchpad-debian-importer \
 	launchpad-librarian
 
 all: ## alias to build
diff --git a/charm/launchpad-debian-importer/README.md b/charm/launchpad-debian-importer/README.md
new file mode 100644
index 0000000..6ae99ca
--- /dev/null
+++ b/charm/launchpad-debian-importer/README.md
@@ -0,0 +1,13 @@
+# Launchpad Debian importer
+
+This charm runs a job that imports Debian packages into Launchpad.
+
+You will need the following relations:
+
+    juju relate launchpad-debian-importer:db postgresql:db
+    juju relate launchpad-debian-importer rabbitmq-server
+
+You will also need to set the `librarian_upload_host` and
+`librarian_upload_port` options to point to a local
+[launchpad-librarian](https://charmhub.io/launchpad-librarian) instance;
+this is not currently handled using relations.
diff --git a/charm/launchpad-debian-importer/charmcraft.yaml b/charm/launchpad-debian-importer/charmcraft.yaml
new file mode 100644
index 0000000..1634d67
--- /dev/null
+++ b/charm/launchpad-debian-importer/charmcraft.yaml
@@ -0,0 +1,60 @@
+type: charm
+bases:
+  - build-on:
+    - name: ubuntu
+      channel: "20.04"
+      architectures: [amd64]
+    run-on:
+    - name: ubuntu
+      channel: "20.04"
+      architectures: [amd64]
+parts:
+  charm-wheels:
+    source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels
+    source-commit: "59b32ae07f98051385c96d6d8e7e02ca4f197fe5"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      "*": charm-wheels/
+    prime:
+      - "-charm-wheels"
+  ols-layers:
+    source: https://git.launchpad.net/ols-charm-deps
+    source-commit: "56d219f60a293a6c73759b8439ef5fdb11e19d1f"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      "*": layers/
+    stage:
+      - layers
+    prime:
+      - "-layers"
+  launchpad-layers:
+    after:
+      - ols-layers
+    source: https://git.launchpad.net/launchpad-layers
+    source-commit: "6a50917f5f6163069ae1661e3320abb5b48173a3"
+    source-submodules: []
+    source-type: git
+    plugin: dump
+    organize:
+      launchpad-base: layers/layer/launchpad-base
+    stage:
+      - layers
+    prime:
+      - "-layers"
+  launchpad-debian-importer:
+    after:
+      - charm-wheels
+      - launchpad-layers
+    source: .
+    plugin: reactive
+    build-snaps: [charm/2.x/stable]
+    build-packages: [libpq-dev]
+    build-environment:
+      - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer
+      - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface
+      - PIP_NO_INDEX: "true"
+      - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels
diff --git a/charm/launchpad-debian-importer/config.yaml b/charm/launchpad-debian-importer/config.yaml
new file mode 100644
index 0000000..6b0c325
--- /dev/null
+++ b/charm/launchpad-debian-importer/config.yaml
@@ -0,0 +1,14 @@
+options:
+  active:
+    type: boolean
+    description: If true, enable jobs that may change the database.
+    default: true
+  debian_mirror_host:
+    type: string
+    description: Hostname of the Debian mirror to use.
+    default: "ftp.debian.org"
+  debian_suites:
+    type: string
+    description: YAML-encoded dict mapping Debian suites to components.
+    default: |
+      sid: ["main", "contrib", "non-free"]
diff --git a/charm/launchpad-debian-importer/layer.yaml b/charm/launchpad-debian-importer/layer.yaml
new file mode 100644
index 0000000..88e2ce1
--- /dev/null
+++ b/charm/launchpad-debian-importer/layer.yaml
@@ -0,0 +1,13 @@
+includes:
+  - layer:launchpad-base
+repo: https://git.launchpad.net/launchpad
+options:
+  apt:
+    packages:
+      - debian-archive-keyring
+      - debmirror
+  ols-pg:
+    databases:
+      db:
+        name: launchpad_dev
+        roles: gina
diff --git a/charm/launchpad-debian-importer/metadata.yaml b/charm/launchpad-debian-importer/metadata.yaml
new file mode 100644
index 0000000..ff28cae
--- /dev/null
+++ b/charm/launchpad-debian-importer/metadata.yaml
@@ -0,0 +1,15 @@
+name: launchpad-debian-importer
+display-name: launchpad-debian-importer
+summary: Launchpad Debian importer
+maintainer: Colin Watson <[e-mail]>
+description: |
+  Launchpad is an open source suite of tools that help people and teams
+  to work together on software projects.
+
+  This charm runs a job that imports Debian packages into Launchpad.
+tags:
+  # https://juju.is/docs/charm-metadata#heading--charm-store-fields
+  - network
+series:
+  - focal
+subordinate: false
diff --git a/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py b/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py
new file mode 100644
index 0000000..f87e95b
--- /dev/null
+++ b/charm/launchpad-debian-importer/reactive/launchpad-debian-importer.py
@@ -0,0 +1,62 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import os.path
+
+import yaml
+from charmhelpers.core import hookenv, host, templating
+from charms.launchpad.base import (
+    configure_cron,
+    configure_lazr,
+    get_service_config,
+)
+from charms.reactive import set_state, when, when_not
+from ols import base
+
+
+def configure_logrotate(config):
+    hookenv.log("Writing logrotate configuration.")
+    templating.render(
+        "logrotate.conf.j2",
+        "/etc/logrotate.d/launchpad-debian-importer",
+        config,
+        perms=0o644,
+    )
+
+
+@when("launchpad.base.configured")
+@when_not("service.configured")
+def configure():
+    config = get_service_config()
+    config["debian_suites"] = yaml.safe_load(config["debian_suites"])
+    config["debian_components"] = []
+    for suite, components in config["debian_suites"].items():
+        for component in components:
+            if component not in config["debian_components"]:
+                config["debian_components"].append(component)
+    config["mirror_dir"] = os.path.join(base.base_dir(), "mirror")
+    host.mkdir(
+        config["mirror_dir"], owner=base.user(), group=base.user(), perms=0o755
+    )
+    config["scripts_dir"] = os.path.join(base.base_dir(), "scripts")
+    host.mkdir(config["scripts_dir"], perms=0o755)
+    templating.render(
+        "mirror-update.sh.j2",
+        os.path.join(config["scripts_dir"], "mirror-update.sh"),
+        config,
+        perms=0o755,
+    )
+    configure_lazr(
+        config,
+        "launchpad-debian-importer-lazr.conf",
+        "launchpad-debian-importer/launchpad-lazr.conf",
+    )
+    configure_logrotate(config)
+    configure_cron(config, "crontab.j2")
+
+    set_state("service.configured")
+
+
+@when("service.configured")
+def check_is_running():
+    hookenv.status_set("active", "Ready")
diff --git a/charm/launchpad-debian-importer/templates/crontab.j2 b/charm/launchpad-debian-importer/templates/crontab.j2
new file mode 100644
index 0000000..2f9d7e2
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/crontab.j2
@@ -0,0 +1,15 @@
+TZ=UTC
+MAILFROM={{ bounce_address }}
+MAILTO={{ cron_mailto }}
+LPCONFIG=launchpad-debian-importer
+
+{% if active -%}
+# Mirror from Debian and then run the importer.  Debian publishes at "52
+# 1,7,13,19", but we also need to allow time for the above mirror to update.
+25 4,10,16,22 * * * {{ scripts_dir }}/mirror-update.sh >> {{ logs_dir }}/mirror-update.log 2>&1 && {{ code_dir }}/scripts/gina.py {{ debian_suites.keys()|reverse|join(" ") }} -q --log-file=INFO:{{ logs_dir }}/gina.log
+
+{% endif -%}
+# Catch up with publishing OOPSes that were temporarily spooled to disk due
+# to RabbitMQ being unavailable.
+*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_username }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_vhost }} --repo {{ oopses_dir }} --key ""
+
diff --git a/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf b/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf
new file mode 100644
index 0000000..2152ea5
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/launchpad-debian-importer-lazr.conf
@@ -0,0 +1,20 @@
+# Public configuration data.  The contents of this file may be freely shared
+# with developers if needed for debugging.
+
+# A schema's sections, keys, and values are automatically inherited, except
+# for '.optional' sections.  Update this config to override key values.
+# Values are strings, except for numbers that look like ints.  The tokens
+# true, false, and none are treated as True, False, and None.
+
+{% from "macros.j2" import opt -%}
+
+[meta]
+extends: ../launchpad-base-lazr.conf
+
+{% for suite, components in debian_suites.items() -%}
+[gina_target.{{ suite }}]
+components: {{ components|join(",") }}
+source_only: True
+root: {{ mirror_dir }}/
+
+{% endfor -%}
diff --git a/charm/launchpad-debian-importer/templates/logrotate.conf.j2 b/charm/launchpad-debian-importer/templates/logrotate.conf.j2
new file mode 100644
index 0000000..51d1421
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/logrotate.conf.j2
@@ -0,0 +1,11 @@
+{{ logs_dir }}/*.log
+{
+    rotate 21
+    daily
+    dateext
+    delaycompress
+    compress
+    notifempty
+    missingok
+}
+
diff --git a/charm/launchpad-debian-importer/templates/mirror-update.sh.j2 b/charm/launchpad-debian-importer/templates/mirror-update.sh.j2
new file mode 100644
index 0000000..446c3af
--- /dev/null
+++ b/charm/launchpad-debian-importer/templates/mirror-update.sh.j2
@@ -0,0 +1,24 @@
+#! /bin/sh
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+# Part of the launchpad-debian-importer Juju charm.
+
+set -e
+
+DEST="{{ mirror_dir }}/"
+HOST="{{ debian_mirror_host }}"
+DISTS="{{ debian_suites.keys()|join(',') }}"
+SECTIONS="{{ debian_components|join(',') }}"
+
+{% if http_proxy -%}
+http_proxy="{{ http_proxy }}"
+ftp_proxy="{{ http_proxy }}"
+export http_proxy ftp_proxy
+
+{% endif -%}
+echo "Starting Mirror: $(date)"
+rm -rf "${DEST}/.temp"
+debmirror --keyring /usr/share/keyrings/debian-archive-keyring.gpg --root=debian --host="$HOST" --method=http --dist="$DISTS" --section="$SECTIONS" --arch=none --diff=none --progress --rsync-extra=none --allow-dist-rename "$DEST"
+echo "Finished Mirror: $(date)"
+