Merge ~tcuthbert/turku/+git/turku-snap:development into turku:main

[Thomas Cuthbert]

0f78a8f... by Thomas Cuthbert on 2022-01-05

add: turku-storage and staticfiles compilation

Unmerged commits

0f78a8f... by Thomas Cuthbert on 2022-01-05

add: turku-storage and staticfiles compilation

bf1ae73... by Thomas Cuthbert on 2021-11-20

feat: initial turku-api snap config

diff --git a/README.md b/README.md
index 2163b0b..c37c583 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,4 @@
+<<<<<<< README.md
 # turku-storage
 
 ## About Turku
@@ -102,3 +103,53 @@ This program is free software: you can redistribute it and/or modify it under th
 This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+=======
+# Turku API Snap
+
+## Overview
+
+- installs turku-api from LP.
+- sets PYTHONPATH order as:
+  `$SNAP_DATA/turku-api:$SNAP/lib/python3.6/site-packages`
+- copies the module into `$SNAP_DATA`, this works around turku's local settings
+  import loading from `turku_api.local_settings`.
+  - then it symlinks `$SNAP_COMMON/local_settings.py -> $SNAP_DATA/turku_api/local_settings.py`.
+- gunicorn is included and is the snap app command.
+- basic hooks:
+  - bind (listener:port)
+    - listener is validated via the rgxg package
+  - workers
+
+## Build
+
+```
+snapcraft clean
+snapcraft
+```
+
+## Install
+
+```
+snap=
+snap install "$snap" --dangerous --devmode
+```
+
+## Configure
+
+```
+listener=
+port=
+workers=
+snap set turku-api bind.listener=$listener
+snap set turku-api bind.port=$port
+snap set turku-api workers=$workers
+```
+
+## Todo
+
+- [ ] test that snap upgrades don't break things.
+- [ ] test local_settings.py actually works.
+- [ ] test when running upstream as an upstream server.
+- [ ] not critical, but the `turku-api-wrapper` script should be called from
+  `${SNAP}/bin`
+>>>>>>> README.md
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
new file mode 100644
index 0000000..e4c2443
--- /dev/null
+++ b/snap/snapcraft.yaml
@@ -0,0 +1,115 @@
+name: turku-api # you probably want to 'snapcraft register <name>'
+base: core18
+version: '0.1+git'
+summary: Turku API
+description: |
+  Turku is an agent-based backup system where the server doing the backups has
+  no direct access to the machine being backed up.  Instead, the machine's
+  agent coordinates with an API server and opens a reverse tunnel to the
+  storage server when it is time to do a backup.
+
+grade: devel # must be 'stable' to release into candidate/stable channels
+confinement: strict # use 'strict' once you have the right plugs and slots
+
+apps:
+
+  turku-api:
+    daemon: simple
+    plugs: [network-bind]
+    environment:
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+      PYTHONPATH: $SNAP_DATA/turku-api:$SNAP/lib/python3.6/site-packages
+    command: snap/hooks/turku-api-wrapper
+
+  turku-admin:
+    environment:
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+      PYTHONPATH: $SNAP_DATA/turku-api:$SNAP/lib/python3.6/site-packages
+    command: bin/turku-admin
+
+  turku-storage-ping:
+    environment:
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+      PYTHONPATH: $SNAP/lib/python3.6/python3.6/site-packages
+    command: bin/turku-storage-ping
+
+  turku-storage-update-config:
+    environment:
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+      PYTHONPATH: $SNAP/lib/python3.6/python3.6/site-packages
+    command: bin/turku-storage-update-config
+
+hooks:
+  configure:
+    plugs: [network-bind]
+
+parts:
+
+  turku-api:
+    plugin: python
+    source-type: git
+    source: https://git.launchpad.net/~turku/turku/+git/turku-api
+    source-branch: main
+    python-version: python3
+    python-packages: [psycopg2, gunicorn]
+    requirements: [requirements.txt]
+    prime:
+      - -.venv
+      - -venv
+      - -env
+      - -lib/python3.6/python3.6/site-packages/turku*
+    build-packages:
+      - libpq-dev
+    stage-packages:
+      - libpq5
+    override-build: |
+      snapcraftctl build
+
+      cp -r ./turku_api $SNAPCRAFT_PART_INSTALL/
+
+    override-stage: |
+      snapcraftctl stage
+
+      export PYTHONPATH="$SNAPCRAFT_PART_INSTALL:$SNAPCRAFT_PART_INSTALL/lib/python3.6/site-packages"
+      export DJANGO_SETTINGS_MODULE="turku_api.settings"
+
+      mkdir "static"
+      printf '%s = "%s"\n' "STATIC_ROOT" "static" > "$SNAPCRAFT_PART_INSTALL/turku_api/local_settings.py"
+      "$SNAPCRAFT_PART_INSTALL/bin/django-admin" collectstatic
+      rm "$SNAPCRAFT_PART_INSTALL/turku_api/local_settings.py"
+
+      cp -r static "$SNAPCRAFT_PART_INSTALL/"
+
+    after: [turku-admin, hooks]
+
+  turku-storage:
+    plugin: python
+    source-type: git
+    source: https://git.launchpad.net/turku
+    source-branch: main
+    python-version: python3
+    requirements: [requirements.txt]
+    prime:
+      - -.venv
+      - -venv
+      - -env
+    after: [turku-admin, hooks]
+
+  turku-admin:
+    plugin: dump
+    source: src/turku-api/
+    stage:
+      - bin/*
+      - config/*
+
+  hooks:
+    plugin: dump
+    source: src/hooks/
+    stage-packages:
+      - rgxg
+    organize:
+      bin/: snap/hooks/
diff --git a/src/hooks/bin/configure b/src/hooks/bin/configure
new file mode 100755
index 0000000..74f23fa
--- /dev/null
+++ b/src/hooks/bin/configure
@@ -0,0 +1,52 @@
+#!/bin/sh -e
+
+. "$SNAP/snap/hooks/turku-management"
+
+handle_port_config()
+{
+        http_port="$(http_port)"
+
+        # Validate HTTP port
+        if ! expr "$http_port" : '^[0-9]\+$' > /dev/null; then
+                echo "\"$http_port\" is not a valid HTTP port" >&2
+                return 1
+        fi
+
+        set_http_port "$http_port"
+
+        snapctl restart turku-api
+}
+
+handle_listener_config()
+{
+        listener="$(listener)"
+
+        # Validate listener
+        if ! rgxg cidr "$listener"/32; then
+               echo "\"$listener\" is not a valid IP address" >&2
+               return 1
+        fi
+
+        set_listener "$listener"
+
+        snapctl restart turku-api
+}
+
+handle_workers_config()
+{
+        workers="$(workers)"
+
+        # Validate gunicorn workers
+        if ! expr "$workers" : '^[0-9]\+$' > /dev/null; then
+               echo "\"$workers\" is not a valid number" >&2
+               return 1
+        fi
+
+        set_workers "$workers"
+
+        snapctl restart turku-api
+}
+
+handle_port_config
+handle_listener_config
+handle_workers_config
diff --git a/src/hooks/bin/install b/src/hooks/bin/install
new file mode 100755
index 0000000..17ee20d
--- /dev/null
+++ b/src/hooks/bin/install
@@ -0,0 +1,11 @@
+#!/bin/sh -e
+
+if [ ! -d "$SNAP_DATA/turku-api" ]; then
+  mkdir -p "$SNAP_DATA/turku-api"
+fi
+
+cp -r "$SNAP/turku_api" "$SNAP_DATA/turku-api/"
+cp -r "$SNAP/static" "$SNAP_DATA/"
+
+touch "$SNAP_DATA/turku-api/turku_api/local_settings.py"
+ln -fs "$SNAP_DATA/turku-api/turku_api/local_settings.py" "$SNAP_COMMON/local_settings.py" 
diff --git a/src/hooks/bin/turku-api-wrapper b/src/hooks/bin/turku-api-wrapper
new file mode 100755
index 0000000..9650198
--- /dev/null
+++ b/src/hooks/bin/turku-api-wrapper
@@ -0,0 +1,11 @@
+#!/bin/sh -eu
+
+. "$SNAP/snap/hooks/turku-management"
+
+TURKU_LISTENER="$(listener)"
+TURKU_HTTP_PORT="$(http_port)"
+TURKU_WORKERS="$(workers)"
+
+export GUNICORN_CMD_ARGS="--bind '${TURKU_LISTENER}:${TURKU_HTTP_PORT}' --workers '${TURKU_WORKERS}'"
+
+"$SNAP/bin/gunicorn" turku_api.wsgi
diff --git a/src/hooks/bin/turku-management b/src/hooks/bin/turku-management
new file mode 100755
index 0000000..cc79356
--- /dev/null
+++ b/src/hooks/bin/turku-management
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+DEFAULT_HTTP_PORT="8000"
+DEFAULT_LISTENER="127.0.0.1"  # gunicorn default
+
+http_port()
+{
+        port="$(snapctl get bind.port)"
+        if [ -z "$port" ]; then
+                port="$DEFAULT_HTTP_PORT"
+                set_http_port $port
+        fi
+        echo "$port"
+}
+
+set_http_port()
+{
+        snapctl set bind.port="$1"
+}
+
+listener()
+{
+        listener="$(snapctl get bind.listener)"
+        if [ -z "$listener" ]; then
+                listener="$DEFAULT_LISTENER"
+                set_listener $listener
+        fi
+        echo "$listener"
+}
+
+set_listener()
+{
+        snapctl set bind.listener="$1"
+}
+
+workers()
+{
+        workers="$(snapctl get workers)"
+        if [ -z "$workers" ]; then
+                workers="$(expr $(nproc) - 1)"
+                set_workers $workers
+        fi
+        echo "$workers"
+}
+
+set_workers()
+{
+        snapctl set workers="$1"
+}
diff --git a/src/turku-api/bin/turku-admin b/src/turku-api/bin/turku-admin
new file mode 100755
index 0000000..19da088
--- /dev/null
+++ b/src/turku-api/bin/turku-admin
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -eu
+
+export PYTHONPATH="$SNAP_DATA/turku-api"
+export DJANGO_SETTINGS_MODULE=turku_api.settings
+
+"$SNAP/bin/django-admin" "${@}"
diff --git a/src/turku-api/config/local_settings.py b/src/turku-api/config/local_settings.py
new file mode 100644
index 0000000..84b9a63
--- /dev/null
+++ b/src/turku-api/config/local_settings.py
@@ -0,0 +1,5 @@
+import os
+
+BASE_DIR = os.path.dirname(os.path.dirname(__file__))
+
+STATIC_ROOT = os.path.join(BASE_DIR, "static")