Merge ~cjwatson/launchpad:build-tarball into launchpad:master

[Colin Watson]

Support publishing deployment artifact to Swift

This adds `build-tarball` and `publish-tarball` make targets, which will help us to write a Jenkins job that builds a deployment artifact and publishes it to Swift for use by future deployment machinery.

The new `ols-vms.conf` file is to assist integration with lp:ols-jenkaas, which is where we currently do publication to Swift for other projects.

Aside from the new build and publish scripts, I had to adjust the existing wheel-building target a bit to also build wheels for the packages in `requirements/setup.txt`.

[Jürgen Gmach (jugmac00)]

LGTM, given that I have not much knowledge about swift (what about a quick introduction at one of the next standups?)

Also, I wonder about the big picture of this MP.

I thought future deployment will be done with Juju/Charms, so there would be no more need for interwoven bash scripts and make files?

review: Approve

[Colin Watson (cjwatson)]

Swift is OpenStack's distributed object store component. See https://docs.openstack.org/swift/latest/ for documentation. In PS5 the same API is provided by ceph-radosgw (https://docs.ceph.com/en/latest/radosgw/index.html), but it ends up being much the same. We use it for the Launchpad librarian, but it's also a handy way to store blobs for deployment.

Juju charms will indeed eventually let us refactor a whole bunch of confusing deployment machinery, but they don't help with the preliminary step of building code artifacts for deployment; that still has to be done separately one way or another. This work will replace something that's currently done by a combination of carob:~pqm/bin/update-launchpad-built-code.sh (which builds a tree) and lp:deployment-manager (which among other things rsyncs that tree out to target machines), as well as some even more confusing and less coherent things on (qa)staging.

[Jürgen Gmach (jugmac00)]

Thanks for the detailed replies!

diff --git a/Makefile b/Makefile
index 8b00a2e..69ad953 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,8 @@ PY=$(WD)/bin/py
 PYTHONPATH:=$(WD)/lib:${PYTHONPATH}
 VERBOSITY=-vv
 
+DEPENDENCY_REPO ?= https://git.launchpad.net/lp-source-dependencies
+
 # virtualenv and pip fail if setlocale fails, so force a valid locale.
 PIP_ENV := LC_ALL=C.UTF-8
 # Run with "make PIP_NO_INDEX=" if you want pip to find software
@@ -95,6 +97,20 @@ PIP_BIN = \
     bin/watch_jsbuild \
     bin/with-xvfb
 
+# Create archives in labelled directories (e.g.
+# <rev-id>/$(PROJECT_NAME).tar.gz)
+TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+TARBALL_FILE_NAME = launchpad.tar.gz
+TARBALL_BUILD_DIR = dist/$(TARBALL_BUILD_LABEL)
+TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
+
+SWIFT_CONTAINER_NAME ?= launchpad-builds
+# This must match the object path used by fetch_payload in the ols charm
+# layer.
+SWIFT_OBJECT_PATH = \
+       launchpad-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
+
+
 # DO NOT ALTER : this should just build by default
 .PHONY: default
 default: inplace
@@ -173,6 +189,17 @@ inplace: build logs clean_logs codehosting-dir
 .PHONY: build
 build: compile apidoc jsbuild css_combine
 
+# Bootstrap download-cache and sourcecode.  Useful for CI jobs that want to
+# set these up from scratch.
+.PHONY: bootstrap
+bootstrap:
+	if [ -d download-cache/.git ]; then \
+		git -C download-cache pull; \
+	else \
+		git clone --depth=1 $(DEPENDENCY_REPO) download-cache; \
+	fi
+	utilities/update-sourcecode
+
 # LP_SOURCEDEPS_PATH should point to the sourcecode directory, but we
 # want the parent directory where the download-cache and env directories
 # are. We re-use the variable that is using for the rocketfuel-get script.
@@ -258,29 +285,48 @@ requirements/combined.txt: \
 		--include requirements/launchpad.txt \
 		>"$@"
 
-# This target is used by LOSAs to prepare a build to be pushed out to
-# destination machines.  We only want wheels: they are the expensive bits,
-# and the other bits might run into problems like bug 575037.  This target
-# runs pip, builds a wheelhouse with predictable paths that can be used even
-# if the build is pushed to a different path on the destination machines,
-# and then removes everything created except for the wheels.
-#
 # It doesn't seem to be straightforward to build a wheelhouse of all our
 # dependencies without also building a useless wheel of Launchpad itself;
 # fortunately that doesn't take too long, and we just remove it afterwards.
-.PHONY: build_wheels
-build_wheels: $(PIP_BIN) requirements/combined.txt
+.PHONY: build_wheels_only
+build_wheels_only: $(PIP_BIN) requirements/combined.txt
 	$(RM) -r wheelhouse wheels
+	$(SHHH) $(PIP) wheel -w wheels -r requirements/setup.txt
 	$(SHHH) $(PIP) wheel \
 		-c requirements/setup.txt -c requirements/combined.txt \
 		-w wheels .
 	$(RM) wheels/lp-[0-9]*.whl
-	$(MAKE) clean_pip
+
+# This target is used by deployment machinery to prepare a build to be
+# pushed out to destination machines.  We only want wheels: they are the
+# expensive bits, and the other bits might run into problems like bug
+# 575037.  This target runs pip, builds a wheelhouse with predictable paths
+# that can be used even if the build is pushed to a different path on the
+# destination machines, and then removes everything created except for the
+# wheels.
+.PHONY: build_wheels
+build_wheels: build_wheels_only
+	$(MAKE) clean_js clean_pip
 
 # Compatibility
 .PHONY: build_eggs
 build_eggs: build_wheels
 
+# Build a tarball that can be unpacked and built on another machine,
+# including all the wheels we need.  This will eventually supersede
+# build_wheels.
+.PHONY: build-tarball
+build-tarball:
+	utilities/build-tarball $(TARBALL_BUILD_DIR)
+
+# Publish a buildable tarball to Swift.
+.PHONY: publish-tarball
+publish-tarball: build-tarball
+	[ ! -e ~/.config/swift/launchpad ] || . ~/.config/swift/launchpad; \
+	utilities/publish-to-swift --debug \
+		$(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
+		$(TARBALL_BUILD_PATH)
+
 # setuptools won't touch files that would have the same contents, but for
 # Make's sake we need them to get fresh timestamps, so we touch them after
 # building.
diff --git a/ols-vms.conf b/ols-vms.conf
new file mode 100644
index 0000000..cd26bce
--- /dev/null
+++ b/ols-vms.conf
@@ -0,0 +1,11 @@
+# Options defined here provide defaults for all sections
+vm.architecture = amd64
+vm.release = xenial
+
+apt.sources = ppa:launchpad/ppa
+vm.packages = launchpad-dependencies
+
+[launchpad]
+vm.class = lxd
+vm.update = True
+jenkaas.secrets = swift/launchpad:.config/swift/launchpad
diff --git a/utilities/build-tarball b/utilities/build-tarball
new file mode 100755
index 0000000..6923320
--- /dev/null
+++ b/utilities/build-tarball
@@ -0,0 +1,43 @@
+#! /bin/sh
+#
+# Copyright 2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+set -e
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 OUTPUT-DIRECTORY" >&2
+    exit 2
+fi
+output_dir="$1"
+
+# Build wheels to include in the distributed tarball.
+# XXX cjwatson 2021-12-10: It's unfortunate that this script is called from
+# the Makefile and also has to call out to the Makefile, but this is
+# difficult to disentangle until we refactor our build system to use
+# something higher-level than pip.
+make build_wheels_only
+
+# Ensure that we have an updated idea of this tree's version.
+scripts/update-version-info.sh
+
+echo "Creating deployment tarball in $output_dir"
+
+# Prepare a file list.
+mkdir -p "$output_dir"
+(
+    git ls-files | sed 's,^,./,'
+    # Most of download-cache is unnecessary since we include a wheelhouse
+    # instead, but JavaScript-enabled builds need yarn.
+    find ./download-cache/dist/ -name yarn-\*.tar.gz
+    # The subdirectories of sourcecode may be symlinks.  Force tar to
+    # include the actual files instead.
+    find ./sourcecode/*/ \
+        \( -name .bzr -o -name __pycache__ \) -prune -o -type f -print
+    echo ./version-info.py
+    find ./wheels/ -name \*.whl -print
+) | sort >"$output_dir/.files"
+
+# Create the tarball.
+tar -czf "$output_dir/launchpad.tar.gz" --no-recursion \
+    --files-from "$output_dir/.files"
diff --git a/utilities/publish-to-swift b/utilities/publish-to-swift
new file mode 100755
index 0000000..84de4c1
--- /dev/null
+++ b/utilities/publish-to-swift
@@ -0,0 +1,160 @@
+#! /usr/bin/python3 -S
+#
+# Copyright 2021 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Publish a built tarball to Swift for deployment."""
+
+import _pythonpath  # noqa: F401
+
+from argparse import ArgumentParser
+import os
+
+import iso8601
+import requests
+from swiftclient.service import (
+    get_conn,
+    process_options,
+    SwiftService,
+    SwiftUploadObject,
+    )
+from swiftclient.shell import add_default_args
+
+
+def ensure_container_privs(options, container_name):
+    """Ensure that the container exists and is world-readable.
+
+    This allows us to give services suitable credentials for getting the
+    built code from a container.
+    """
+    options = dict(options)
+    options["read_acl"] = ".r:*"
+    with SwiftService(options=options) as swift:
+        swift.post(container=container_name)
+
+
+def get_swift_storage_url(options):
+    """Return the storage URL under which Swift objects are published."""
+    return get_conn(options).get_auth()[0]
+
+
+def publish_file_to_swift(options, container_name, object_path, local_path,
+                          overwrite=True):
+    """Publish a file to a Swift container."""
+    storage_url = get_swift_storage_url(options)
+
+    with SwiftService(options=options) as swift:
+        stat_results = swift.stat(
+            container=container_name, objects=[object_path])
+        if stat_results and next(stat_results)["success"]:
+            print("Object {} already published to {}.".format(
+                object_path, container_name))
+            if not overwrite:
+                return
+
+        print("Publishing {} to {} as {}.".format(
+            local_path, container_name, object_path))
+        for r in swift.upload(
+                container_name,
+                [SwiftUploadObject(local_path, object_name=object_path)]):
+            if not r["success"]:
+                raise r["error"]
+
+        print("Published file: {}/{}/{}".format(
+            storage_url, container_name, object_path))
+
+
+def prune_old_files_from_swift(options, container_name, object_dir):
+    """Prune files from Swift that we no longer need."""
+    response = requests.head("https://launchpad.net/")
+    response.raise_for_status()
+    production_revision = response.headers["X-VCS-Revision"]
+
+    with SwiftService(options=options) as swift:
+        objs = {}
+        production_mtime = None
+        for stats in swift.list(
+                container=container_name,
+                options={"prefix": "{}/".format(object_dir)}):
+            if not stats["success"]:
+                raise stats["error"]
+            for item in stats["listing"]:
+                if item.get("subdir") is None:
+                    mtime = iso8601.parse_date(item["last_modified"])
+                    objs[item["name"]] = mtime
+                    if item["name"].startswith(
+                            "{}/{}/".format(object_dir, production_revision)):
+                        production_mtime = mtime
+
+        if production_mtime is None:
+            print(
+                "No file in {} corresponding to production revision {}; "
+                "not pruning.".format(container_name, production_revision))
+            return
+
+        for object_name, mtime in sorted(objs.items()):
+            if mtime < production_mtime:
+                print("Pruning {} (older than production)".format(object_name))
+                for r in swift.delete(
+                        container=container_name, objects=[object_name]):
+                    if not r["success"]:
+                        raise r["error"]
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument("container_name")
+    parser.add_argument("swift_object_path")
+    parser.add_argument("local_path")
+    add_default_args(parser)
+    args = parser.parse_args()
+
+    if args.debug:
+        # Print OpenStack-related environment variables for ease of
+        # debugging.  Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
+        # be secret, but for safety we only show unredacted contents of
+        # variables specifically known to be safe.  See "swift --os-help"
+        # for most of these.
+        safe_keys = {
+            "OS_AUTH_URL",
+            "OS_AUTH_VERSION",
+            "OS_CACERT",
+            "OS_CERT",
+            "OS_ENDPOINT_TYPE",
+            "OS_IDENTITY_API_VERSION",
+            "OS_INTERFACE",
+            "OS_KEY",
+            "OS_PROJECT_DOMAIN_ID",
+            "OS_PROJECT_DOMAIN_NAME",
+            "OS_PROJECT_ID",
+            "OS_PROJECT_NAME",
+            "OS_REGION_NAME",
+            "OS_SERVICE_TYPE",
+            "OS_STORAGE_URL",
+            "OS_TENANT_ID",
+            "OS_TENANT_NAME",
+            "OS_USERNAME",
+            "OS_USER_DOMAIN_ID",
+            "OS_USER_DOMAIN_NAME",
+            "OS_USER_ID",
+            }
+        for key, value in sorted(os.environ.items()):
+            if key.startswith("OS_"):
+                if key not in safe_keys:
+                    value = "<redacted>"
+                print("{}: {}".format(key, value))
+
+    options = vars(args)
+    process_options(options)
+
+    overwrite = "FORCE_REBUILD" in os.environ
+    ensure_container_privs(options, args.container_name)
+    publish_file_to_swift(
+        options, args.container_name, args.swift_object_path, args.local_path,
+        overwrite=overwrite)
+    prune_old_files_from_swift(
+        options, args.container_name, args.swift_object_path.split("/")[0])
+
+
+if __name__ == "__main__":
+    main()