Merge ~cjwatson/launchpad:publish-db-tarballs-to-swift into launchpad:master

[Colin Watson]

Support publishing tarballs of db-devel to Swift

Unlike our other environments, Launchpad's staging environment is built from the `db-stable` branch, which is fed from `db-devel`. To support deploying this using Juju charms, we need to build a separate stream of deployment artifacts from the `db-devel` branch, old versions of which are pruned on a different schedule.

(Discovered while trying to set up auto-upgrades for the new Juju-deployed staging appservers. After this lands, we'll also need a change to lp:ols-jenkaas to actually run the builds.)

[Guruprasad (lgp171188)]

LGTM 👍🏼

review: Approve

diff --git a/Makefile b/Makefile
index 7bd08d1..f60189b 100644
--- a/Makefile
+++ b/Makefile
@@ -88,7 +88,14 @@ PIP_BIN = \
 
 # Create archives in labelled directories (e.g.
 # <rev-id>/$(PROJECT_NAME).tar.gz)
-TARBALL_BUILD_LABEL ?= $(shell git rev-parse HEAD)
+GIT_BRANCH := $(shell git branch --show-current)
+TARBALL_REVISION ?= $(shell git rev-parse HEAD)
+ifeq ($(GIT_BRANCH),db-devel)
+TARBALL_SUFFIX := db
+else
+TARBALL_SUFFIX :=
+endif
+TARBALL_BUILD_LABEL := $(TARBALL_REVISION)$(if $(TARBALL_SUFFIX),-$(TARBALL_SUFFIX))
 TARBALL_FILE_NAME = launchpad.tar.gz
 TARBALL_BUILD_DIR = dist/$(TARBALL_BUILD_LABEL)
 TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
@@ -304,7 +311,7 @@ publish-tarball: build-tarball
 	[ ! -e ~/.config/swift/launchpad ] || . ~/.config/swift/launchpad; \
 	utilities/publish-to-swift --debug \
 		$(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
-		$(TARBALL_BUILD_PATH)
+		$(TARBALL_BUILD_PATH) $(TARBALL_SUFFIX)
 
 # setuptools won't touch files that would have the same contents, but for
 # Make's sake we need them to get fresh timestamps, so we touch them after
diff --git a/utilities/publish-to-swift b/utilities/publish-to-swift
index 93280d8..3d13f8c 100755
--- a/utilities/publish-to-swift
+++ b/utilities/publish-to-swift
@@ -8,6 +8,7 @@
 import _pythonpath  # noqa: F401
 
 import os
+import re
 from argparse import ArgumentParser
 
 import iso8601
@@ -76,15 +77,26 @@ def publish_file_to_swift(
         )
 
 
-def prune_old_files_from_swift(options, container_name, object_dir):
+def prune_old_files_from_swift(options, container_name, object_dir, suffix):
     """Prune files from Swift that we no longer need."""
-    response = requests.head("https://launchpad.net/")
-    response.raise_for_status()
-    production_revision = response.headers["X-VCS-Revision"]
+    if suffix:
+        suffix = "-" + suffix
+    if suffix.endswith("-db"):
+        response = requests.head("https://staging.launchpad.net/")
+        if not response.ok:
+            # Staging is routinely down in order to restore its database
+            # from a recent production dump, so don't consider this an
+            # error; just skip pruning if we can't determine its revision.
+            print("staging.launchpad.net is down; not pruning.")
+            return
+    else:
+        response = requests.head("https://launchpad.net/")
+        response.raise_for_status()
+    deployed_revision = response.headers["X-VCS-Revision"]
 
     with SwiftService(options=options) as swift:
         objs = {}
-        production_mtime = None
+        deployed_mtime = None
         for stats in swift.list(
             container=container_name,
             options={"prefix": "{}/".format(object_dir)},
@@ -92,24 +104,35 @@ def prune_old_files_from_swift(options, container_name, object_dir):
             if not stats["success"]:
                 raise stats["error"]
             for item in stats["listing"]:
+                if not item["name"].startswith("{}/".format(object_dir)):
+                    continue
+                # Only consider pruning builds that have the given suffix.
+                # (For example, if we're publishing a build with the suffix
+                # "-db", then we consider pruning builds with the label
+                # "<commit>-db" but not plain "<commit>".)
+                build_label = item["name"].split("/")[1]
+                if not re.match(r"^.{40}%s$" % re.escape(suffix), build_label):
+                    continue
                 if item.get("subdir") is None:
                     mtime = iso8601.parse_date(item["last_modified"])
                     objs[item["name"]] = mtime
-                    if item["name"].startswith(
-                        "{}/{}/".format(object_dir, production_revision)
-                    ):
-                        production_mtime = mtime
+                    if build_label == deployed_revision + suffix:
+                        deployed_mtime = mtime
 
-        if production_mtime is None:
+        if deployed_mtime is None:
             print(
-                "No file in {} corresponding to production revision {}; "
-                "not pruning.".format(container_name, production_revision)
+                "No file in {} corresponding to deployed revision {}; "
+                "not pruning.".format(container_name, deployed_revision)
             )
             return
 
         for object_name, mtime in sorted(objs.items()):
-            if mtime < production_mtime:
-                print("Pruning {} (older than production)".format(object_name))
+            if mtime < deployed_mtime:
+                print(
+                    "Pruning {} (older than deployed revision)".format(
+                        object_name
+                    )
+                )
                 for r in swift.delete(
                     container=container_name, objects=[object_name]
                 ):
@@ -122,6 +145,7 @@ def main():
     parser.add_argument("container_name")
     parser.add_argument("swift_object_path")
     parser.add_argument("local_path")
+    parser.add_argument("suffix", nargs="?", default="")
     add_default_args(parser)
     args = parser.parse_args()
 
@@ -173,7 +197,10 @@ def main():
         overwrite=overwrite,
     )
     prune_old_files_from_swift(
-        options, args.container_name, args.swift_object_path.split("/")[0]
+        options,
+        args.container_name,
+        args.swift_object_path.split("/")[0],
+        args.suffix,
     )