Merge ~cjwatson/turnip/+git/turnipcake:publish-tarball into ~canonical-launchpad-branches/turnip/+git/turnipcake:master

Proposed by Colin Watson on 2020-06-12
Status: Merged
Approved by: Colin Watson on 2020-06-29
Approved revision: 76a4b700ea9279226a6481f146cd372a798b5869
Merged at revision: 76a4b700ea9279226a6481f146cd372a798b5869
Proposed branch: ~cjwatson/turnip/+git/turnipcake:publish-tarball
Merge into: ~canonical-launchpad-branches/turnip/+git/turnipcake:master
Diff against target: 177 lines (+137/-2)
4 files modified
Makefile (+12/-1)
dependencies-devel.txt (+1/-0)
ols-vms.conf (+1/-1)
publish-to-swift (+123/-0)
Reviewer Review Type Date Requested Status
Ioana Lasc 2020-06-12 Approve on 2020-06-12
Review via email: mp+385659@code.launchpad.net

Commit message

Add a "make publish-tarball" target

Description of the change

This will help us to write a Jenkins job that builds a deployment artifact and publishes it to Swift for use by the deployment machinery.

This is mostly copied from lp-signing.

To post a comment you must log in.
Ioana Lasc (ilasc) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/Makefile b/Makefile
2index f0e720c..020b1d7 100644
3--- a/Makefile
4+++ b/Makefile
5@@ -22,6 +22,11 @@ TARBALL_BUILDS_DIR ?= build
6 TARBALL_BUILD_DIR = $(TARBALL_BUILDS_DIR)/$(TARBALL_BUILD_LABEL)
7 TARBALL_BUILD_PATH = $(TARBALL_BUILD_DIR)/$(TARBALL_FILE_NAME)
8
9+SWIFT_CONTAINER_NAME ?= turnipcake-builds
10+# This must match the object path used by install_payload in the turnip-base
11+# charm layer.
12+SWIFT_OBJECT_PATH = turnipcake-builds/$(TARBALL_BUILD_LABEL)/$(TARBALL_FILE_NAME)
13+
14 build: $(ENV)
15
16 turnipcake/version_info.py:
17@@ -91,4 +96,10 @@ build-tarball:
18 --exclude env \
19 ./
20
21-.PHONY: build check clean dist lint run migrate build-tarball
22+publish-tarball: build-tarball
23+ [ ! -e ~/.config/swift/turnip ] || . ~/.config/swift/turnip; \
24+ ./publish-to-swift --debug \
25+ $(SWIFT_CONTAINER_NAME) $(SWIFT_OBJECT_PATH) \
26+ $(TARBALL_BUILD_PATH)
27+
28+.PHONY: build check clean dist lint run migrate build-tarball publish-tarball
29diff --git a/dependencies-devel.txt b/dependencies-devel.txt
30new file mode 100644
31index 0000000..e3bee0e
32--- /dev/null
33+++ b/dependencies-devel.txt
34@@ -0,0 +1 @@
35+python3-swiftclient
36diff --git a/ols-vms.conf b/ols-vms.conf
37index 7b93d08..15fbdbf 100644
38--- a/ols-vms.conf
39+++ b/ols-vms.conf
40@@ -3,7 +3,7 @@ vm.architecture = amd64
41 vm.release = bionic
42
43 apt.sources = ppa:launchpad/ppa
44-vm.packages = @charm/packages.txt
45+vm.packages = @dependencies-devel.txt, @charm/packages.txt
46
47 [turnipcake]
48 vm.class = lxd
49diff --git a/publish-to-swift b/publish-to-swift
50new file mode 100755
51index 0000000..e5cfd59
52--- /dev/null
53+++ b/publish-to-swift
54@@ -0,0 +1,123 @@
55+#! /usr/bin/python3
56+
57+"""Publish a built tarball to Swift for deployment."""
58+
59+from argparse import ArgumentParser
60+import os
61+import re
62+import subprocess
63+import sys
64+
65+
66+def ensure_container_privs(container_name):
67+ """Ensure that the container exists and is world-readable.
68+
69+ This allows us to give services suitable credentials for getting the
70+ built code from a container.
71+ """
72+ subprocess.run(["swift", "post", container_name, "--read-acl", ".r:*"])
73+
74+
75+def get_swift_storage_url():
76+ # This is a bit cumbersome, but probably still easier than bothering
77+ # with swiftclient.
78+ auth = subprocess.run(
79+ ["swift", "auth"],
80+ stdout=subprocess.PIPE, check=True,
81+ universal_newlines=True).stdout.splitlines()
82+ return [
83+ line.split("=", 1)[1] for line in auth
84+ if line.startswith("export OS_STORAGE_URL=")][0]
85+
86+
87+def publish_file_to_swift(container_name, object_path, local_path,
88+ overwrite=True):
89+ """Publish a file to a Swift container."""
90+ storage_url = get_swift_storage_url()
91+
92+ already_published = False
93+ # Some swift versions unhelpfully exit 0 regardless of whether the
94+ # object exists.
95+ try:
96+ stats = subprocess.run(
97+ ["swift", "stat", container_name, object_path],
98+ stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=True,
99+ universal_newlines=True).stdout
100+ if re.search(
101+ r"Object: %s$" % re.escape(object_path), stats, flags=re.M):
102+ already_published = True
103+ except subprocess.CalledProcessError:
104+ pass
105+
106+ if already_published:
107+ print("Object {} already published to {}.".format(
108+ object_path, container_name))
109+ if not overwrite:
110+ return
111+
112+ print("Publishing {} to {} as {}.".format(
113+ local_path, container_name, object_path))
114+ try:
115+ subprocess.run(
116+ ["swift", "upload", "--object-name", object_path,
117+ container_name, local_path])
118+ except subprocess.CalledProcessError:
119+ sys.exit("Failed to upload {} to {} as {}".format(
120+ local_path, container_name, object_path))
121+
122+ print("Published file: {}/{}/{}".format(
123+ storage_url, container_name, object_path))
124+
125+
126+def main():
127+ parser = ArgumentParser()
128+ parser.add_argument("--debug", action="store_true", default=False)
129+ parser.add_argument("container_name")
130+ parser.add_argument("swift_object_path")
131+ parser.add_argument("local_path")
132+ args = parser.parse_args()
133+
134+ if args.debug:
135+ # Print OpenStack-related environment variables for ease of
136+ # debugging. Only OS_AUTH_TOKEN and OS_PASSWORD currently seem to
137+ # be secret, but for safety we only show unredacted contents of
138+ # variables specifically known to be safe. See "swift --os-help"
139+ # for most of these.
140+ safe_keys = {
141+ "OS_AUTH_URL",
142+ "OS_AUTH_VERSION",
143+ "OS_CACERT",
144+ "OS_CERT",
145+ "OS_ENDPOINT_TYPE",
146+ "OS_IDENTITY_API_VERSION",
147+ "OS_INTERFACE",
148+ "OS_KEY",
149+ "OS_PROJECT_DOMAIN_ID",
150+ "OS_PROJECT_DOMAIN_NAME",
151+ "OS_PROJECT_ID",
152+ "OS_PROJECT_NAME",
153+ "OS_REGION_NAME",
154+ "OS_SERVICE_TYPE",
155+ "OS_STORAGE_URL",
156+ "OS_TENANT_ID",
157+ "OS_TENANT_NAME",
158+ "OS_USERNAME",
159+ "OS_USER_DOMAIN_ID",
160+ "OS_USER_DOMAIN_NAME",
161+ "OS_USER_ID",
162+ }
163+ for key, value in sorted(os.environ.items()):
164+ if key.startswith("OS_"):
165+ if key not in safe_keys:
166+ value = "<redacted>"
167+ print("{}: {}".format(key, value))
168+
169+ overwrite = "FORCE_REBUILD" in os.environ
170+ ensure_container_privs(args.container_name)
171+ publish_file_to_swift(
172+ args.container_name, args.swift_object_path, args.local_path,
173+ overwrite=overwrite)
174+
175+
176+if __name__ == "__main__":
177+ main()

Subscribers

People subscribed via source and target branches