Merge lp:~cjwatson/charms/trusty/turnip/code-in-swift into lp:~canonical-launchpad-branches/charms/trusty/turnip/devel

[Colin Watson]

Add support for fetching turnip payload from Swift.

Add support for fetching turnip payload from Swift.

I haven't actually tested this yet, but it will be a lot easier to do on qastaging than locally! The code is modelled reasonably closely on that in the certification charm (lp:~gnuoy/canonical-is-charms/certification-python), so it should be more or less right.

[Kit Randel (blr)]

LGTM, although would it be worth factoring out the name of the tarball?

[lp:~cjwatson/charms/trusty/turnip/code-in-swift]

82. By Colin Watson on 2015-06-18

Factor out tarball name.

[Colin Watson (cjwatson)]

This has long since been superseded by the in-tree charm.

Unmerged revisions

82. By Colin Watson on 2015-06-18

Factor out tarball name.

81. By Colin Watson on 2015-06-17

Add support for fetching turnip payload from Swift.

=== modified file 'config.yaml'
--- config.yaml	2015-05-22 15:42:11 +0000
+++ config.yaml	2015-06-18 13:52:10 +0000
@@ -37,6 +37,10 @@
     type: string
     default: '19417'
     description: API port.
+  code_src:
+    type: string
+    default: 'local'
+    description: Code source: 'local' or 'swift'.
   revision:
     type: string
     default: '1'
@@ -120,6 +124,30 @@
     description: |
       Hosts that should be allowed to rsync logs. Note that this relies on
       basenode.
+  swift_username:
+    type: string
+    default: ""
+    description: Username to use when accessing Swift.
+  swift_password:
+    type: string
+    default: ""
+    description: Password to use when accessing Swift.
+  swift_auth_url:
+    type: string
+    default: ""
+    description: URL for authenticating against Keystone.
+  swift_region_name:
+    type: string
+    default: ""
+    description: Swift region.
+  swift_tenant_name:
+    type: string
+    default: ""
+    description: Entity that owns resources.
+  swift_container_name:
+    type: string
+    default: ""
+    description: Container to put objects in.
 
   # apt configuration used by charmhelpers.
   install_sources:

=== modified file 'hooks/actions.py'
--- hooks/actions.py	2015-05-22 15:42:11 +0000
+++ hooks/actions.py	2015-06-18 13:52:10 +0000
@@ -1,5 +1,6 @@
 import base64
 import grp
+import hashlib
 import os
 import pwd
 import shutil
@@ -39,6 +40,7 @@
 LOGS_DIR = os.path.join(BASE_DIR, 'logs')
 DATA_DIR = os.path.join(BASE_DIR, 'data')
 KEY_DIR = os.path.join(BASE_DIR, 'keys')
+CODE_TARBALL = 'turnip.tar.gz'
 
 CODE_USER = config['code_user']
 CODE_GROUP = config['code_group']
@@ -87,16 +89,90 @@
     host.add_user_to_group(USER, CGIT_GROUP)
 
 
+def get_swift_creds(config):
+    return {
+        'user': config['swift_username'],
+        'project': config['swift_tenant_name'],
+        'password': config['swift_password'],
+        'authurl': config['swift_auth_url'],
+        'region': config['swift_region_name'],
+        }
+
+
+def swift_base_cmd(**swift_creds):
+    return [
+        'swift',
+        '--os-username=' + swift_creds['user'],
+        '--os-tenant-name=' + swift_creds['project'],
+        '--os-password=' + swift_creds['password'],
+        '--os-auth-url=' + swift_creds['authurl'],
+        '--os-region-name=' + swift_creds['region'],
+        ]
+
+
+def swift_get_etag(name, container=None, **swift_creds):
+    cmd = swift_base_cmd(**swift_creds) + ['stat', container, name]
+    file_stat = subprocess.check_output(cmd).splitlines()
+    for line in file_stat:
+        words = line.split()
+        if words[0] == 'ETag:':
+            return words[1]
+
+
+def swift_fetch(source, target, container=None, **swift_creds):
+    cmd = swift_base_cmd(**swift_creds) + [
+        'download', '--output=' + target, container, source]
+    subprocess.check_call(cmd)
+
+
+def md5sum(filename):
+    md5 = hashlib.md5()
+    with open(filename, 'rb') as f:
+        for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
+            md5.update(chunk)
+    return md5.hexdigest()
+
+
 def unpack_source(service_name):
     hookenv.log('Deploying source...')
 
     make_srv_location()
 
     # Copy source archive
-    archive_path = os.path.join(BASE_DIR, 'turnip.tar.gz')
+    archive_path = os.path.join(BASE_DIR, CODE_TARBALL)
+    if os.path.exists(archive_path):
+        archive_md5sum = md5sum(archive_path)
+    else:
+        archive_md5sum = None
 
-    with open(os.path.join(CHARM_FILES_DIR, 'turnip.tar.gz')) as file:
-        host.write_file(archive_path, file.read(), perms=0o644)
+    if config['code_src'] == 'swift':
+        swift_creds = get_swift_creds(config)
+        swift_container = config['swift_container_name']
+        swift_etag = swift_get_etag(
+            CODE_TARBALL, container=swift_container, **swift_creds)
+        if swift_etag is None:
+            raise Exception(
+                'code_src == swift, but %s not found in Swift' % CODE_TARBALL)
+        if archive_md5sum is not None and swift_etag == archive_md5sum:
+            hookenv.log(
+                'Skipping code deployment: checksum of %s matches checksum '
+                'of file in Swift' % archive_path)
+            return
+        swift_fetch(
+            CODE_TARBALL, archive_path, container=swift_container,
+            **swift_creds)
+    elif config['code_src'] == 'local':
+        charm_path = os.path.join(CHARM_FILES_DIR, CODE_TARBALL)
+        if archive_md5sum is not None and md5sum(charm_path) == archive_md5sum:
+            hookenv.log(
+                'Skipping code deployment: checksum of %s matches checksum '
+                'of %s' % (archive_path, charm_path))
+            return
+        with open(charm_path) as file:
+            host.write_file(archive_path, file.read(), perms=0o644)
+    else:
+        raise Exception(
+            "Unsupported code_src value: '%s'" % config['code_src'])
 
     # Unpack source
     archive.extract_tarfile(archive_path, CODE_DIR)
@@ -109,7 +185,10 @@
 def install_packages(service_name):
     hookenv.log('Installing system packages...')
     fetch.configure_sources(update=True)
-    fetch.apt_install(REQUIRED_PACKAGES, fatal=True)
+    packages = list(REQUIRED_PACKAGES)
+    if config['code_src'] == 'swift':
+        packages.append('python-swiftclient')
+    fetch.apt_install(packages, fatal=True)
 
 
 def install_python_packages(service_name):