Merge lp:~stub/charms/precise/postgresql/repoless into lp:charms/postgresql

[Stuart Bishop]

Embed test charms and remove JUJU_REPOSITORY requirement

A common issue with running the PostgreSQL charm is getting JUJU_REPOSITORY setup.

Because the PostgreSQL charm and thus its tests is supported on both trusty and precise, the test suite detects the juju environment's default-series and uses that as the series to use, unless overridden. This in itself isn't a problem, except that the automatic test runner is attempting to run tests with precise, but has a juju environment setup with default-series of trusty, and everything explodes.

Also, the postgresql-psql charm is not in the charm store under trusty, so needed to be in JUJU_REPOSITORY. I elected to not get it pushed to trusty, as its current dual purpose of being a general purpose command line client and for driving tests means it works poorly on both.

While the automatic test runner is getting sorted out, lets do some improvements I'd been planning on anyway. Embed the the client charm our tests need, and build a JUJU_REPOSITORY dynamically.

This still leaves the charm detecting the series to use, so the automatic test runner will continue to actually be running trusty tests when it was hoping for precise, but at least they should pass now. This cannot be fixed until the test environment is setup with a correct default-series, or some other mechanism is available for detecting the series to test against (such as an environment variable).

[lp:~stub/charms/precise/postgresql/repoless]

165. By Stuart Bishop on 2014-10-14

Ignore regenerated charmhelpers

166. By Stuart Bishop on 2014-10-24

Merged data-checksums into repoless.

Unmerged revisions

166. By Stuart Bishop on 2014-10-24

Merged data-checksums into repoless.

165. By Stuart Bishop on 2014-10-14

Ignore regenerated charmhelpers

=== added directory 'lib/test-client-charm'
=== added file 'lib/test-client-charm/config.yaml'
--- lib/test-client-charm/config.yaml	1970-01-01 00:00:00 +0000
+++ lib/test-client-charm/config.yaml	2014-10-14 08:59:11 +0000
@@ -0,0 +1,13 @@
+options:
+  database:
+    default: ""
+    type: string
+    description: |
+        Database to connect 'db' relationships to, overriding the
+        generated default database name.
+  roles:
+    default: ""
+    type: string
+    description: |
+        Comma separated list of roles for PostgreSQL to grant to the database
+        user.
 
=== added directory 'lib/test-client-charm/hooks'
=== added symlink 'lib/test-client-charm/hooks/config-changed'
=== target is u'hooks.py'
=== added symlink 'lib/test-client-charm/hooks/db-admin-relation-broken'
=== target is u'./hooks.py'
=== added symlink 'lib/test-client-charm/hooks/db-admin-relation-changed'
=== target is u'./hooks.py'
=== added symlink 'lib/test-client-charm/hooks/db-admin-relation-joined'
=== target is u'./hooks.py'
=== added symlink 'lib/test-client-charm/hooks/db-relation-broken'
=== target is u'./hooks.py'
=== added symlink 'lib/test-client-charm/hooks/db-relation-changed'
=== target is u'./hooks.py'
=== added symlink 'lib/test-client-charm/hooks/db-relation-joined'
=== target is u'./hooks.py'
=== added file 'lib/test-client-charm/hooks/hooks.py'
--- lib/test-client-charm/hooks/hooks.py	1970-01-01 00:00:00 +0000
+++ lib/test-client-charm/hooks/hooks.py	2014-10-14 08:59:11 +0000
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+
+import os.path
+import re
+import shutil
+import sys
+from textwrap import dedent
+
+from charmhelpers.core import hookenv, host
+from charmhelpers.core.hookenv import log, DEBUG, INFO
+from charmhelpers import fetch
+
+
+CLIENT_RELATION_TYPES = frozenset(['db', 'db-admin'])
+
+DATA_DIR = os.path.join(
+    '/var/lib/units', hookenv.local_unit().replace('/', '-'))
+SCRIPT_DIR = os.path.join(DATA_DIR, 'bin')
+PGPASS_DIR = os.path.join(DATA_DIR, 'pgpass')
+
+
+def update_system_path():
+    org_lines = open('/etc/environment', 'rb').readlines()
+    env_lines = []
+
+    for line in org_lines:
+        if line.startswith('PATH=') and SCRIPT_DIR not in line:
+            line = re.sub(
+                """(['"]?)$""",
+                ":{}\\1".format(SCRIPT_DIR),
+                line, 1)
+        env_lines.append(line)
+
+    if org_lines != env_lines:
+        content = '\n'.join(env_lines)
+        host.write_file('/etc/environment', content, perms=0o644)
+
+
+def all_relations(relation_types=CLIENT_RELATION_TYPES):
+    for reltype in relation_types:
+        for relid in hookenv.relation_ids(reltype):
+            for unit in hookenv.related_units(relid):
+                yield reltype, relid, unit, hookenv.relation_get(
+                    unit=unit, rid=relid)
+
+
+def rebuild_all_relations():
+    config = hookenv.config()
+
+    # Clear out old scripts and pgpass files
+    if os.path.exists(SCRIPT_DIR):
+        shutil.rmtree(SCRIPT_DIR)
+    if os.path.exists(PGPASS_DIR):
+        shutil.rmtree(PGPASS_DIR)
+    host.mkdir(DATA_DIR, perms=0o755)
+    host.mkdir(SCRIPT_DIR, perms=0o755)
+    host.mkdir(PGPASS_DIR, group='ubuntu', perms=0o750)
+
+    for _, relid, unit, relation in all_relations(relation_types=['db']):
+        log("{} {} {!r}".format(relid, unit, relation), DEBUG)
+
+        def_str = '<DEFAULT>'
+        if config['database'] != relation.get('database', ''):
+            log("Switching from database {} to {}".format(
+                relation.get('database', '') or def_str,
+                config['database'] or def_str), INFO)
+
+        if config['roles'] != relation.get('roles', ''):
+            log("Updating granted roles from {} to {}".format(
+                relation.get('roles', '') or def_str,
+                config['roles'] or def_str))
+
+        hookenv.relation_set(
+            relid, database=config['database'], roles=config['roles'])
+
+        if 'user' in relation:
+            rebuild_relation(relid, unit, relation)
+
+    for _, relid, unit, relation in all_relations(relation_types=['db-admin']):
+        log("{} {} {!r}".format(relid, unit, relation), DEBUG)
+        if 'user' in relation:
+            rebuild_relation(relid, unit, relation)
+
+
+def rebuild_relation(relid, unit, relation):
+    relname = relid.split(':')[0]
+    unitname = unit.replace('/', '-')
+    this_unit = hookenv.local_unit()
+
+    allowed_units = relation.get('allowed-units', '')
+    if this_unit not in allowed_units.split():
+        log("Not yet authorized on {}".format(relid), INFO)
+        return
+
+    script_name = 'psql-{}-{}'.format(relname, unitname)
+    build_script(script_name, relation)
+    state = relation.get('state', None)
+    if state in ('master', 'hot standby'):
+        script_name = 'psql-{}-{}'.format(relname, state.replace(' ', '-'))
+        build_script(script_name, relation)
+
+
+def build_script(script_name, relation):
+    # Install a wrapper to psql that connects it to the desired database
+    # by default. One wrapper per unit per relation.
+    script_path = os.path.abspath(os.path.join(SCRIPT_DIR, script_name))
+    pgpass_path = os.path.abspath(os.path.join(PGPASS_DIR, script_name))
+    script = dedent("""\
+        #!/bin/sh
+        exec env \\
+            PGHOST={host} PGPORT={port} PGDATABASE={database} \\
+            PGUSER={user} PGPASSFILE={pgpass} \\
+            psql $@
+        """).format(
+            host=relation['host'],
+            port=relation['port'],
+            database=relation.get('database', ''),  # db-admin has no database
+            user=relation['user'],
+            pgpass=pgpass_path)
+    log("Generating wrapper {}".format(script_path), INFO)
+    host.write_file(
+        script_path, script, owner="ubuntu", group="ubuntu", perms=0o700)
+
+    # The wrapper requires access to the password, stored in a .pgpass
+    # file so it isn't exposed in an environment variable or on the
+    # command line.
+    pgpass = "*:*:*:{user}:{password}".format(
+        user=relation['user'], password=relation['password'])
+    host.write_file(
+        pgpass_path, pgpass, owner="ubuntu", group="ubuntu", perms=0o400)
+
+
+hooks = hookenv.Hooks()
+
+
[e-mail]()
+def install():
+    fetch.apt_install(
+        ['language-pack-en', 'postgresql-client', 'python-psycopg2'],
+        fatal=True)
+    update_system_path()
+
+
[e-mail]()
+def upgrade_charm():
+    # Per Bug #1205286, we can't store scripts and passwords in the
+    # charm directory.
+    if os.path.exists('bin'):
+        shutil.rmtree('bin')
+    if os.path.exists('pgpass'):
+        shutil.rmtree('pgpass')
+    update_system_path()
+    return rebuild_all_relations()
+
+
[e-mail](
+    'config-changed', 'db-admin-relation-broken',
+    'db-admin-relation-changed', 'db-admin-relation-joined',
+    'db-relation-broken', 'db-relation-changed', 'db-relation-joined')
+def rebuild_hook():
+    return rebuild_all_relations()
+
+
+if __name__ == '__main__':
+    hooks.execute(sys.argv)
 
=== added symlink 'lib/test-client-charm/hooks/install'
=== target is u'./hooks.py'
=== added file 'lib/test-client-charm/hooks/start'
--- lib/test-client-charm/hooks/start	1970-01-01 00:00:00 +0000
+++ lib/test-client-charm/hooks/start	2014-10-14 08:59:11 +0000
@@ -0,0 +1,3 @@
+#!/bin/sh
+# Do nothing.
+exit 0
 
=== added file 'lib/test-client-charm/hooks/stop'
--- lib/test-client-charm/hooks/stop	1970-01-01 00:00:00 +0000
+++ lib/test-client-charm/hooks/stop	2014-10-14 08:59:11 +0000
@@ -0,0 +1,3 @@
+#!/bin/sh
+# Do nothing.
+exit 0
 
=== added symlink 'lib/test-client-charm/hooks/upgrade-charm'
=== target is u'hooks.py'
=== added file 'lib/test-client-charm/metadata.yaml'
--- lib/test-client-charm/metadata.yaml	1970-01-01 00:00:00 +0000
+++ lib/test-client-charm/metadata.yaml	2014-10-14 08:59:11 +0000
@@ -0,0 +1,13 @@
+name: postgresql-psql
+summary: psql command line access to PostgreSQL services.
+maintainer: [e-mail]
+description: |
+    Access to related PostgreSQL services via the
+    standard psql command line utility.
+categories:
+    - databases
+requires:
+    db:
+        interface: pgsql
+    db-admin:
+        interface: pgsql
 
=== modified file 'test.py'
--- test.py	2014-10-14 08:59:11 +0000
+++ test.py	2014-10-14 08:59:11 +0000
@@ -11,6 +11,7 @@
 
 from datetime import datetime
 import os.path
+import shutil
 import signal
 import socket
 import subprocess
@@ -26,8 +27,8 @@
 
 
 SERIES = os.environ.get('SERIES', 'trusty').strip()
-TEST_CHARM = 'local:{}/postgresql'.format(SERIES)
-PSQL_CHARM = 'local:{}/postgresql-psql'.format(SERIES)
+TEST_CHARM = os.path.dirname(__file__)
+PSQL_CHARM = os.path.join(TEST_CHARM, 'lib', 'test-client-charm')
 
 
 class NotReady(Exception):
@@ -81,6 +82,16 @@
         # Tests may add or change options.
         self.pg_config = dict(version=self.VERSION, pgdg=self.PGDG)
 
+        # Mirror charmhelpers into our support charms, since charms
+        # can't symlink out of their subtree.
+        here = os.path.abspath(os.path.dirname(__file__))
+        main_charmhelpers = os.path.join(here, 'hooks', 'charmhelpers')
+        psql_charmhelpers = os.path.join(here, 'lib', 'test-client-charm',
+                                         'hooks', 'charmhelpers')
+        if os.path.exists(psql_charmhelpers):
+            shutil.rmtree(psql_charmhelpers)
+        shutil.copytree(main_charmhelpers, psql_charmhelpers)
+
         self.juju = self.useFixture(JujuFixture(
             series=SERIES, reuse_machines=True,
             do_teardown='TEST_DONT_TEARDOWN_JUJU' not in os.environ))
 
=== modified file 'testing/jujufixture.py'
--- testing/jujufixture.py	2014-06-10 11:36:17 +0000
+++ testing/jujufixture.py	2014-10-14 08:59:11 +0000
@@ -21,6 +21,7 @@
         super(JujuFixture, self).__init__()
 
         self.series = series
+        assert series, 'series not set'
 
         self.reuse_machines = reuse_machines
 
@@ -47,6 +48,8 @@
     def deploy(self, charm, name=None, num_units=1, config=None):
         cmd = ['deploy']
 
+        charm = self.charm_uri(charm)
+
         if config:
             config_path = os.path.join(
                 self.useFixture(fixtures.TempDir()).path, 'config.yaml')
@@ -185,6 +188,27 @@
         if self.do_teardown:
             self.addCleanup(self.reset)
 
+        # Setup a temporary repository with the magic to find our charms.
+        self.repo_dir = self.useFixture(fixtures.TempDir()).path
+        self.useFixture(fixtures.EnvironmentVariable('JUJU_REPOSITORY',
+                                                     self.repo_dir))
+        self.repo_series_dir = os.path.join(self.repo_dir, self.series)
+        os.mkdir(self.repo_series_dir)
+
+    def charm_uri(self, charm):
+        meta = yaml.safe_load(open(os.path.join(charm, 'metadata.yaml'), 'rb'))
+        name = meta.get('name')
+        assert name, 'charm {} has no name in metadata.yaml'.format(charm)
+        charm_link = os.path.join(self.repo_series_dir, name)
+
+        # Recreate the charm link, which might have changed from the
+        # last deploy.
+        if os.path.exists(charm_link):
+            os.remove(charm_link)
+        os.symlink(charm, os.path.join(self.repo_series_dir, name))
+
+        return 'local:{}/{}'.format(self.series, name)
+
     def reset(self):
         # Tear down any services left running that we know we spawned.
         while True: