Merge lp:~psivaa/uci-engine/rabbitmq-restish-with-proxy into lp:uci-engine/mthood

[Para Siva]

Making rabbitmq and restish charm deployments with proxy. Should be similar changes to the lander-jenkins and python-django. Also contains change to the rabbitmq upstart job to include the proxy env variables.

Making rabbitmq and restish charm deployments with proxy. Spotting is not easy when the charm silently ignores packages listed in yaml for installation :/. Should be similar changes to the lander-jenkins and python-django. Also contains change to the rabbitmq upstart job to include the proxy env variables.

[Andy Doan (doanac)]

one comment about the jinja stuff again

[lp:~psivaa/uci-engine/rabbitmq-restish-with-proxy]

418. By Para Siva on 2014-05-22

Better conditional logic for private-public cloud in rabbitmq charm upstart job

419. By Para Siva on 2014-05-29

Adding no_proxy for the keystone, swift hosts

420. By Para Siva on 2014-05-29

no_proxy in deploy and yaml files

[Para Siva (psivaa)]

This could go in to mthood branch before parking that for a bit.

[Vincent Ladeuil (vila)]

Thanks for the hard work, let's get that in.

review: Approve

=== modified file 'charms/precise/rabbitmq-worker/config.yaml'
--- charms/precise/rabbitmq-worker/config.yaml	2014-03-10 22:25:00 +0000
+++ charms/precise/rabbitmq-worker/config.yaml	2014-05-29 10:25:18 +0000
@@ -18,6 +18,18 @@
   pip-packages:
     type: string
     description: "Pip packages required for this service"
+  proxy-url:
+    type: string
+    default: ""
+    description: "The http proxy url for private clouds."
+  no-proxy:
+    type: string
+    default: ""
+    description: "The host ips to bypass proxy."
+  ppa-gpg-key:
+    type: string
+    default: ""
+    description: GPG key fingerprint for the PPA
   install_root:
     type: string
     description: "The root directory the service will be installed in"
 
=== modified file 'charms/precise/rabbitmq-worker/hooks/hooks.py'
--- charms/precise/rabbitmq-worker/hooks/hooks.py	2014-03-10 22:25:00 +0000
+++ charms/precise/rabbitmq-worker/hooks/hooks.py	2014-05-29 10:25:18 +0000
@@ -81,15 +81,42 @@
     subprocess.check_call(args)
 
 
+def setup_apt_proxy(config):
+    proxy_url = config['proxy-url']
+    apt_conf = textwrap.dedent("""\
+        Acquire::http::proxy "%s";
+        Acquire::https::proxy "%s";
+        Acquire::HTTP::PROXY "%s";
+        Acquire::HTTPS::PROXY "%s";
+        """ % (proxy_url, proxy_url, proxy_url, proxy_url))
+    with open('/etc/apt/apt.conf.d/95proxies', 'w') as f:
+        f.write(apt_conf)
+
+
+def import_ppa_keys(config):
+    cmd = ['apt-key', 'adv', '--keyserver', 'keyserver.ubuntu.com',
+           '--recv-keys', config['ppa-gpg-key']]
+    try:
+        subprocess.check_call(cmd)
+    except:
+        juju_log(MSG_ERROR, "Error importing repo key")
+
+
 def install(config):
-    charmhelpers.fetch.configure_sources(update=True)
-
     pkgs = [x for x in config.get('packages', '').split(' ') if x]
     pkgs.append('python-amqplib')
     pkgs.append('python-pip')
     pkgs.append('bzr')
 
+    if config['proxy-url']:
+        juju_info('Setting up apt with {}'.format(config['proxy-url']))
+        setup_apt_proxy(config)
+
+    if config['ppa-gpg-key']:
+        juju_info('Importing key for {}'.format(config['ppa-gpg-key']))
+        import_ppa_keys(config)
     juju_info('installing apt packages...')
+    charmhelpers.fetch.configure_sources(update=True)
     charmhelpers.fetch.apt_install(pkgs)
 
     pip_pkgs = [x for x in config.get('pip-packages', '').split(' ') if x]
@@ -129,14 +156,21 @@
     setuid {uid}
     setgid {gid}
     chdir {sdir}
-
+    {proxy}
     exec {main}
     ''')
+    proxy = ''
+    if config['proxy-url']:
+        url = config['proxy-url']
+        no_proxy = config['no-proxy']
+        proxy = ('env http_proxy={}\nenv https_proxy={}\n'
+        	 'env no_proxy={}\n'.format(url, url, no_proxy))
     params = {
         'main': config['main'],
         'sdir': _service_dir(config),
         'uid': config.get('uid', 'nobody'),
         'gid': config.get('gid', 'nogroup'),
+        'proxy': proxy,
     }
     path = '/etc/init/{}.conf'.format(_service_name(config))
     with tempfile.NamedTemporaryFile('w', delete=False) as f:
 
=== modified file 'charms/precise/restish/config.yaml'
--- charms/precise/restish/config.yaml	2014-03-10 22:25:00 +0000
+++ charms/precise/restish/config.yaml	2014-05-29 10:25:18 +0000
@@ -41,7 +41,14 @@
     type: string
     default: '/'
     description: The vhost in the rabbitMQ server.
-
+  proxy-url:
+    type: string
+    default: ""
+    description: The http proxy url for private clouds.
+  ppa-gpg-key:
+    type: string
+    default: ""
+    description: GPG key fingerprint for the PPA
   # required for the gunicorn charm:
   port:
     type: int
 
=== modified file 'charms/precise/restish/hooks/hooks.py'
--- charms/precise/restish/hooks/hooks.py	2014-03-10 22:25:00 +0000
+++ charms/precise/restish/hooks/hooks.py	2014-05-29 10:25:18 +0000
@@ -6,6 +6,7 @@
 import subprocess
 import sys
 import time
+import textwrap
 
 import charmhelpers.fetch
 
@@ -82,12 +83,40 @@
     subprocess.check_call(args)
 
 
+def setup_apt_proxy(config):
+    proxy_url = config['proxy-url']
+    apt_conf = textwrap.dedent("""\
+        Acquire::http::proxy "%s";
+        Acquire::https::proxy "%s";
+        Acquire::HTTP::PROXY "%s";
+        Acquire::HTTPS::PROXY "%s";
+        """ % (proxy_url, proxy_url, proxy_url, proxy_url))
+    with open('/etc/apt/apt.conf.d/95proxies', 'w') as f:
+        f.write(apt_conf)
+
+
+def import_ppa_keys(config):
+    cmd = ['apt-key', 'adv', '--keyserver', 'keyserver.ubuntu.com',
+           '--recv-keys', config['ppa-gpg-key']]
+    try:
+        subprocess.check_call(cmd)
+    except:
+        juju_log(MSG_ERROR, "Error importing repo key")
+
+
 def install(config):
     pkgs = [x for x in config['packages'].split(' ') if x]
     pkgs.append('bzr')
     pkgs.append('python-restish')
 
     juju_info('installing apt packages...')
+    if config['proxy-url']:
+        juju_info('Setting up apt with {}'.format(config['proxy-url']))
+        setup_apt_proxy(config)
+    if config['ppa-gpg-key']:
+        juju_info('Importing key for {}'.format(config['ppa-gpg-key']))
+        import_ppa_keys(config)
+
     charmhelpers.fetch.configure_sources(update=True)
     charmhelpers.fetch.apt_install(pkgs)
 
 
=== modified file 'juju-deployer/branch-source-builder.yaml.tmpl'
--- juju-deployer/branch-source-builder.yaml.tmpl	2014-03-14 10:37:28 +0000
+++ juju-deployer/branch-source-builder.yaml.tmpl	2014-05-29 10:25:18 +0000
@@ -8,6 +8,8 @@
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
+                proxy-url: ${CI_PRIVATE_HTTP_PROXY}
+                ppa-gpg-key: ${CI_PPA_GPG_KEY}
                 python_path: ./branch-source-builder:./ci-utils
                 # need non-default package python-amqplib for this service
                 packages: "python-webtest python-mock python-jinja2 python-amqplib"
@@ -29,6 +31,9 @@
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
+                proxy-url: ${CI_PRIVATE_HTTP_PROXY}
+                no-proxy: ${CI_PRIVATE_NO_PROXY}
+                ppa-gpg-key: ${CI_PPA_GPG_KEY}
                 unit-config: include-base64://configs/unit_config.yaml
                 packages: "dput python-dput python-swiftclient lazr.enum"
                 install_sources: |
 
=== modified file 'juju-deployer/deploy.py'
--- juju-deployer/deploy.py	2014-05-07 01:25:30 +0000
+++ juju-deployer/deploy.py	2014-05-29 10:25:18 +0000
@@ -77,6 +77,8 @@
         'CI_MASTER_PPA': 'ppa:ci-engineering-airline/ci-archive',
     }
     missing_required = []
+        if not os.environ.get('CI_PRIVATE_NO_PROXY'):
+            missing_required.append('CI_PRIVATE_NO_PROXY')
     for key in cheetah_vars:
         if key not in os.environ or len(os.environ[key]) == 0:
             default = cheetah_vars[key]
 
=== modified file 'juju-deployer/image-builder.yaml.tmpl'
--- juju-deployer/image-builder.yaml.tmpl	2014-03-14 10:37:28 +0000
+++ juju-deployer/image-builder.yaml.tmpl	2014-05-29 10:25:18 +0000
@@ -8,6 +8,8 @@
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
+                proxy-url: ${CI_PRIVATE_HTTP_PROXY}
+                ppa-gpg-key: ${CI_PPA_GPG_KEY}
                 python_path: ./image-builder:./ci-utils
                 packages: "python-webtest python-mock python-jinja2 python-amqplib"
                 json_status_path: api/v1/status
@@ -28,6 +30,9 @@
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
+                proxy-url: ${CI_PRIVATE_HTTP_PROXY}
+                no-proxy: ${CI_PRIVATE_NO_PROXY}
+                ppa-gpg-key: ${CI_PPA_GPG_KEY}
                 packages: "qemu-utils python-glanceclient python-swiftclient"
                 uid: root
                 unit-config: include-base64://configs/unit_config.yaml
 
=== modified file 'juju-deployer/lander.yaml.tmpl'
--- juju-deployer/lander.yaml.tmpl	2014-05-20 12:33:46 +0000
+++ juju-deployer/lander.yaml.tmpl	2014-05-29 10:25:18 +0000
@@ -7,6 +7,8 @@
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
+                proxy-url: ${CI_PRIVATE_HTTP_PROXY}
+                ppa-gpg-key: ${CI_PPA_GPG_KEY}
                 python_path: "./lander:./ci-utils"
                 # need non-default package python-amqplib for this service
                 packages: "python-webtest python-mock python-jinja2 python-jenkins"
 
=== modified file 'juju-deployer/test-runner.yaml.tmpl'
--- juju-deployer/test-runner.yaml.tmpl	2014-03-10 22:25:00 +0000
+++ juju-deployer/test-runner.yaml.tmpl	2014-05-29 10:25:18 +0000
@@ -13,6 +13,8 @@
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
+                proxy-url: ${CI_PRIVATE_HTTP_PROXY}
+                ppa-gpg-key: ${CI_PPA_GPG_KEY}
                 python_path: test_runner:ci-utils
                 json_status_path: api/v1/status
 
@@ -35,6 +37,9 @@
                 vcs: ${CI_CODE_SOURCE}
                 branch: ${CI_BRANCH}
                 tarball: ${CI_PAYLOAD_URL}
+                proxy-url: ${CI_PRIVATE_HTTP_PROXY}
+                no-proxy: ${CI_PRIVATE_NO_PROXY}
+                ppa-gpg-key: ${CI_PPA_GPG_KEY}
                 packages: "python-requests python-novaclient python-swiftclient"
                 unit-config: include-base64://configs/unit_config.yaml
                 uid: root