Merge bootstack-ops:jjo-p-o-e-creds into bootstack-ops:master

[JuanJo Ciarlante]

[jjo] add p-o-e charm instrumentation for its os-credentials settings + other fixes

[jjo] add p-o-e charm instrumentation for its os-credentials settings + other fixes

diff --git a/ops-bundle/devel/opsdeploy-prometheus.yaml b/ops-bundle/devel/opsdeploy-prometheus.yaml
index fef0e9e..485400c 100644
--- a/ops-bundle/devel/opsdeploy-prometheus.yaml
+++ b/ops-bundle/devel/opsdeploy-prometheus.yaml
@@ -5,7 +5,7 @@ nil:
     services: {}
 
 ops-bundle-prometheus:
-    inherits: [ secrets, bsdeploy ]
+    inherits: [ secrets, bsdeploy, poe-secrets ]
     series: xenial
     overrides:
         snap_proxy: "http://10.76.12.1:8000"
@@ -29,7 +29,7 @@ ops-bundle-prometheus:
         telegraf:
             charm: telegraf
             options:
-                hostname: '{unit}'
+                hostname: '{host}'
                 prometheus_output_port: default
 
 # WIP
@@ -39,9 +39,9 @@ ops-bundle-prometheus:
 #            to: *PROMSTACK_TO
 #
 #        # snap -> needs to be at metal or juju2 lxd
-#        prometheus-openstack-exporter:
-#            charm: prometheus-openstack-exporter
-#            to: *PROMSTACK_TO
+        prometheus-openstack-exporter:
+            charm: prometheus-openstack-exporter
+            to: *PROMSTACK_TO
 
     relations:
 # Connect the grafana -> prometheus -> telegraf stack
@@ -49,7 +49,8 @@ ops-bundle-prometheus:
         - ["telegraf:prometheus-client", "prometheus:target"]
 # Deploy telegraf on : os-hosts xor nova-compute, etc
         - ["telegraf:juju-info", "os-hosts"]
-# WIP
+# WIP: alertmanager
 #        - ["prometheus:alertmanager-service", "prometheus-alertmanager:alertmanager-service"]
+# WIP: prometheus-openstack-exporter
 #        - ["prometheus-openstack-exporter", "keystone"]
-#        - ["prometheus-openstack-exporter", "prometheus:target"]
+        - ["prometheus-openstack-exporter", "prometheus:target"]
diff --git a/ops-bundle/devel/prometheus.codetree b/ops-bundle/devel/prometheus.codetree
index 455ea24..aafc6e1 100644
--- a/ops-bundle/devel/prometheus.codetree
+++ b/ops-bundle/devel/prometheus.codetree
@@ -1,7 +1,6 @@
 # Reporting
-prometheus                 git+https://git.launchpad.net/prometheus-charm
-grafana                    git+https://git.launchpad.net/grafana-charm
-prometheus-alertmanager    git+https://git.launchpad.net/prometheus-alertmanager-charm
-telegraf                   git+https://git.launchpad.net/telegraf-charm
-#prometheus-openstack-exporter cs:~jjo/prometheus-openstack-exporter
+prometheus                 cs:~prometheus-charmers/prometheus
+grafana                    cs:~prometheus-charmers/grafana
+telegraf                   cs:telegraf
 prometheus-openstack-exporter git+https://git.launchpad.net/~jjo/bootstack-ops/+git/prometheus-openstack-exporter-charm
+#prometheus-openstack-exporter cs:~jjo/prometheus-openstack-exporter
diff --git a/ops-bundle/manifest-prometheus b/ops-bundle/manifest-prometheus
index 9bd0bcb..b044a49 100644
--- a/ops-bundle/manifest-prometheus
+++ b/ops-bundle/manifest-prometheus
@@ -3,12 +3,14 @@
 
 # Create bundle and codetree from running env
 script config=scripts/juju_bundle_export.sh OUT_BUNDLE=bsdeploy.yaml OUT_CODETREE=bsdeploy.codetree NAME=bsdeploy
+script config=scripts/prometheus_exporter_creds.py DEPLOY_BUNDLE=bsdeploy.yaml DEPLOY_NAME=bsdeploy OUT_FILE=poe-secrets.yaml
 
 # Collect Charms for running env
 collect config=bsdeploy.codetree
 
 # Collect Charms for added services
 collect config=prometheus.codetree
+# mojo collect --options config=prometheus.codetree
 
 # Some hints for peeking diffs
 #   mojo deploy-diff --options local=secrets,bsdeploy.yaml config=opsdeploy-prometheus.yaml target=ops-bundle-prometheus
@@ -18,4 +20,4 @@ collect config=prometheus.codetree
 # Deploy services
 # "nil" deploy just inherits from running, should not deploy anything
 #deploy local=secrets,bsdeploy.yaml config=opsdeploy-prometheus.yaml target=nil
-deploy local=secrets,bsdeploy.yaml config=opsdeploy-prometheus.yaml target=ops-bundle-prometheus
+deploy local=secrets,poe-secrets.yaml,bsdeploy.yaml config=opsdeploy-prometheus.yaml target=ops-bundle-prometheus
diff --git a/ops-bundle/production/opsdeploy-prometheus.yaml b/ops-bundle/production/opsdeploy-prometheus.yaml
index f9fd133..485400c 100644
--- a/ops-bundle/production/opsdeploy-prometheus.yaml
+++ b/ops-bundle/production/opsdeploy-prometheus.yaml
@@ -5,7 +5,7 @@ nil:
     services: {}
 
 ops-bundle-prometheus:
-    inherits: [ secrets, bsdeploy ]
+    inherits: [ secrets, bsdeploy, poe-secrets ]
     series: xenial
     overrides:
         snap_proxy: "http://10.76.12.1:8000"
@@ -15,23 +15,42 @@ ops-bundle-prometheus:
             charm: prometheus
             to: *PROMSTACK_TO
             options:
+
         # non snap -> could be LXC'd
         grafana:
             charm: grafana
             to: *PROMSTACK_TO
             options:
+                ## XXX: must go to secrets.yaml
+                admin_password: admin
                 install_file: https://grafanarel.s3.amazonaws.com/builds/grafana_4.1.2-1486989747_amd64.deb
 
         # subord -> will land on relation to ppal services (os-hosts or nova-compute, etc)
         telegraf:
             charm: telegraf
             options:
-                hostname: '{unit}'
+                hostname: '{host}'
                 prometheus_output_port: default
 
+# WIP
+#        # snap -> needs to be at metal or juju2 lxd
+#        prometheus-alertmanager:
+#            charm: prometheus-alertmanager
+#            to: *PROMSTACK_TO
+#
+#        # snap -> needs to be at metal or juju2 lxd
+        prometheus-openstack-exporter:
+            charm: prometheus-openstack-exporter
+            to: *PROMSTACK_TO
+
     relations:
 # Connect the grafana -> prometheus -> telegraf stack
         - ["prometheus:grafana-source", "grafana:grafana-source"]
         - ["telegraf:prometheus-client", "prometheus:target"]
 # Deploy telegraf on : os-hosts xor nova-compute, etc
         - ["telegraf:juju-info", "os-hosts"]
+# WIP: alertmanager
+#        - ["prometheus:alertmanager-service", "prometheus-alertmanager:alertmanager-service"]
+# WIP: prometheus-openstack-exporter
+#        - ["prometheus-openstack-exporter", "keystone"]
+        - ["prometheus-openstack-exporter", "prometheus:target"]
diff --git a/ops-bundle/production/prometheus.codetree b/ops-bundle/production/prometheus.codetree
index cd999eb..aafc6e1 100644
--- a/ops-bundle/production/prometheus.codetree
+++ b/ops-bundle/production/prometheus.codetree
@@ -1,7 +1,6 @@
 # Reporting
-prometheus                 git+https://git.launchpad.net/prometheus-charm
-grafana                    git+https://git.launchpad.net/grafana-charm
-#prometheus-alertmanager    git+https://git.launchpad.net/prometheus-alertmanager-charm
-telegraf                   git+https://git.launchpad.net/telegraf-charm
+prometheus                 cs:~prometheus-charmers/prometheus
+grafana                    cs:~prometheus-charmers/grafana
+telegraf                   cs:telegraf
+prometheus-openstack-exporter git+https://git.launchpad.net/~jjo/bootstack-ops/+git/prometheus-openstack-exporter-charm
 #prometheus-openstack-exporter cs:~jjo/prometheus-openstack-exporter
-#prometheus-openstack-exporter git+https://git.launchpad.net/~jjo/bootstack-ops/+git/prometheus-openstack-exporter-charm
diff --git a/ops-bundle/scripts/juju_bundle_export.sh b/ops-bundle/scripts/juju_bundle_export.sh
index 56e68ef..f0c8e93 100755
--- a/ops-bundle/scripts/juju_bundle_export.sh
+++ b/ops-bundle/scripts/juju_bundle_export.sh
@@ -8,5 +8,6 @@ PY_SCRIPT=${0%.sh}.py
 (set -x
 $PY_SCRIPT -o ${MOJO_LOCAL_DIR}/${OUT_BUNDLE} -c ${MOJO_SPEC_DIR}/${MOJO_STAGE}/${OUT_CODETREE} --name ${NAME}
 # HACK: above script will create xenial/CHARM while mojo likes ./CHARM 
+rm -rf ${MOJO_BUILD_DIR:?}/*
 ln -sfn . ${MOJO_BUILD_DIR}/xenial
 )
diff --git a/ops-bundle/scripts/prometheus_exporter_creds.py b/ops-bundle/scripts/prometheus_exporter_creds.py
new file mode 100755
index 0000000..8ccfb6e
--- /dev/null
+++ b/ops-bundle/scripts/prometheus_exporter_creds.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+# XXX(jjo) HACK to ease os-credentials settings for p-o-e charm,
+# crafted from bundle's keystone settings as saved from running env
+# NOTE this is very mojo specific, as it not only relies on MOJO_LOCAL_DIR
+#      but also wants args passed as env vars (mojo manifest line)
+import os
+import yaml
+import logging
+
+logging.basicConfig()
+logger = logging.getLogger()
+
+# The bundle file where running env was saved
+deploy_bundle = "{}/{}".format(
+    os.environ['MOJO_LOCAL_DIR'],
+    os.environ['DEPLOY_BUNDLE'])
+
+# Then deployment name inside the bundle, e.g. 'bsdeploy'
+deploy_name = os.environ['DEPLOY_NAME']
+
+# Output secrets.yaml file for p-o-e charm, it'll have
+# 'os-credentials' and 'ssl_ca' settings
+out_file = os.environ.get('OUT_FILE')
+if out_file:
+    out_file = "{}/{}".format(os.environ['MOJO_LOCAL_DIR'], out_file)
+
+# Output deployment name which contains p-o-e charm secrets
+out_name = os.environ.get('OUT_NAME', 'poe-secrets')
+auth_url = os.environ.get('OS_AUTH_URL')
+
+# Load DEPLOY_BUNDLE[DEPLOY_NAME]'s keystone config
+deploy = yaml.load(open(deploy_bundle).read())[deploy_name]
+ks_config = deploy['services']['keystone']['options']
+
+# Having OS_AUTH_URL from the running shell env would be great,
+# else try some educated guess
+if not auth_url:
+    logging.warning("No 'OS_AUTH_URL' in environment, guessing from keystone")
+    hostname = ks_config.get('os-public-hostname', ks_config.get('vip'))
+    proto = 'https' if ks_config.get('https-service-endpoints') else 'http'
+    auth_url = '{}://{}:5000/v2.0'.format(proto, hostname)
+
+logging.info(
+    "out_file: {}, out_name: {}, deploy_name: {}, auth_url: {}".format(
+        out_file, out_name, deploy_name, auth_url))
+
+# os-credentials must be a string'd YAML, i.e. can't directly be a python dict
+# (as yaml.dump would save it as a yaml dict itself, while juju only supports
+# "flat" non-structured settings)
+poe_settings = {
+    'os-credentials': '{{username: {}, password: {}, region_name: {}, auth_url: "{}" }}'.format(  # noqa
+            'Admin', ks_config['admin-password'],
+            ks_config['region'], auth_url)
+}
+
+# Only save ssl_ca if present in keystone
+ssl_ca = ks_config.get('ssl_ca')
+if ssl_ca:
+    poe_settings.update({'ssl_ca': ssl_ca})
+
+# Output mini-bundle with just p-o-e secrets, meant to be used
+# as part of local= mojo manifest line, then inherited from
+# prometheus.yaml main bundle
+out_content = {
+    out_name: {
+        'services': {
+            'prometheus-openstack-exporter': {
+                'options': poe_settings
+            }
+        }
+    }
+}
+
+# print to stdout if no OUT_FILE
+print_kw = {'file': open(out_file, 'w')} if out_file else {}
+print(yaml.dump(out_content, default_flow_style=False), **print_kw)