Merge bootstack-ops:juju-bundle-export-fixes into bootstack-ops:master

[Alvaro Uria]

fa8a3a6... by Alvaro Uria on 2017-07-12

juju_bundle_export.py exclude services (--exclude)

 * Add [--exclude APP_NAME] to filter the list of services and relations
   --exclude can be repeated multiple times
 * Add docstrings for a couple more internal methods (to gather charms
 interface for relation disambiguation
 * Make --fullstatus mutually exclusive with --charm or --exclude
 parameters.

[Alvaro Uria (aluria)]

I've also fixed DEPLOY_NAME default to "juju switch" output instead of "dummy-stage0" (which could be passed via "--name dummy-stage0").

20a3ff3... by Alvaro Uria on 2017-07-12

minor fix on parser.print_help()

[Brad Marshall (brad-marshall)]

This looks good, much better way of doing the exclude parameter than I had done. As per my comment, why is fullstatus mutually exclusive with exclude? I need both for prometheus deployments.

review: Needs Fixing

[Brad Marshall (brad-marshall)]

I can confirm that by removing the mutually exclusive check for fullstatus and exclude I can do the prometheus deployment.

d5e77cb... by Alvaro Uria on 2017-07-14

Fix: --fullstatus is only mutually exclusive with --charm

d24fe76... by Alvaro Uria on 2017-07-14

minor fix: remove comment about --exclude/--fullstatus being mutually exclusive

[Brad Marshall (brad-marshall)]

Looks good to me, +1

review: Approve

[Alvaro Uria (aluria)]

changes squashed into master

diff --git a/ops-bundle/scripts/juju_bundle_export.py b/ops-bundle/scripts/juju_bundle_export.py
index 8256cdf..9d3f27c 100755
--- a/ops-bundle/scripts/juju_bundle_export.py
+++ b/ops-bundle/scripts/juju_bundle_export.py
@@ -2,6 +2,7 @@
 # Standalone script to export running environment as a juju-deployer bundle
 #
 # Author: JuanJo Ciarlante <[e-mail]>
+# Modified: Alvaro Uria <[e-mail]>
 # License: GPLv3+
 #
 # Eg usage:
@@ -37,6 +38,10 @@ CHARM_DIR_MAP = {
 
 
 def charmstore_get(series, charm):
+    """charmstore_get(str, str) -> str
+
+    ie. charmstore_get('xenial', 'ubuntu') -> 'cs:xenial/ubuntu-10'
+    """
     cs_reply = requests.get('https://api.jujucharms.com/v4/search', params={
         'name': charm,
         'series': series,
@@ -64,6 +69,12 @@ class Bundle(object):
     # as juju2 doesn't have a relations dict, need to build it from
     # each service iteration
     def _get_rel_name(self, src, tgt):
+        """_get_rel_name(str, str) -> str
+
+        ['ceph:client', 'nova-compute:ceph'],
+          src == 'nova-compute' and tgt == 'ceph'
+          r == 'client'
+        """
         if not self.env_status:
             logger.warning("_get_rel_name called without initialized env_status")
             return None
@@ -74,6 +85,11 @@ class Bundle(object):
         return None
 
     def _load_rels(self, svc_name):
+        """_loads_rel(str) -> set((str, str), (str, str)...)
+
+        ['ceph:client', 'nova-compute:ceph'],
+          svc_name == 'nova-compute' -> r_name == 'ceph'
+        """
         rels = set()
         svc_rels = self.env_status['services'][svc_name].get(
             'relations', {})
@@ -85,6 +101,9 @@ class Bundle(object):
                 # Skip peer relations
                 if r_svc == svc_name:
                     continue
+                # if charm is excluded, skip it
+                elif self.args.exclude and r_svc in self.args.exclude:
+                    continue
                 rr_name = self._get_rel_name(svc_name, r_svc)
                 rels.add(
                     tuple(sorted([
@@ -93,45 +112,85 @@ class Bundle(object):
         return rels
 
     def export(self):
+        """Parses Juju FullStatus from apiserver (json format).
+           Stores juju-deployer services/relations into self.deploy
+
+        juju_version: 1
+          https://github.com/juju/juju/blob/1.25/apiserver/params/status.go#L64
+        juju_version: 2
+          https://github.com/juju/juju/blob/2.2/apiserver/params/status.go#L107
+        """
         self.env_status = self.env.status()
         services = {}
         relations = set()
-        subordto_key = [None, 'SubordinateTo', 'subordinate-to'][self.env.juju_version]
-        charm_key = [None, 'Charm', 'charm'][self.env.juju_version]
+        subordto_key = {1: 'SubordinateTo'}.get(self.env.juju_version,
+                                                'subordinate-to')
+        charm_key = {1: 'Charm'}.get(self.env.juju_version,
+                                     'charm')
         # print( yaml.dump(self.env_status.get('services'), default_flow_style=False))
         for service_name, service_val in self.env_status.get('services', {}).items():
+            # if charm is excluded, skip it
+            if self.args.exclude and service_name in self.args.exclude:
+                continue
+
             units = service_val.get('units', None)
             subord_to = service_val.get(subordto_key)
+
+            # skip subordinate charms for fake stage0 (minimal info)
+            if subord_to and not self.args.fullstatus:
+                continue
+
             # Subordinate units show no Units
             if units or subord_to:
                 charm = '-'.join(service_val[charm_key].split('-')[0:-1])
+
+                # if charm name specified, do not process unmatched ones
+                if self.args.charm_name and \
+                   self.args.charm_name not in charm:
+                    continue
+
                 service = {
                     'charm': charm
                 }
                 if units:
                     service['num_units'] = len(units)
-                if not self.args.minimal:
+
+                # List all service/application options
+                # not needed when generating a fake stage0 (minimal)
+                # needed to redeploy the environment
+                if self.args.fullstatus:
                     config = self.env.get_config(service_name)
                     options = {k: v['value'] for k, v in config.items()
                                if not v.get('default', False)}
                     if options:
                         service['options'] = options
+                    relations.update(self._load_rels(service_name))
+
                 services[service_name] = service
-            relations.update(self._load_rels(service_name))
-        relations = [list(rel) for rel in sorted(relations)]
 
-        self.deploy = {
-            'services': services,
-            'relations': relations,
-        }
+        if self.args.fullstatus:
+            relations = [list(rel) for rel in sorted(relations)]
+            self.deploy = {'services': services,
+                           'relations': relations}
+        else:
+            self.deploy = {'services': services}
+
         self.resolve_charm_local_urls()
         return (self.env_name, self.deploy, self.codetree)
 
     def init_local_files(self):
+        """Stores codetree file into self.local_charm_url_map dict
+           Initializes self.local_charms_dir
+        """
         self.local_charm_url_map = {}
         bootstack_dir = self.args.bootstackdir.format(env_name=self.env_name)
+
+        # XXX(aluria) Support mojo
+        # collect_file = 'collect'
+        collect_file = 'customer/configs/bootstack-charms.bzr'
         bs_codetree_file = os.path.join(bootstack_dir,
-                                        "customer/configs/bootstack-charms.bzr")
+                                        collect_file)
+
         if (os.path.exists(bs_codetree_file)):
             with open(bs_codetree_file) as bs_ct_f:
                 for line in bs_ct_f:
@@ -141,23 +200,32 @@ class Bundle(object):
                     if (len(key_val) != 2):
                         continue
                     self.local_charm_url_map.update({key_val[0]: key_val[1]})
-        self.local_charms_dir = None
-        local_dir = os.path.join(bootstack_dir, "charms")
+
+        local_dir = os.path.join(bootstack_dir, 'charms')
         if os.path.exists(local_dir):
             self.local_charms_dir = local_dir
+        else:
+            self.local_charms_dir = None
 
     def local_charm_dir(self, charm_dir):
-        if self.local_charms_dir and os.path.exists(
-                os.path.join(self.local_charms_dir, charm_dir)):
+        """Returns path to local charm
+        """
+        if self.local_charms_dir and \
+           os.path.exists(os.path.join(self.local_charms_dir,
+                                       charm_dir)):
             return os.path.join(self.local_charms_dir, charm_dir)
         return None
 
     def resolve_charm_local_urls(self):
+        """Parses codetree file
+           Rewrites charm reference if not found locally
+        """
         self.init_local_files()
         for service, service_val in self.deploy['services'].items():
             charm_url = service_val['charm']
             if charm_url.startswith('local:'):
-                service_val['charm'] = self.get_charm_local_url(service, charm_url)
+                service_val['charm'] = self.get_charm_local_url(service,
+                                                                charm_url)
 
     def get_charm_local_url(self, service, charm_url):
         """Return a line for codetree, in the form of
@@ -184,11 +252,13 @@ class Bundle(object):
                 charm, charm_dir).format(series=series)
 
         # Try to resolve local charm in this order:
-        for func, args in (
-            (self.local_charm_dir, [charm_dir]),
-            (self.local_charm_url_map.get, [charm_dir]),
-            (charmstore_get, [series, charm]),
-        ):
+        # 1) charm found on local charms folder
+        # 2) def found on codetree file
+        # 3) download from the charmstore
+        for func, args in ((self.local_charm_dir, [charm_dir]),
+                           (self.local_charm_url_map.get, [charm_dir]),
+                           (charmstore_get, [series, charm]),
+                          ):
             charm_url_new = func(*args)
             # If we're overriding the charm_url to point locally::
             # - point bundle to local charm dir
@@ -200,7 +270,11 @@ class Bundle(object):
         return charm_url
 
     def get_named_bundle(self):
-        deploy_name = self.args.deploy_name if self.args.deploy_name else self.env_name
+        if self.args.deploy_name:
+            deploy_name = self.args.deploy_name
+        else:
+            deploy_name = self.env_name
+
         named_bundle = ({deploy_name: self.deploy})
         return yaml.dump(named_bundle, default_flow_style=False)
 
@@ -216,35 +290,52 @@ def parse_args():
     parser = argparse.ArgumentParser(
         description='Export juju environment as a bundle')
     parser.add_argument('env_name', nargs='?',
-                        help='Environment name (if not default')
-    parser.add_argument('-o', dest='outfile', nargs='?',
-                        type=argparse.FileType('w'), default=sys.stdout)
-    parser.add_argument('-c', dest='codetreefile', nargs='?',
-                        type=argparse.FileType('w'), default=None)
+                        help='Environment name (if not default)')
     parser.add_argument('-b', dest='bootstackdir', nargs='?',
                         help='Local charms directory',
                         default='/home/jujumanage/{env_name}')
+    parser.add_argument('-c', dest='codetreefile', nargs='?',
+                        type=argparse.FileType('w'), default=None)
+    parser.add_argument('-o', dest='outfile', nargs='?',
+                        type=argparse.FileType('w'), default=sys.stdout)
     parser.add_argument('--name', dest='deploy_name', nargs='?',
                         help='Deployment name to save, instead of <JUJU_ENV>')
-    parser.add_argument('--minimal', dest='minimal', action='store_true',
-                        help='Do not include service "options"')
     parser.add_argument('--no-hacks', dest='no_hacks', action='store_true',
                         help='Do not apply bootstack naming hacks')
     parser.add_argument('--debug', action='store_true', default=False,
                         help='show debugging')
-    return parser.parse_args(sys.argv[1:])
+    parser.add_argument('--fullstatus', dest='fullstatus', action='store_true',
+                        help='Includes service "options", and relations')
+    parser.add_argument('--charm', dest='charm_name', nargs='?',
+                        help='Filter services by charm name used')
+    parser.add_argument('--exclude', action='append', default=[],
+                        help='Exclude services by their name')
+
+    # XXX(aluria) if "--charm ubuntu --fullstatus",
+    # services listed are only the ones using the "ubuntu" charm
+    # relations listed could be other than the ones using the
+    # "ubuntu" charm
+    result = parser.parse_args(sys.argv[1:])
+    if result.fullstatus and result.charm_name:
+        print('Error: --fullstatus is mutually exclusive with '
+              '--charm parameter.\n')
+        parser.print_help()
+        sys.exit(1)
+    return result
 
 
 def main():
     args = parse_args()
+
     if args.debug:
         logger.setLevel(logging.DEBUG)
+
     bundle = Bundle(args)
     bundle.export()
     print(bundle.get_named_bundle(), file=args.outfile)
+
     if args.codetreefile:
         print(bundle.get_codetree(), file=args.codetreefile)
 
-
 if __name__ == '__main__':
     main()