Merge lp:~harlowja/cloud-init/ug-cleanup-part-deuce into lp:~cloud-init-dev/cloud-init/trunk

[Joshua Harlow]

[lp:~harlowja/cloud-init/ug-cleanup-part-deuce]

673. By Joshua Harlow on 2012-09-28

Add a comment as to why distros can't be
imported without being renamed due to
previous usage of the attribute 'distros'

674. By Joshua Harlow on 2012-09-28

Make byobu more tolerant of the user not being
located and warn when it is not found + only
run the shell command when actual contents
exist to run.

675. By Joshua Harlow on 2012-09-28

Sync with head and fix conflicts.

=== modified file 'cloudinit/config/cc_byobu.py'
--- cloudinit/config/cc_byobu.py	2012-06-23 03:58:50 +0000
+++ cloudinit/config/cc_byobu.py	2012-09-28 21:23:20 +0000
@@ -18,12 +18,17 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
+from cloudinit import distros as ds
+
 from cloudinit import util
 
 distros = ['ubuntu', 'debian']
 
 
-def handle(name, cfg, _cloud, log, args):
+def handle(name, cfg, cloud, log, args):
     if len(args) != 0:
         value = args[0]
     else:
@@ -56,16 +61,20 @@
 
     shcmd = ""
     if mod_user:
-        user = util.get_cfg_option_str(cfg, "user", "ubuntu")
-        shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst)
-        shcmd += " || X=$(($X+1)); "
+        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ds.extract_default(users)
+        if not user:
+            log.warn(("No default byobu user provided, "
+                      "can not launch %s for the default user"), bl_inst)
+        else:
+            shcmd += " sudo -Hu \"%s\" byobu-launcher-%s" % (user, bl_inst)
+            shcmd += " || X=$(($X+1)); "
     if mod_sys:
         shcmd += "echo \"%s\" | debconf-set-selections" % dc_val
         shcmd += " && dpkg-reconfigure byobu --frontend=noninteractive"
         shcmd += " || X=$(($X+1)); "
 
-    cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")]
-
-    log.debug("Setting byobu to %s", value)
-
-    util.subp(cmd, capture=False)
+    if len(shcmd):
+        cmd = ["/bin/sh", "-c", "%s %s %s" % ("X=0;", shcmd, "exit $X")]
+        log.debug("Setting byobu to %s", value)
+        util.subp(cmd, capture=False)

=== modified file 'cloudinit/config/cc_set_passwords.py'
--- cloudinit/config/cc_set_passwords.py	2012-08-31 21:45:15 +0000
+++ cloudinit/config/cc_set_passwords.py	2012-09-28 21:23:20 +0000
@@ -20,6 +20,11 @@
 
 import sys
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
+from cloudinit import distros as ds
+
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -50,18 +55,10 @@
         expire = util.get_cfg_option_bool(chfg, 'expire', expire)
 
     if not plist and password:
-        user = cloud.distro.get_default_user()
-
-        if 'users' in cfg:
-
-            user_zero = cfg['users'][0]
-
-            if isinstance(user_zero, dict) and 'name' in user_zero:
-                user = user_zero['name']
-
+        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ds.extract_default(users)
         if user:
             plist = "%s:%s" % (user, password)
-
         else:
             log.warn("No default or defined user to change password for.")
 

=== modified file 'cloudinit/config/cc_ssh.py'
--- cloudinit/config/cc_ssh.py	2012-08-31 18:45:40 +0000
+++ cloudinit/config/cc_ssh.py	2012-09-28 21:23:20 +0000
@@ -21,6 +21,11 @@
 import glob
 import os
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
+from cloudinit import distros as ds
+
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -102,16 +107,8 @@
                                       " %s to file %s"), keytype, keyfile)
 
     try:
-        # TODO(utlemming): consolidate this stanza that occurs in:
-        # cc_ssh_import_id, cc_set_passwords, maybe cc_users_groups.py
-        user = cloud.distro.get_default_user()
-
-        if 'users' in cfg:
-            user_zero = cfg['users'][0]
-
-            if user_zero != "default":
-                user = user_zero
-
+        (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
+        (user, _user_config) = ds.extract_default(users)
         disable_root = util.get_cfg_option_bool(cfg, "disable_root", True)
         disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts",
                                                     DISABLE_ROOT_OPTS)

=== modified file 'cloudinit/config/cc_ssh_authkey_fingerprints.py'
--- cloudinit/config/cc_ssh_authkey_fingerprints.py	2012-09-28 20:35:53 +0000
+++ cloudinit/config/cc_ssh_authkey_fingerprints.py	2012-09-28 21:23:20 +0000
@@ -21,7 +21,11 @@
 
 from prettytable import PrettyTable
 
-from cloudinit import distros
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
+from cloudinit import distros as ds
+
 from cloudinit import ssh_util
 from cloudinit import util
 
@@ -41,8 +45,10 @@
         hasher = hashlib.new(hash_meth)
         hasher.update(base64.b64decode(b64_text))
         return ":".join(_split_hash(hasher.hexdigest()))
-    except TypeError:
+    except (TypeError, ValueError):
         # Raised when b64 not really b64...
+        # or when the hash type is not really
+        # a known/supported hash type...
         return '?'
 
 
@@ -92,8 +98,8 @@
 
     hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5")
     extract_func = ssh_util.extract_authorized_keys
-    (users, _groups) = distros.normalize_users_groups(cfg, cloud.distro)
+    (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
     for (user_name, _cfg) in users.items():
         (auth_key_fn, auth_key_entries) = extract_func(user_name, cloud.paths)
-        _pprint_key_entries(user_name, auth_key_fn, auth_key_entries,
-                            hash_meth)
+        _pprint_key_entries(user_name, auth_key_fn,
+                            auth_key_entries, hash_meth)

=== modified file 'cloudinit/config/cc_ssh_import_id.py'
--- cloudinit/config/cc_ssh_import_id.py	2012-08-31 19:36:05 +0000
+++ cloudinit/config/cc_ssh_import_id.py	2012-09-28 21:23:20 +0000
@@ -18,6 +18,11 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
+from cloudinit import distros as ds
+
 from cloudinit import util
 import pwd
 
@@ -39,33 +44,27 @@
         return
 
     # import for cloudinit created users
+    (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro)
     elist = []
-    for user_cfg in cfg['users']:
-        user = None
+    for (user, user_cfg) in users.items():
         import_ids = []
-
-        if isinstance(user_cfg, str) and user_cfg == "default":
-            user = cloud.distro.get_default_user()
-            if not user:
-                continue
-
+        if user_cfg['default']:
             import_ids = util.get_cfg_option_list(cfg, "ssh_import_id", [])
-
-        elif isinstance(user_cfg, dict):
-            user = None
-            import_ids = []
-
+        else:
             try:
-                user = user_cfg['name']
                 import_ids = user_cfg['ssh_import_id']
-
-                if import_ids and isinstance(import_ids, str):
-                    import_ids = str(import_ids).split(',')
-
             except:
-                log.debug("user %s is not configured for ssh_import" % user)
+                log.debug("User %s is not configured for ssh_import_id", user)
                 continue
 
+        try:
+            import_ids = util.uniq_merge(import_ids)
+            import_ids = [str(i) for i in import_ids]
+        except:
+            log.debug("User %s is not correctly configured for ssh_import_id",
+                      user)
+            continue
+
         if not len(import_ids):
             continue
 

=== modified file 'cloudinit/config/cc_users_groups.py'
--- cloudinit/config/cc_users_groups.py	2012-09-28 20:35:53 +0000
+++ cloudinit/config/cc_users_groups.py	2012-09-28 21:23:20 +0000
@@ -16,7 +16,10 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import distros
+# Ensure this is aliased to a name not 'distros'
+# since the module attribute 'distros'
+# is a list of distros that are supported, not a sub-module
+from cloudinit import distros as ds
 
 from cloudinit.settings import PER_INSTANCE
 
@@ -24,7 +27,7 @@
 
 
 def handle(name, cfg, cloud, _log, _args):
-    (users, groups) = distros.normalize_users_groups(cfg, cloud.distro)
+    (users, groups) = ds.normalize_users_groups(cfg, cloud.distro)
     for (name, members) in groups.items():
         cloud.distro.create_group(name, members)
     for (user, config) in users.items():

=== modified file 'cloudinit/distros/__init__.py'
--- cloudinit/distros/__init__.py	2012-09-28 19:38:48 +0000
+++ cloudinit/distros/__init__.py	2012-09-28 21:23:20 +0000
@@ -24,6 +24,7 @@
 from StringIO import StringIO
 
 import abc
+import itertools
 import os
 import re
 
@@ -186,8 +187,10 @@
             'gecos': "%s" % (self.default_user.title()),
             'sudo': "ALL=(ALL) NOPASSWD:ALL",
         }
-        if self.default_user_groups:
-            user_cfg['groups'] = _uniq_merge_sorted(self.default_user_groups)
+        def_groups = self.default_user_groups
+        if not def_groups:
+            def_groups = []
+        user_cfg['groups'] = util.uniq_merge_sorted(def_groups)
         return user_cfg
 
     def create_user(self, name, **kwargs):
@@ -398,39 +401,27 @@
     return default
 
 
-def _uniq_merge_sorted(*lists):
-    return sorted(_uniq_merge(*lists))
-
-
-def _uniq_merge(*lists):
-    combined_list = []
-    for a_list in lists:
-        if isinstance(a_list, (str, basestring)):
-            a_list = a_list.strip().split(",")
-        else:
-            a_list = [str(a) for a in a_list]
-        a_list = [a.strip() for a in a_list if a.strip()]
-        combined_list.extend(a_list)
-    uniq_list = []
-    for a in combined_list:
-        if a in uniq_list:
-            continue
-        else:
-            uniq_list.append(a)
-    return uniq_list
-
-
+# Normalizes a input group configuration
+# which can be a comma seperated list of
+# group names, or a list of group names
+# or a python dictionary of group names
+# to a list of members of that group.
+#
+# The output is a dictionary of group
+# names => members of that group which
+# is the standard form used in the rest
+# of cloud-init
 def _normalize_groups(grp_cfg):
     if isinstance(grp_cfg, (str, basestring, list)):
         c_grp_cfg = {}
-        for i in _uniq_merge(grp_cfg):
+        for i in util.uniq_merge(grp_cfg):
             c_grp_cfg[i] = []
         grp_cfg = c_grp_cfg
 
     groups = {}
     if isinstance(grp_cfg, (dict)):
         for (grp_name, grp_members) in grp_cfg.items():
-            groups[grp_name] = _uniq_merge_sorted(grp_members)
+            groups[grp_name] = util.uniq_merge_sorted(grp_members)
     else:
         raise TypeError(("Group config must be list, dict "
                          " or string types only and not %s") %
@@ -438,6 +429,21 @@
     return groups
 
 
+# Normalizes a input group configuration
+# which can be a comma seperated list of
+# user names, or a list of string user names
+# or a list of dictionaries with components
+# that define the user config + 'name' (if
+# a 'name' field does not exist then the
+# default user is assumed to 'own' that
+# configuration.
+#
+# The output is a dictionary of user
+# names => user config which is the standard 
+# form used in the rest of cloud-init. Note
+# the default user will have a special config
+# entry 'default' which will be marked as true
+# all other users will be marked as false.
 def _normalize_users(u_cfg, def_user_cfg=None):
     if isinstance(u_cfg, (dict)):
         ad_ucfg = []
@@ -453,12 +459,12 @@
                                  " for key %s") % (util.obj_name(v), k))
         u_cfg = ad_ucfg
     elif isinstance(u_cfg, (str, basestring)):
-        u_cfg = _uniq_merge_sorted(u_cfg)
+        u_cfg = util.uniq_merge_sorted(u_cfg)
 
     users = {}
     for user_config in u_cfg:
         if isinstance(user_config, (str, basestring, list)):
-            for u in _uniq_merge(user_config):
+            for u in util.uniq_merge(user_config):
                 if u and u not in users:
                     users[u] = {}
         elif isinstance(user_config, (dict)):
@@ -491,22 +497,59 @@
 
     # Fixup the default user into the real
     # default user name and replace it...
+    def_user = None
     if users and 'default' in users:
         def_config = users.pop('default')
         if def_user_cfg:
+            # Pickup what the default 'real name' is
+            # and any groups that are provided by the
+            # default config
             def_user = def_user_cfg.pop('name')
             def_groups = def_user_cfg.pop('groups', [])
+            # Pickup any config + groups for that user name
+            # that we may have previously extracted
             parsed_config = users.pop(def_user, {})
-            users_groups = _uniq_merge_sorted(parsed_config.get('groups', []),
-                                              def_groups)
+            parsed_groups = parsed_config.get('groups', [])
+            # Now merge our extracted groups with
+            # anything the default config provided
+            users_groups = util.uniq_merge_sorted(parsed_groups, def_groups)
             parsed_config['groups'] = ",".join(users_groups)
+            # The real config for the default user is the
+            # combination of the default user config provided
+            # by the distro, the default user config provided
+            # by the above merging for the user 'default' and
+            # then the parsed config from the user's 'real name'
+            # which does not have to be 'default' (but could be)
             users[def_user] = util.mergemanydict([def_user_cfg,
                                                   def_config,
                                                   parsed_config])
 
+    # Ensure that only the default user that we
+    # found (if any) is actually marked as being
+    # the default user
+    if users:
+        for (uname, uconfig) in users.items():
+            if def_user and uname == def_user:
+                uconfig['default'] = True
+            else:
+                uconfig['default'] = False
+
     return users
 
 
+# Normalizes a set of user/users and group
+# dictionary configuration into a useable
+# format that the rest of cloud-init can
+# understand using the default user
+# provided by the input distrobution (if any)
+# to allow for mapping of the 'default' user.
+#
+# Output is a dictionary of group names -> [member] (list)
+# and a dictionary of user names -> user configuration (dict)
+#
+# If 'user' exists it will override
+# the 'users'[0] entry (if a list) otherwise it will
+# just become an entry in the returned dictionary (no override)
 def normalize_users_groups(cfg, distro):
     if not cfg:
         cfg = {}
@@ -548,6 +591,33 @@
     return (users, groups)
 
 
+# Given a user dictionary config it will
+# extract the default user name and user config
+# from that list and return that tuple or
+# return (None, None) if no default user is
+# found in the given input
+def extract_default(users, default_name=None, default_config=None):
+    if not users:
+        users = {}
+
+    def safe_find(entry):
+        config = entry[1]
+        if not config or 'default' not in config:
+            return False
+        else:
+            return config['default']
+
+    tmp_users = users.items()
+    tmp_users = dict(itertools.ifilter(safe_find, tmp_users))
+    if not tmp_users:
+        return (default_name, default_config)
+    else:
+        name = tmp_users.keys()[0]
+        config = tmp_users[name]
+        config.pop('default', None)
+        return (name, config)
+
+
 def fetch(name):
     locs = importer.find_module(name,
                                 ['', __name__],

=== modified file 'cloudinit/util.py'
--- cloudinit/util.py	2012-09-28 20:31:50 +0000
+++ cloudinit/util.py	2012-09-28 21:23:20 +0000
@@ -249,6 +249,36 @@
             raise
 
 
+# Merges X lists, and then keeps the
+# unique ones, but orders by sort order
+# instead of by the original order
+def uniq_merge_sorted(*lists):
+    return sorted(uniq_merge(*lists))
+
+
+# Merges X lists and then iterates over those
+# and only keeps the unique items (order preserving)
+# and returns that merged and uniqued list as the
+# final result.
+#
+# Note: if any entry is a string it will be
+# split on commas and empty entries will be
+# evicted and merged in accordingly.
+def uniq_merge(*lists):
+    combined_list = []
+    for a_list in lists:
+        if isinstance(a_list, (str, basestring)):
+            a_list = a_list.strip().split(",")
+            # Kickout the empty ones
+            a_list = [a for a in a_list if len(a)]
+        combined_list.extend(a_list)
+    uniq_list = []
+    for i in combined_list:
+        if i not in uniq_list:
+            uniq_list.append(i)
+    return uniq_list
+
+
 def clean_filename(fn):
     for (k, v) in FN_REPLACEMENTS.iteritems():
         fn = fn.replace(k, v)

=== modified file 'tests/unittests/test_distros/test_user_data_normalize.py'
--- tests/unittests/test_distros/test_user_data_normalize.py	2012-09-28 01:30:01 +0000
+++ tests/unittests/test_distros/test_user_data_normalize.py	2012-09-28 21:23:20 +0000
@@ -119,8 +119,8 @@
         (users, _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
-        self.assertEquals({}, users['joe'])
-        self.assertEquals({}, users['bob'])
+        self.assertEquals({'default': False}, users['joe'])
+        self.assertEquals({'default': False}, users['bob'])
 
     def test_users_simple(self):
         distro = self._make_distro('ubuntu')
@@ -133,8 +133,8 @@
         (users, _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
-        self.assertEquals({}, users['joe'])
-        self.assertEquals({}, users['bob'])
+        self.assertEquals({'default': False}, users['joe'])
+        self.assertEquals({'default': False}, users['bob'])
 
     def test_users_old_user(self):
         distro = self._make_distro('ubuntu', 'bob')
@@ -179,8 +179,7 @@
         }
         (users, _groups) = self._norm(ug_cfg, distro)
         self.assertIn('zetta', users)
-        ug_cfg = {
-        }
+        ug_cfg = {}
         (users, groups) = self._norm(ug_cfg, distro)
         self.assertEquals({}, users)
         self.assertEquals({}, groups)
@@ -198,6 +197,35 @@
                           users['bob']['groups'])
         self.assertEquals(True,
                           users['bob']['blah'])
+        self.assertEquals(True,
+                          users['bob']['default'])
+
+    def test_users_dict_extract(self):
+        distro = self._make_distro('ubuntu', 'bob')
+        ug_cfg = {
+            'users': [
+                'default',
+            ],
+        }
+        (users, _groups) = self._norm(ug_cfg, distro)
+        self.assertIn('bob', users)
+        (name, config) = distros.extract_default(users)
+        self.assertEquals(name, 'bob')
+        expected_config = {}
+        def_config = None
+        try:
+            def_config = distro.get_default_user()
+        except NotImplementedError:
+            pass
+        if not def_config:
+            def_config = {}
+        expected_config.update(def_config)
+
+        # Ignore these for now
+        expected_config.pop('name', None)
+        expected_config.pop('groups', None)
+        config.pop('groups', None)
+        self.assertEquals(config, expected_config)
 
     def test_users_dict_default(self):
         distro = self._make_distro('ubuntu', 'bob')
@@ -210,6 +238,8 @@
         self.assertIn('bob', users)
         self.assertEquals(",".join(distro.get_default_user()['groups']),
                           users['bob']['groups'])
+        self.assertEquals(True,
+                          users['bob']['default'])
 
     def test_users_dict_trans(self):
         distro = self._make_distro('ubuntu')
@@ -223,8 +253,8 @@
         (users, _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
-        self.assertEquals({'tr_me': True}, users['joe'])
-        self.assertEquals({}, users['bob'])
+        self.assertEquals({'tr_me': True, 'default': False}, users['joe'])
+        self.assertEquals({'default': False}, users['bob'])
 
     def test_users_dict(self):
         distro = self._make_distro('ubuntu')
@@ -237,5 +267,5 @@
         (users, _groups) = self._norm(ug_cfg, distro)
         self.assertIn('joe', users)
         self.assertIn('bob', users)
-        self.assertEquals({}, users['joe'])
-        self.assertEquals({}, users['bob'])
+        self.assertEquals({'default': False}, users['joe'])
+        self.assertEquals({'default': False}, users['bob'])