Merge lp:~smoser/cloud-init/trunk.joyent-cleanup into lp:~cloud-init-dev/cloud-init/trunk

[Scott Moser]

smartos: separate client out of datasource and support network config

This does:
 * support reading 'sdc:nics' and converting it to internal
   'network-config' format.
 * change the datasource to be a local datasource (which is required
   to apply networking configuration).
   This change requires some rework of init to allow a datasource
   to be found locally but not resolve all userdata/metadata until
   networking is up. (that is coming)
 * support 'list', 'put', and 'delete' in the metadata client.
 * add 'get_json' to get known values that have json content.
 * add a simple main to datasource demonstrating how to read.
 * separate out the "legacy" behavior of having optionally
   base64 encoded data into JoyentMetadataLegacySerialClient.

Note, tests have been dropped that expected datasource.md[key] to be
base64 encoded. Now, the client takes care of decoding that. There
is not any real reason to store the base64 values in the datasource.

[lp:~smoser/cloud-init/trunk.joyent-cleanup]

1230. By Scott Moser on 2016-05-27

assertEquals

1231. By Scott Moser on 2016-05-27

return dict not None on get_config_obj

[Scott Moser (smoser)]

note, this does depend on https://code.launchpad.net/~smoser/cloud-init/trunk.fix-networking/ in order to have the datasource sanely declare that it is 'local'. Otherwise user-data would be pulled and such at local point (without networking).

[lp:~smoser/cloud-init/trunk.joyent-cleanup]

1232. By Scott Moser on 2016-05-27

assertEqual

1233. By Scott Moser on 2016-05-31

use constants for kvm and lx-brand

=== modified file 'cloudinit/sources/DataSourceSmartOS.py'
--- cloudinit/sources/DataSourceSmartOS.py	2016-04-12 16:57:50 +0000
+++ cloudinit/sources/DataSourceSmartOS.py	2016-05-31 17:14:35 +0000
@@ -32,13 +32,13 @@
 #       http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
 #       Comments with "@datadictionary" are snippets of the definition
 
+import base64
 import binascii
-import contextlib
+import json
 import os
 import random
 import re
 import socket
-import stat
 
 import serial
 
@@ -64,14 +64,36 @@
     'operator-script': ('sdc:operator-script', False),
 }
 
+SMARTOS_ATTRIB_JSON = {
+    # Cloud-init Key : (SmartOS Key known JSON)
+    'network-data': 'sdc:nics',
+}
+
+SMARTOS_ENV_LX_BRAND = "lx-brand"
+SMARTOS_ENV_KVM = "kvm"
+
 DS_NAME = 'SmartOS'
 DS_CFG_PATH = ['datasource', DS_NAME]
+NO_BASE64_DECODE = [
+    'iptables_disable',
+    'motd_sys_info',
+    'root_authorized_keys',
+    'sdc:datacenter_name',
+    'sdc:uuid'
+    'user-data',
+    'user-script',
+]
+
+METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock'
+SERIAL_DEVICE = '/dev/ttyS1'
+SERIAL_TIMEOUT = 60
+
 # BUILT-IN DATASOURCE CONFIGURATION
 #  The following is the built-in configuration. If the values
 #  are not set via the system configuration, then these default
 #  will be used:
 #    serial_device: which serial device to use for the meta-data
-#    seed_timeout: how long to wait on the device
+#    serial_timeout: how long to wait on the device
 #    no_base64_decode: values which are not base64 encoded and
 #            are fetched directly from SmartOS, not meta-data values
 #    base64_keys: meta-data keys that are delivered in base64
@@ -81,16 +103,10 @@
 #    fs_setup: describes how to format the ephemeral drive
 #
 BUILTIN_DS_CONFIG = {
-    'serial_device': '/dev/ttyS1',
-    'metadata_sockfile': '/native/.zonecontrol/metadata.sock',
-    'seed_timeout': 60,
-    'no_base64_decode': ['root_authorized_keys',
-                         'motd_sys_info',
-                         'iptables_disable',
-                         'user-data',
-                         'user-script',
-                         'sdc:datacenter_name',
-                         'sdc:uuid'],
+    'serial_device': SERIAL_DEVICE,
+    'serial_timeout': SERIAL_TIMEOUT,
+    'metadata_sockfile': METADATA_SOCKFILE,
+    'no_base64_decode': NO_BASE64_DECODE,
     'base64_keys': [],
     'base64_all': False,
     'disk_aliases': {'ephemeral0': '/dev/vdb'},
@@ -154,59 +170,40 @@
 
 
 class DataSourceSmartOS(sources.DataSource):
+    _unset = "_unset"
+    smartos_environ = _unset
+    md_client = _unset
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
-        self.is_smartdc = None
         self.ds_cfg = util.mergemanydict([
             self.ds_cfg,
             util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
             BUILTIN_DS_CONFIG])
 
         self.metadata = {}
+        self.network_data = None
+        self._network_config = None
 
-        # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
-        # report 'BrandZ virtual linux' as the kernel version
-        if os.uname()[3].lower() == 'brandz virtual linux':
-            LOG.debug("Host is SmartOS, guest in Zone")
-            self.is_smartdc = True
-            self.smartos_type = 'lx-brand'
-            self.cfg = {}
-            self.seed = self.ds_cfg.get("metadata_sockfile")
-        else:
-            self.is_smartdc = True
-            self.smartos_type = 'kvm'
-            self.seed = self.ds_cfg.get("serial_device")
-            self.cfg = BUILTIN_CLOUD_CONFIG
-            self.seed_timeout = self.ds_cfg.get("serial_timeout")
-        self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode')
-        self.b64_keys = self.ds_cfg.get('base64_keys')
-        self.b64_all = self.ds_cfg.get('base64_all')
         self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
+        self.smartos_type = None
+
+        self._init()
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
-        return "%s [seed=%s]" % (root, self.seed)
-
-    def _get_seed_file_object(self):
-        if not self.seed:
-            raise AttributeError("seed device is not set")
-
-        if self.smartos_type == 'lx-brand':
-            if not stat.S_ISSOCK(os.stat(self.seed).st_mode):
-                LOG.debug("Seed %s is not a socket", self.seed)
-                return None
-            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-            sock.connect(self.seed)
-            return sock.makefile('rwb')
-        else:
-            if not stat.S_ISCHR(os.stat(self.seed).st_mode):
-                LOG.debug("Seed %s is not a character device")
-                return None
-            ser = serial.Serial(self.seed, timeout=self.seed_timeout)
-            if not ser.isOpen():
-                raise SystemError("Unable to open %s" % self.seed)
-            return ser
-        return None
+        return "%s [client=%s]" % (root, self.md_client)
+
+    def _init(self):
+        if self.smartos_environ == self._unset:
+            self.smartos_type = get_smartos_environ()
+
+        if self.md_client == self._unset:
+            self.md_client = jmc_client_factory(
+                smartos_type=self.smartos_type,
+                metadata_sockfile=self.ds_cfg['metadata_sockfile'],
+                serial_device=self.ds_cfg['serial_device'],
+                serial_timeout=self.ds_cfg['serial_timeout'])
 
     def _set_provisioned(self):
         '''Mark the instance provisioning state as successful.
@@ -225,50 +222,26 @@
                       '/'.join([svc_path, 'provision_success']))
 
     def get_data(self):
+        self._init()
+
         md = {}
         ud = ""
 
-        if not device_exists(self.seed):
-            LOG.debug("No metadata device '%s' found for SmartOS datasource",
-                      self.seed)
-            return False
-
-        uname_arch = os.uname()[4]
-        if uname_arch.startswith("arm") or uname_arch == "aarch64":
-            # Disabling because dmidcode in dmi_data() crashes kvm process
-            LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)")
-            return False
-
-        # SDC KVM instances will provide dmi data, LX-brand does not
-        if self.smartos_type == 'kvm':
-            dmi_info = dmi_data()
-            if dmi_info is None:
-                LOG.debug("No dmidata utility found")
-                return False
-
-            system_type = dmi_info
-            if 'smartdc' not in system_type.lower():
-                LOG.debug("Host is not on SmartOS. system_type=%s",
-                          system_type)
-                return False
-            LOG.debug("Host is SmartOS, guest in KVM")
-
-        seed_obj = self._get_seed_file_object()
-        if seed_obj is None:
-            LOG.debug('Seed file object not found.')
-            return False
-        with contextlib.closing(seed_obj) as seed:
-            b64_keys = self.query('base64_keys', seed, strip=True, b64=False)
-            if b64_keys is not None:
-                self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
-
-            b64_all = self.query('base64_all', seed, strip=True, b64=False)
-            if b64_all is not None:
-                self.b64_all = util.is_true(b64_all)
-
-            for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
-                smartos_noun, strip = attribute
-                md[ci_noun] = self.query(smartos_noun, seed, strip=strip)
+        if not self.smartos_type:
+            LOG.debug("Not running on smartos")
+            return False
+
+        if not self.md_client.exists():
+            LOG.debug("No metadata device '%r' found for SmartOS datasource",
+                      self.md_client)
+            return False
+
+        for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
+            smartos_noun, strip = attribute
+            md[ci_noun] = self.md_client.get(smartos_noun, strip=strip)
+
+        for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items():
+            md[ci_noun] = self.md_client.get_json(smartos_noun)
 
         # @datadictionary: This key may contain a program that is written
         # to a file in the filesystem of the guest on each boot and then
@@ -318,6 +291,7 @@
         self.metadata = util.mergemanydict([md, self.metadata])
         self.userdata_raw = ud
         self.vendordata_raw = md['vendor-data']
+        self.network_data = md['network-data']
 
         self._set_provisioned()
         return True
@@ -326,69 +300,20 @@
         return self.ds_cfg['disk_aliases'].get(name)
 
     def get_config_obj(self):
-        return self.cfg
+        if self.smartos_type == SMARTOS_ENV_KVM:
+            return BUILTIN_CLOUD_CONFIG
+        return {}
 
     def get_instance_id(self):
         return self.metadata['instance-id']
 
-    def query(self, noun, seed_file, strip=False, default=None, b64=None):
-        if b64 is None:
-            if noun in self.smartos_no_base64:
-                b64 = False
-            elif self.b64_all or noun in self.b64_keys:
-                b64 = True
-
-        return self._query_data(noun, seed_file, strip=strip,
-                                default=default, b64=b64)
-
-    def _query_data(self, noun, seed_file, strip=False,
-                    default=None, b64=None):
-        """Makes a request via "GET <NOUN>"
-
-           In the response, the first line is the status, while subsequent
-           lines are is the value. A blank line with a "." is used to
-           indicate end of response.
-
-           If the response is expected to be base64 encoded, then set
-           b64encoded to true. Unfortantely, there is no way to know if
-           something is 100% encoded, so this method relies on being told
-           if the data is base64 or not.
-        """
-
-        if not noun:
-            return False
-
-        response = JoyentMetadataClient(seed_file).get_metadata(noun)
-
-        if response is None:
-            return default
-
-        if b64 is None:
-            b64 = self._query_data('b64-%s' % noun, seed_file, b64=False,
-                                   default=False, strip=True)
-            b64 = util.is_true(b64)
-
-        resp = None
-        if b64 or strip:
-            resp = "".join(response).rstrip()
-        else:
-            resp = "".join(response)
-
-        if b64:
-            try:
-                return util.b64d(resp)
-            # Bogus input produces different errors in Python 2 and 3;
-            # catch both.
-            except (TypeError, binascii.Error):
-                LOG.warn("Failed base64 decoding key '%s'", noun)
-                return resp
-
-        return resp
-
-
-def device_exists(device):
-    """Symplistic method to determine if the device exists or not"""
-    return os.path.exists(device)
+    @property
+    def network_config(self):
+        if self._network_config is None:
+            if self.network_data is not None:
+                self._network_config = (
+                    convert_smartos_network_data(self.network_data))
+        return self._network_config
 
 
 class JoyentMetadataFetchException(Exception):
@@ -407,8 +332,11 @@
         r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)'
         r'( (?P<payload>.+))?)')
 
-    def __init__(self, metasource):
-        self.metasource = metasource
+    def __init__(self, smartos_type=None, fp=None):
+        if smartos_type is None:
+            smartos_type = get_smartos_environ()
+        self.smartos_type = smartos_type
+        self.fp = fp
 
     def _checksum(self, body):
         return '{0:08x}'.format(
@@ -436,37 +364,227 @@
         LOG.debug('Value "%s" found.', value)
         return value
 
-    def get_metadata(self, metadata_key):
-        LOG.debug('Fetching metadata key "%s"...', metadata_key)
+    def request(self, rtype, param=None):
         request_id = '{0:08x}'.format(random.randint(0, 0xffffffff))
-        message_body = '{0} GET {1}'.format(request_id,
-                                            util.b64e(metadata_key))
+        message_body = ' '.join((request_id, rtype,))
+        if param:
+            message_body += ' ' + base64.b64encode(param.encode()).decode()
         msg = 'V2 {0} {1} {2}\n'.format(
             len(message_body), self._checksum(message_body), message_body)
         LOG.debug('Writing "%s" to metadata transport.', msg)
-        self.metasource.write(msg.encode('ascii'))
-        self.metasource.flush()
+
+        need_close = False
+        if not self.fp:
+            self.open_transport()
+            need_close = True
+
+        self.fp.write(msg.encode('ascii'))
+        self.fp.flush()
 
         response = bytearray()
-        response.extend(self.metasource.read(1))
+        response.extend(self.fp.read(1))
         while response[-1:] != b'\n':
-            response.extend(self.metasource.read(1))
+            response.extend(self.fp.read(1))
+
+        if need_close:
+            self.close_transport()
+
         response = response.rstrip().decode('ascii')
         LOG.debug('Read "%s" from metadata transport.', response)
 
         if 'SUCCESS' not in response:
             return None
 
-        return self._get_value_from_frame(request_id, response)
-
-
-def dmi_data():
-    sys_type = util.read_dmi_data("system-product-name")
-
-    if not sys_type:
-        return None
-
-    return sys_type
+        value = self._get_value_from_frame(request_id, response)
+        return value
+
+    def get(self, key, default=None, strip=False):
+        result = self.request(rtype='GET', param=key)
+        if result is None:
+            return default
+        if result and strip:
+            result = result.strip()
+        return result
+
+    def get_json(self, key, default=None):
+        result = self.get(key, default=default)
+        if result is None:
+            return default
+        return json.loads(result)
+
+    def list(self):
+        result = self.request(rtype='KEYS')
+        if result:
+            result = result.split('\n')
+        return result
+
+    def put(self, key, val):
+        param = b' '.join([base64.b64encode(i.encode())
+                           for i in (key, val)]).decode()
+        return self.request(rtype='PUT', param=param)
+
+    def delete(self, key):
+        return self.request(rtype='DELETE', param=key)
+
+    def close_transport(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def __enter__(self):
+        if self.fp:
+            return self
+        self.open_transport()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close_transport()
+        return
+
+    def open_transport(self):
+        raise NotImplementedError
+
+
+class JoyentMetadataSocketClient(JoyentMetadataClient):
+    def __init__(self, socketpath):
+        self.socketpath = socketpath
+
+    def open_transport(self):
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.connect(self.socketpath)
+        self.fp = sock.makefile('rwb')
+
+    def exists(self):
+        return os.path.exists(self.socketpath)
+
+    def __repr__(self):
+        return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath)
+
+
+class JoyentMetadataSerialClient(JoyentMetadataClient):
+    def __init__(self, device, timeout=10, smartos_type=None):
+        super(JoyentMetadataSerialClient, self).__init__(smartos_type)
+        self.device = device
+        self.timeout = timeout
+
+    def exists(self):
+        return os.path.exists(self.device)
+
+    def open_transport(self):
+        ser = serial.Serial(self.device, timeout=self.timeout)
+        if not ser.isOpen():
+            raise SystemError("Unable to open %s" % self.device)
+        self.fp = ser
+
+    def __repr__(self):
+        return "%s(device=%s, timeout=%s)" % (
+            self.__class__.__name__, self.device, self.timeout)
+
+
+class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient):
+    """V1 of the protocol was not safe for all values.
+    Thus, we allowed the user to pass values in as base64 encoded.
+    Users may still reasonably expect to be able to send base64 data
+    and have it transparently decoded.  So even though the V2 format is
+    now used, and is safe (using base64 itself), we keep legacy support.
+
+    The way for a user to do this was:
+      a.) specify 'base64_keys' key whose value is a comma delimited
+          list of keys that were base64 encoded.
+      b.) base64_all: string interpreted as a boolean that indicates
+          if all keys are base64 encoded.
+      c.) set a key named b64-<keyname> with a boolean indicating that
+          <keyname> is base64 encoded."""
+
+    def __init__(self, device, timeout=10, smartos_type=None):
+        s = super(JoyentMetadataLegacySerialClient, self)
+        s.__init__(device, timeout, smartos_type)
+        self.base64_keys = None
+        self.base64_all = None
+
+    def _init_base64_keys(self, reset=False):
+        if reset:
+            self.base64_keys = None
+            self.base64_all = None
+
+        keys = None
+        if self.base64_all is None:
+            keys = self.list()
+            if 'base64_all' in keys:
+                self.base64_all = util.is_true(self._get("base64_all"))
+            else:
+                self.base64_all = False
+
+        if self.base64_all:
+            # short circuit if base64_all is true
+            return
+
+        if self.base64_keys is None:
+            if keys is None:
+                keys = self.list()
+            b64_keys = set()
+            if 'base64_keys' in keys:
+                b64_keys = set(self._get("base64_keys").split(","))
+
+            # now add any b64-<keyname> that has a true value
+            for key in [k[3:] for k in keys if k.startswith("b64-")]:
+                if util.is_true(self._get(key)):
+                    b64_keys.add(key)
+                else:
+                    if key in b64_keys:
+                        b64_keys.remove(key)
+
+            self.base64_keys = b64_keys
+
+    def _get(self, key, default=None, strip=False):
+        return (super(JoyentMetadataLegacySerialClient, self).
+                get(key, default=default, strip=strip))
+
+    def is_b64_encoded(self, key, reset=False):
+        if key in NO_BASE64_DECODE:
+            return False
+
+        self._init_base64_keys(reset=reset)
+        if self.base64_all:
+            return True
+
+        return key in self.base64_keys
+
+    def get(self, key, default=None, strip=False):
+        mdefault = object()
+        val = self._get(key, strip=False, default=mdefault)
+        if val is mdefault:
+            return default
+
+        if self.is_b64_encoded(key):
+            try:
+                val = base64.b64decode(val.encode()).decode()
+            # Bogus input produces different errors in Python 2 and 3
+            except (TypeError, binascii.Error):
+                LOG.warn("Failed base64 decoding key '%s': %s", key, val)
+
+        if strip:
+            val = val.strip()
+
+        return val
+
+
+def jmc_client_factory(
+        smartos_type=None, metadata_sockfile=METADATA_SOCKFILE,
+        serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT,
+        uname_version=None):
+
+    if smartos_type is None:
+        smartos_type = get_smartos_environ(uname_version)
+
+    if smartos_type == SMARTOS_ENV_KVM:
+        return JoyentMetadataLegacySerialClient(
+            device=serial_device, timeout=serial_timeout,
+            smartos_type=smartos_type)
+    elif smartos_type == SMARTOS_ENV_LX_BRAND:
+        return JoyentMetadataSocketClient(socketpath=metadata_sockfile)
+
+    raise ValueError("Unknown value for smartos_type: %s" % smartos_type)
 
 
 def write_boot_content(content, content_f, link=None, shebang=False,
@@ -522,15 +640,138 @@
                 util.ensure_dir(os.path.dirname(link))
                 os.symlink(content_f, link)
         except IOError as e:
-            util.logexc(LOG, "failed establishing content link", e)
+            util.logexc(LOG, "failed establishing content link: %s", e)
+
+
+def get_smartos_environ(uname_version=None, product_name=None,
+                        uname_arch=None):
+    uname = os.uname()
+    if uname_arch is None:
+        uname_arch = uname[4]
+
+    if uname_arch.startswith("arm") or uname_arch == "aarch64":
+        return None
+
+    # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
+    # report 'BrandZ virtual linux' as the kernel version
+    if uname_version is None:
+        uname_version = uname[3]
+    if uname_version.lower() == 'brandz virtual linux':
+        return SMARTOS_ENV_LX_BRAND
+
+    if product_name is None:
+        system_type = util.read_dmi_data("system-product-name")
+    else:
+        system_type = product_name
+
+    if system_type and 'smartdc' in system_type.lower():
+        return SMARTOS_ENV_KVM
+
+    return None
+
+
+# Covert SMARTOS 'sdc:nics' data to network_config yaml
+def convert_smartos_network_data(network_data=None):
+    """Return a dictionary of network_config by parsing provided
+       SMARTOS sdc:nics configuration data
+
+    sdc:nics data is a dictionary of properties of a nic and the ip
+    configuration desired.  Additional nic dictionaries are appended
+    to the list.
+
+    Converting the format is straightforward though it does include
+    duplicate information as well as data which appears to be relevant
+    to the hostOS rather than the guest.
+
+    For each entry in the nics list returned from query sdc:nics, we
+    create a type: physical entry, and extract the interface properties:
+    'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'.  The remaining
+    keys are related to ip configuration.  For each ip in the 'ips' list
+    we create a subnet entry under 'subnets' pairing the ip to a one in
+    the 'gateways' list.
+    """
+
+    valid_keys = {
+        'physical': [
+            'mac_address',
+            'mtu',
+            'name',
+            'params',
+            'subnets',
+            'type',
+        ],
+        'subnet': [
+            'address',
+            'broadcast',
+            'dns_nameservers',
+            'dns_search',
+            'gateway',
+            'metric',
+            'netmask',
+            'pointopoint',
+            'routes',
+            'scope',
+            'type',
+        ],
+    }
+
+    config = []
+    for nic in network_data:
+        cfg = {k: v for k, v in nic.items()
+               if k in valid_keys['physical']}
+        cfg.update({
+            'type': 'physical',
+            'name': nic['interface']})
+        if 'mac' in nic:
+            cfg.update({'mac_address': nic['mac']})
+
+        subnets = []
+        for ip, gw in zip(nic['ips'], nic['gateways']):
+            subnet = {k: v for k, v in nic.items()
+                      if k in valid_keys['subnet']}
+            subnet.update({
+                'type': 'static',
+                'address': ip,
+                'gateway': gw,
+            })
+            subnets.append(subnet)
+        cfg.update({'subnets': subnets})
+        config.append(cfg)
+
+    return {'version': 1, 'config': config}
 
 
 # Used to match classes to dependencies
 datasources = [
-    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )),
 ]
 
 
 # Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import sys
+    jmc = jmc_client_factory()
+    if len(sys.argv) == 1:
+        keys = (list(SMARTOS_ATTRIB_JSON.keys()) +
+                list(SMARTOS_ATTRIB_MAP.keys()))
+    else:
+        keys = sys.argv[1:]
+
+    data = {}
+    for key in keys:
+        if key in SMARTOS_ATTRIB_JSON:
+            keyname = SMARTOS_ATTRIB_JSON[key]
+            data[key] = jmc.get_json(keyname)
+        else:
+            if key in SMARTOS_ATTRIB_MAP:
+                keyname, strip = SMARTOS_ATTRIB_MAP[key]
+            else:
+                keyname, strip = (key, False)
+            val = jmc.get(keyname, strip=strip)
+            data[key] = jmc.get(keyname, strip=strip)
+
+    print(json.dumps(data, indent=1))

=== modified file 'tests/unittests/test_datasource/test_smartos.py'
--- tests/unittests/test_datasource/test_smartos.py	2016-05-12 20:43:11 +0000
+++ tests/unittests/test_datasource/test_smartos.py	2016-05-31 17:14:35 +0000
@@ -25,6 +25,7 @@
 from __future__ import print_function
 
 from binascii import crc32
+import json
 import os
 import os.path
 import re
@@ -40,12 +41,49 @@
 from cloudinit.sources import DataSourceSmartOS
 from cloudinit.util import b64e
 
-from .. import helpers
+from ..helpers import mock, FilesystemMockingTestCase, TestCase
 
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+SDC_NICS = json.loads("""
+[
+    {
+        "nic_tag": "external",
+        "primary": true,
+        "mtu": 1500,
+        "model": "virtio",
+        "gateway": "8.12.42.1",
+        "netmask": "255.255.255.0",
+        "ip": "8.12.42.102",
+        "network_uuid": "992fc7ce-6aac-4b74-aed6-7b9d2c6c0bfe",
+        "gateways": [
+            "8.12.42.1"
+        ],
+        "vlan_id": 324,
+        "mac": "90:b8:d0:f5:e4:f5",
+        "interface": "net0",
+        "ips": [
+            "8.12.42.102/24"
+        ]
+    },
+    {
+        "nic_tag": "sdc_overlay/16187209",
+        "gateway": "192.168.128.1",
+        "model": "virtio",
+        "mac": "90:b8:d0:a5:ff:cd",
+        "netmask": "255.255.252.0",
+        "ip": "192.168.128.93",
+        "network_uuid": "4cad71da-09bc-452b-986d-03562a03a0a9",
+        "gateways": [
+            "192.168.128.1"
+        ],
+        "vlan_id": 2,
+        "mtu": 8500,
+        "interface": "net1",
+        "ips": [
+            "192.168.128.93/22"
+        ]
+    }
+]
+""")
 
 MOCK_RETURNS = {
     'hostname': 'test-host',
@@ -60,79 +98,66 @@
     'sdc:vendor-data': '\n'.join(['VENDOR_DATA', '']),
     'user-data': '\n'.join(['something', '']),
     'user-script': '\n'.join(['/bin/true', '']),
+    'sdc:nics': json.dumps(SDC_NICS),
 }
 
 DMI_DATA_RETURN = 'smartdc'
 
 
-def get_mock_client(mockdata):
-    class MockMetadataClient(object):
-
-        def __init__(self, serial):
-            pass
-
-        def get_metadata(self, metadata_key):
-            return mockdata.get(metadata_key)
-    return MockMetadataClient
-
-
-class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
+class PsuedoJoyentClient(object):
+    def __init__(self, data=None):
+        if data is None:
+            data = MOCK_RETURNS.copy()
+        self.data = data
+        return
+
+    def get(self, key, default=None, strip=False):
+        if key in self.data:
+            r = self.data[key]
+            if strip:
+                r = r.strip()
+        else:
+            r = default
+        return r
+
+    def get_json(self, key, default=None):
+        result = self.get(key, default=default)
+        if result is None:
+            return default
+        return json.loads(result)
+
+    def exists(self):
+        return True
+
+
+class TestSmartOSDataSource(FilesystemMockingTestCase):
     def setUp(self):
         super(TestSmartOSDataSource, self).setUp()
 
+        dsmos = 'cloudinit.sources.DataSourceSmartOS'
+        patcher = mock.patch(dsmos + ".jmc_client_factory")
+        self.jmc_cfact = patcher.start()
+        self.addCleanup(patcher.stop)
+        patcher = mock.patch(dsmos + ".get_smartos_environ")
+        self.get_smartos_environ = patcher.start()
+        self.addCleanup(patcher.stop)
+
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
+        self.paths = c_helpers.Paths({'cloud_dir': self.tmp})
+
         self.legacy_user_d = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, self.legacy_user_d)
-
-        # If you should want to watch the logs...
-        self._log = None
-        self._log_file = None
-        self._log_handler = None
-
-        # patch cloud_dir, so our 'seed_dir' is guaranteed empty
-        self.paths = c_helpers.Paths({'cloud_dir': self.tmp})
-
-        self.unapply = []
-        super(TestSmartOSDataSource, self).setUp()
+        self.orig_lud = DataSourceSmartOS.LEGACY_USER_D
+        DataSourceSmartOS.LEGACY_USER_D = self.legacy_user_d
 
     def tearDown(self):
-        helpers.FilesystemMockingTestCase.tearDown(self)
-        if self._log_handler and self._log:
-            self._log.removeHandler(self._log_handler)
-        apply_patches([i for i in reversed(self.unapply)])
+        DataSourceSmartOS.LEGACY_USER_D = self.orig_lud
         super(TestSmartOSDataSource, self).tearDown()
 
-    def _patchIn(self, root):
-        self.restore()
-        self.patchOS(root)
-        self.patchUtils(root)
-
-    def apply_patches(self, patches):
-        ret = apply_patches(patches)
-        self.unapply += ret
-
-    def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None,
-                is_lxbrand=False):
-        mod = DataSourceSmartOS
-
-        if mockdata is None:
-            mockdata = MOCK_RETURNS
-
-        if dmi_data is None:
-            dmi_data = DMI_DATA_RETURN
-
-        def _dmi_data():
-            return dmi_data
-
-        def _os_uname():
-            if not is_lxbrand:
-                # LP: #1243287. tests assume this runs, but running test on
-                # arm would cause them all to fail.
-                return ('LINUX', 'NODENAME', 'RELEASE', 'VERSION', 'x86_64')
-            else:
-                return ('LINUX', 'NODENAME', 'RELEASE', 'BRANDZ VIRTUAL LINUX',
-                        'X86_64')
+    def _get_ds(self, mockdata=None, mode=DataSourceSmartOS.SMARTOS_ENV_KVM,
+                sys_cfg=None, ds_cfg=None):
+        self.jmc_cfact.return_value = PsuedoJoyentClient(mockdata)
+        self.get_smartos_environ.return_value = mode
 
         if sys_cfg is None:
             sys_cfg = {}
@@ -141,44 +166,8 @@
             sys_cfg['datasource'] = sys_cfg.get('datasource', {})
             sys_cfg['datasource']['SmartOS'] = ds_cfg
 
-        self.apply_patches([(mod, 'LEGACY_USER_D', self.legacy_user_d)])
-        self.apply_patches([
-            (mod, 'JoyentMetadataClient', get_mock_client(mockdata))])
-        self.apply_patches([(mod, 'dmi_data', _dmi_data)])
-        self.apply_patches([(os, 'uname', _os_uname)])
-        self.apply_patches([(mod, 'device_exists', lambda d: True)])
-        dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None,
-                                     paths=self.paths)
-        self.apply_patches([(dsrc, '_get_seed_file_object', mock.MagicMock())])
-        return dsrc
-
-    def test_seed(self):
-        # default seed should be /dev/ttyS1
-        dsrc = self._get_ds()
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual('kvm', dsrc.smartos_type)
-        self.assertEqual('/dev/ttyS1', dsrc.seed)
-
-    def test_seed_lxbrand(self):
-        # default seed should be /dev/ttyS1
-        dsrc = self._get_ds(is_lxbrand=True)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual('lx-brand', dsrc.smartos_type)
-        self.assertEqual('/native/.zonecontrol/metadata.sock', dsrc.seed)
-
-    def test_issmartdc(self):
-        dsrc = self._get_ds()
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertTrue(dsrc.is_smartdc)
-
-    def test_issmartdc_lxbrand(self):
-        dsrc = self._get_ds(is_lxbrand=True)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertTrue(dsrc.is_smartdc)
+        return DataSourceSmartOS.DataSourceSmartOS(
+            sys_cfg, distro=None, paths=self.paths)
 
     def test_no_base64(self):
         ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True}
@@ -214,58 +203,6 @@
         self.assertEqual(MOCK_RETURNS['hostname'],
                          dsrc.metadata['local-hostname'])
 
-    def test_base64_all(self):
-        # metadata provided base64_all of true
-        my_returns = MOCK_RETURNS.copy()
-        my_returns['base64_all'] = "true"
-        for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64e(my_returns[k])
-
-        dsrc = self._get_ds(mockdata=my_returns)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual(MOCK_RETURNS['hostname'],
-                         dsrc.metadata['local-hostname'])
-        self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
-                         dsrc.userdata_raw)
-        self.assertEqual(MOCK_RETURNS['root_authorized_keys'],
-                         dsrc.metadata['public-keys'])
-        self.assertEqual(MOCK_RETURNS['disable_iptables_flag'],
-                         dsrc.metadata['iptables_disable'])
-        self.assertEqual(MOCK_RETURNS['enable_motd_sys_info'],
-                         dsrc.metadata['motd_sys_info'])
-
-    def test_b64_userdata(self):
-        my_returns = MOCK_RETURNS.copy()
-        my_returns['b64-cloud-init:user-data'] = "true"
-        my_returns['b64-hostname'] = "true"
-        for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64e(my_returns[k])
-
-        dsrc = self._get_ds(mockdata=my_returns)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual(MOCK_RETURNS['hostname'],
-                         dsrc.metadata['local-hostname'])
-        self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
-                         dsrc.userdata_raw)
-        self.assertEqual(MOCK_RETURNS['root_authorized_keys'],
-                         dsrc.metadata['public-keys'])
-
-    def test_b64_keys(self):
-        my_returns = MOCK_RETURNS.copy()
-        my_returns['base64_keys'] = 'hostname,ignored'
-        for k in ('hostname',):
-            my_returns[k] = b64e(my_returns[k])
-
-        dsrc = self._get_ds(mockdata=my_returns)
-        ret = dsrc.get_data()
-        self.assertTrue(ret)
-        self.assertEqual(MOCK_RETURNS['hostname'],
-                         dsrc.metadata['local-hostname'])
-        self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
-                         dsrc.userdata_raw)
-
     def test_userdata(self):
         dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
@@ -275,6 +212,13 @@
         self.assertEqual(MOCK_RETURNS['cloud-init:user-data'],
                          dsrc.userdata_raw)
 
+    def test_sdc_nics(self):
+        dsrc = self._get_ds(mockdata=MOCK_RETURNS)
+        ret = dsrc.get_data()
+        self.assertTrue(ret)
+        self.assertEqual(json.loads(MOCK_RETURNS['sdc:nics']),
+                          dsrc.metadata['network-data'])
+
     def test_sdc_scripts(self):
         dsrc = self._get_ds(mockdata=MOCK_RETURNS)
         ret = dsrc.get_data()
@@ -430,18 +374,7 @@
                          mydscfg['disk_aliases']['FOO'])
 
 
-def apply_patches(patches):
-    ret = []
-    for (ref, name, replace) in patches:
-        if replace is None:
-            continue
-        orig = getattr(ref, name)
-        setattr(ref, name, replace)
-        ret.append((ref, name, orig))
-    return ret
-
-
-class TestJoyentMetadataClient(helpers.FilesystemMockingTestCase):
+class TestJoyentMetadataClient(FilesystemMockingTestCase):
 
     def setUp(self):
         super(TestJoyentMetadataClient, self).setUp()
@@ -481,7 +414,8 @@
                        mock.Mock(return_value=self.request_id)))
 
     def _get_client(self):
-        return DataSourceSmartOS.JoyentMetadataClient(self.serial)
+        return DataSourceSmartOS.JoyentMetadataClient(
+            fp=self.serial, smartos_type=DataSourceSmartOS.SMARTOS_ENV_KVM)
 
     def assertEndsWith(self, haystack, prefix):
         self.assertTrue(haystack.endswith(prefix),
@@ -495,7 +429,7 @@
 
     def test_get_metadata_writes_a_single_line(self):
         client = self._get_client()
-        client.get_metadata('some_key')
+        client.get('some_key')
         self.assertEqual(1, self.serial.write.call_count)
         written_line = self.serial.write.call_args[0][0]
         print(type(written_line))
@@ -505,7 +439,7 @@
 
     def _get_written_line(self, key='some_key'):
         client = self._get_client()
-        client.get_metadata(key)
+        client.get(key)
         return self.serial.write.call_args[0][0]
 
     def test_get_metadata_writes_bytes(self):
@@ -549,32 +483,32 @@
 
     def test_get_metadata_reads_a_line(self):
         client = self._get_client()
-        client.get_metadata('some_key')
+        client.get('some_key')
         self.assertEqual(self.metasource_data_len, self.serial.read.call_count)
 
     def test_get_metadata_returns_valid_value(self):
         client = self._get_client()
-        value = client.get_metadata('some_key')
+        value = client.get('some_key')
         self.assertEqual(self.metadata_value, value)
 
     def test_get_metadata_throws_exception_for_incorrect_length(self):
         self.response_parts['length'] = 0
         client = self._get_client()
         self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException,
-                          client.get_metadata, 'some_key')
+                          client.get, 'some_key')
 
     def test_get_metadata_throws_exception_for_incorrect_crc(self):
         self.response_parts['crc'] = 'deadbeef'
         client = self._get_client()
         self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException,
-                          client.get_metadata, 'some_key')
+                          client.get, 'some_key')
 
     def test_get_metadata_throws_exception_for_request_id_mismatch(self):
         self.response_parts['request_id'] = 'deadbeef'
         client = self._get_client()
         client._checksum = lambda _: self.response_parts['crc']
         self.assertRaises(DataSourceSmartOS.JoyentMetadataFetchException,
-                          client.get_metadata, 'some_key')
+                          client.get, 'some_key')
 
     def test_get_metadata_returns_None_if_value_not_found(self):
         self.response_parts['payload'] = ''
@@ -582,4 +516,24 @@
         self.response_parts['length'] = 17
         client = self._get_client()
         client._checksum = lambda _: self.response_parts['crc']
-        self.assertIsNone(client.get_metadata('some_key'))
+        self.assertIsNone(client.get('some_key'))
+
+
+class TestNetworkConversion(TestCase):
+
+    def test_convert_simple(self):
+        expected = {
+            'version': 1,
+            'config': [
+                {'name': 'net0', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '8.12.42.1',
+                              'netmask': '255.255.255.0',
+                              'address': '8.12.42.102/24'}],
+                 'mtu': 1500, 'mac_address': '90:b8:d0:f5:e4:f5'},
+                {'name': 'net1', 'type': 'physical',
+                 'subnets': [{'type': 'static', 'gateway': '192.168.128.1',
+                              'netmask': '255.255.252.0',
+                              'address': '192.168.128.93/22'}],
+                 'mtu': 8500, 'mac_address': '90:b8:d0:a5:ff:cd'}]}
+        found = DataSourceSmartOS.convert_smartos_network_data(SDC_NICS)
+        self.assertEqual(expected, found)