Merge ~chad.smith/cloud-init:ubuntu/devel into cloud-init:ubuntu/devel

[Chad Smith]

Perform upstream snapshot from tip of master for release into Cosmic

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:c4c6301f2e5f6fe9960ceb7f670224f87b27e091
https://jenkins.ubuntu.com/server/job/cloud-init-ci/178/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/178/rebuild

review: Approve (continuous-integration)

diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index ac72ac4..a604825 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -276,27 +276,27 @@ def maybe_cleanup_default(net_name, did_init, create, attach,
     if net_name != _DEFAULT_NETWORK_NAME or not did_init:
         return
 
-    fail_assume_enoent = " failed. Assuming it did not exist."
-    succeeded = " succeeded."
+    fail_assume_enoent = "failed. Assuming it did not exist."
+    succeeded = "succeeded."
     if create:
-        msg = "Deletion of lxd network '%s'" % net_name
+        msg = "Deletion of lxd network '%s' %s"
         try:
             _lxc(["network", "delete", net_name])
-            LOG.debug(msg + succeeded)
+            LOG.debug(msg, net_name, succeeded)
         except util.ProcessExecutionError as e:
             if e.exit_code != 1:
                 raise e
-            LOG.debug(msg + fail_assume_enoent)
+            LOG.debug(msg, net_name, fail_assume_enoent)
 
     if attach:
-        msg = "Removal of device '%s' from profile '%s'" % (nic_name, profile)
+        msg = "Removal of device '%s' from profile '%s' %s"
         try:
             _lxc(["profile", "device", "remove", profile, nic_name])
-            LOG.debug(msg + succeeded)
+            LOG.debug(msg, nic_name, profile, succeeded)
         except util.ProcessExecutionError as e:
             if e.exit_code != 1:
                 raise e
-            LOG.debug(msg + fail_assume_enoent)
+            LOG.debug(msg, nic_name, profile, fail_assume_enoent)
 
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/config/cc_rh_subscription.py b/cloudinit/config/cc_rh_subscription.py
index 1c67943..edee01e 100644
--- a/cloudinit/config/cc_rh_subscription.py
+++ b/cloudinit/config/cc_rh_subscription.py
@@ -126,7 +126,6 @@ class SubscriptionManager(object):
         self.enable_repo = self.rhel_cfg.get('enable-repo')
         self.disable_repo = self.rhel_cfg.get('disable-repo')
         self.servicelevel = self.rhel_cfg.get('service-level')
-        self.subman = ['subscription-manager']
 
     def log_success(self, msg):
         '''Simple wrapper for logging info messages. Useful for unittests'''
@@ -173,21 +172,12 @@ class SubscriptionManager(object):
         cmd = ['identity']
 
         try:
-            self._sub_man_cli(cmd)
+            _sub_man_cli(cmd)
         except util.ProcessExecutionError:
             return False
 
         return True
 
-    def _sub_man_cli(self, cmd, logstring_val=False):
-        '''
-        Uses the prefered cloud-init subprocess def of util.subp
-        and runs subscription-manager.  Breaking this to a
-        separate function for later use in mocking and unittests
-        '''
-        cmd = self.subman + cmd
-        return util.subp(cmd, logstring=logstring_val)
-
     def rhn_register(self):
         '''
         Registers the system by userid and password or activation key
@@ -209,7 +199,7 @@ class SubscriptionManager(object):
                 cmd.append("--serverurl={0}".format(self.server_hostname))
 
             try:
-                return_out = self._sub_man_cli(cmd, logstring_val=True)[0]
+                return_out = _sub_man_cli(cmd, logstring_val=True)[0]
             except util.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
@@ -232,7 +222,7 @@ class SubscriptionManager(object):
 
             # Attempting to register the system only
             try:
-                return_out = self._sub_man_cli(cmd, logstring_val=True)[0]
+                return_out = _sub_man_cli(cmd, logstring_val=True)[0]
             except util.ProcessExecutionError as e:
                 if e.stdout == "":
                     self.log_warn("Registration failed due "
@@ -255,7 +245,7 @@ class SubscriptionManager(object):
                .format(self.servicelevel)]
 
         try:
-            return_out = self._sub_man_cli(cmd)[0]
+            return_out = _sub_man_cli(cmd)[0]
         except util.ProcessExecutionError as e:
             if e.stdout.rstrip() != '':
                 for line in e.stdout.split("\n"):
@@ -273,7 +263,7 @@ class SubscriptionManager(object):
     def _set_auto_attach(self):
         cmd = ['attach', '--auto']
         try:
-            return_out = self._sub_man_cli(cmd)[0]
+            return_out = _sub_man_cli(cmd)[0]
         except util.ProcessExecutionError as e:
             self.log_warn("Auto-attach failed with: {0}".format(e))
             return False
@@ -292,12 +282,12 @@ class SubscriptionManager(object):
 
         # Get all available pools
         cmd = ['list', '--available', '--pool-only']
-        results = self._sub_man_cli(cmd)[0]
+        results = _sub_man_cli(cmd)[0]
         available = (results.rstrip()).split("\n")
 
         # Get all consumed pools
         cmd = ['list', '--consumed', '--pool-only']
-        results = self._sub_man_cli(cmd)[0]
+        results = _sub_man_cli(cmd)[0]
         consumed = (results.rstrip()).split("\n")
 
         return available, consumed
@@ -309,14 +299,14 @@ class SubscriptionManager(object):
         '''
 
         cmd = ['repos', '--list-enabled']
-        return_out = self._sub_man_cli(cmd)[0]
+        return_out = _sub_man_cli(cmd)[0]
         active_repos = []
         for repo in return_out.split("\n"):
             if "Repo ID:" in repo:
                 active_repos.append((repo.split(':')[1]).strip())
 
         cmd = ['repos', '--list-disabled']
-        return_out = self._sub_man_cli(cmd)[0]
+        return_out = _sub_man_cli(cmd)[0]
 
         inactive_repos = []
         for repo in return_out.split("\n"):
@@ -346,7 +336,7 @@ class SubscriptionManager(object):
         if len(pool_list) > 0:
             cmd.extend(pool_list)
             try:
-                self._sub_man_cli(cmd)
+                _sub_man_cli(cmd)
                 self.log.debug("Attached the following pools to your "
                                "system: %s", (", ".join(pool_list))
                                .replace('--pool=', ''))
@@ -423,7 +413,7 @@ class SubscriptionManager(object):
             cmd.extend(enable_list)
 
         try:
-            self._sub_man_cli(cmd)
+            _sub_man_cli(cmd)
         except util.ProcessExecutionError as e:
             self.log_warn("Unable to alter repos due to {0}".format(e))
             return False
@@ -439,4 +429,15 @@ class SubscriptionManager(object):
     def is_configured(self):
         return bool((self.userid and self.password) or self.activation_key)
 
+
+def _sub_man_cli(cmd, logstring_val=False):
+    '''
+    Uses the prefered cloud-init subprocess def of util.subp
+    and runs subscription-manager.  Breaking this to a
+    separate function for later use in mocking and unittests
+    '''
+    return util.subp(['subscription-manager'] + cmd,
+                     logstring=logstring_val)
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 365af96..b9ade90 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -28,7 +28,8 @@ DMI_PRODUCT_NOVA = 'OpenStack Nova'
 DMI_PRODUCT_COMPUTE = 'OpenStack Compute'
 VALID_DMI_PRODUCT_NAMES = [DMI_PRODUCT_NOVA, DMI_PRODUCT_COMPUTE]
 DMI_ASSET_TAG_OPENTELEKOM = 'OpenTelekomCloud'
-VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM]
+DMI_ASSET_TAG_ORACLE_CLOUD = 'OracleCloud.com'
+VALID_DMI_ASSET_TAGS = [DMI_ASSET_TAG_OPENTELEKOM, DMI_ASSET_TAG_ORACLE_CLOUD]
 
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index f92e8b5..ad8cfb9 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -564,7 +564,7 @@ class JoyentMetadataSerialClient(JoyentMetadataClient):
                     continue
                 LOG.warning('Unexpected response "%s" during flush', response)
             except JoyentMetadataTimeoutException:
-                LOG.warning('Timeout while initializing metadata client. ' +
+                LOG.warning('Timeout while initializing metadata client. '
                             'Is the host metadata service running?')
         LOG.debug('Got "invalid command".  Flush complete.')
         self.fp.timeout = timeout
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index f424316..06e613f 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -103,14 +103,14 @@ class DataSource(object):
     url_timeout = 10    # timeout for each metadata url read attempt
     url_retries = 5     # number of times to retry url upon 404
 
-    # The datasource defines a list of supported EventTypes during which
+    # The datasource defines a set of supported EventTypes during which
     # the datasource can react to changes in metadata and regenerate
     # network configuration on metadata changes.
     # A datasource which supports writing network config on each system boot
-    # would set update_events = {'network': [EventType.BOOT]}
+    # would call update_events['network'].add(EventType.BOOT).
 
     # Default: generate network config on new instance id (first boot).
-    update_events = {'network': [EventType.BOOT_NEW_INSTANCE]}
+    update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])}
 
     # N-tuple listing default values for any metadata-related class
     # attributes cached on an instance by a process_data runs. These attribute
@@ -475,8 +475,8 @@ class DataSource(object):
             for update_scope, update_events in self.update_events.items():
                 if event in update_events:
                     if not supported_events.get(update_scope):
-                        supported_events[update_scope] = []
-                    supported_events[update_scope].append(event)
+                        supported_events[update_scope] = set()
+                    supported_events[update_scope].add(event)
         for scope, matched_events in supported_events.items():
             LOG.debug(
                 "Update datasource metadata and %s config due to events: %s",
@@ -490,6 +490,8 @@ class DataSource(object):
             result = self.get_data()
             if result:
                 return True
+        LOG.debug("Datasource %s not updated for events: %s", self,
+                  ', '.join(source_event_types))
         return False
 
     def check_instance_id(self, sys_cfg):
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index dcd221b..9e939c1 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -429,8 +429,9 @@ class TestDataSource(CiTestCase):
 
     def test_update_metadata_only_acts_on_supported_update_events(self):
         """update_metadata won't get_data on unsupported update events."""
+        self.datasource.update_events['network'].discard(EventType.BOOT)
         self.assertEqual(
-            {'network': [EventType.BOOT_NEW_INSTANCE]},
+            {'network': set([EventType.BOOT_NEW_INSTANCE])},
             self.datasource.update_events)
 
         def fake_get_data():
diff --git a/cloudinit/tests/test_util.py b/cloudinit/tests/test_util.py
index 6a31e50..edb0c18 100644
--- a/cloudinit/tests/test_util.py
+++ b/cloudinit/tests/test_util.py
@@ -57,6 +57,34 @@ OS_RELEASE_CENTOS = dedent("""\
     REDHAT_SUPPORT_PRODUCT_VERSION="7"
 """)
 
+OS_RELEASE_REDHAT_7 = dedent("""\
+    NAME="Red Hat Enterprise Linux Server"
+    VERSION="7.5 (Maipo)"
+    ID="rhel"
+    ID_LIKE="fedora"
+    VARIANT="Server"
+    VARIANT_ID="server"
+    VERSION_ID="7.5"
+    PRETTY_NAME="Red Hat"
+    ANSI_COLOR="0;31"
+    CPE_NAME="cpe:/o:redhat:enterprise_linux:7.5:GA:server"
+    HOME_URL="https://www.redhat.com/"
+    BUG_REPORT_URL="https://bugzilla.redhat.com/"
+
+    REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 7"
+    REDHAT_BUGZILLA_PRODUCT_VERSION=7.5
+    REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux"
+    REDHAT_SUPPORT_PRODUCT_VERSION="7.5"
+""")
+
+REDHAT_RELEASE_CENTOS_6 = "CentOS release 6.10 (Final)"
+REDHAT_RELEASE_CENTOS_7 = "CentOS Linux release 7.5.1804 (Core)"
+REDHAT_RELEASE_REDHAT_6 = (
+    "Red Hat Enterprise Linux Server release 6.10 (Santiago)")
+REDHAT_RELEASE_REDHAT_7 = (
+    "Red Hat Enterprise Linux Server release 7.5 (Maipo)")
+
+
 OS_RELEASE_DEBIAN = dedent("""\
     PRETTY_NAME="Debian GNU/Linux 9 (stretch)"
     NAME="Debian GNU/Linux"
@@ -337,6 +365,12 @@ class TestGetLinuxDistro(CiTestCase):
         if path == '/etc/os-release':
             return 1
 
+    @classmethod
+    def redhat_release_exists(self, path):
+        """Side effect function """
+        if path == '/etc/redhat-release':
+            return 1
+
     @mock.patch('cloudinit.util.load_file')
     def test_get_linux_distro_quoted_name(self, m_os_release, m_path_exists):
         """Verify we get the correct name if the os-release file has
@@ -356,8 +390,48 @@ class TestGetLinuxDistro(CiTestCase):
         self.assertEqual(('ubuntu', '16.04', 'xenial'), dist)
 
     @mock.patch('cloudinit.util.load_file')
-    def test_get_linux_centos(self, m_os_release, m_path_exists):
-        """Verify we get the correct name and release name on CentOS."""
+    def test_get_linux_centos6(self, m_os_release, m_path_exists):
+        """Verify we get the correct name and release name on CentOS 6."""
+        m_os_release.return_value = REDHAT_RELEASE_CENTOS_6
+        m_path_exists.side_effect = TestGetLinuxDistro.redhat_release_exists
+        dist = util.get_linux_distro()
+        self.assertEqual(('centos', '6.10', 'Final'), dist)
+
+    @mock.patch('cloudinit.util.load_file')
+    def test_get_linux_centos7_redhat_release(self, m_os_release, m_exists):
+        """Verify the correct release info on CentOS 7 without os-release."""
+        m_os_release.return_value = REDHAT_RELEASE_CENTOS_7
+        m_exists.side_effect = TestGetLinuxDistro.redhat_release_exists
+        dist = util.get_linux_distro()
+        self.assertEqual(('centos', '7.5.1804', 'Core'), dist)
+
+    @mock.patch('cloudinit.util.load_file')
+    def test_get_linux_redhat7_osrelease(self, m_os_release, m_path_exists):
+        """Verify redhat 7 read from os-release."""
+        m_os_release.return_value = OS_RELEASE_REDHAT_7
+        m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists
+        dist = util.get_linux_distro()
+        self.assertEqual(('redhat', '7.5', 'Maipo'), dist)
+
+    @mock.patch('cloudinit.util.load_file')
+    def test_get_linux_redhat7_rhrelease(self, m_os_release, m_path_exists):
+        """Verify redhat 7 read from redhat-release."""
+        m_os_release.return_value = REDHAT_RELEASE_REDHAT_7
+        m_path_exists.side_effect = TestGetLinuxDistro.redhat_release_exists
+        dist = util.get_linux_distro()
+        self.assertEqual(('redhat', '7.5', 'Maipo'), dist)
+
+    @mock.patch('cloudinit.util.load_file')
+    def test_get_linux_redhat6_rhrelease(self, m_os_release, m_path_exists):
+        """Verify redhat 6 read from redhat-release."""
+        m_os_release.return_value = REDHAT_RELEASE_REDHAT_6
+        m_path_exists.side_effect = TestGetLinuxDistro.redhat_release_exists
+        dist = util.get_linux_distro()
+        self.assertEqual(('redhat', '6.10', 'Santiago'), dist)
+
+    @mock.patch('cloudinit.util.load_file')
+    def test_get_linux_copr_centos(self, m_os_release, m_path_exists):
+        """Verify we get the correct name and release name on COPR CentOS."""
         m_os_release.return_value = OS_RELEASE_CENTOS
         m_path_exists.side_effect = TestGetLinuxDistro.os_release_exists
         dist = util.get_linux_distro()
diff --git a/cloudinit/util.py b/cloudinit/util.py
index d0b0e90..5068096 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -576,12 +576,42 @@ def get_cfg_option_int(yobj, key, default=0):
     return int(get_cfg_option_str(yobj, key, default=default))
 
 
+def _parse_redhat_release(release_file=None):
+    """Return a dictionary of distro info fields from /etc/redhat-release.
+
+    Dict keys will align with /etc/os-release keys:
+        ID, VERSION_ID, VERSION_CODENAME
+    """
+
+    if not release_file:
+        release_file = '/etc/redhat-release'
+    if not os.path.exists(release_file):
+        return {}
+    redhat_release = load_file(release_file)
+    redhat_regex = (
+        r'(?P<name>.+) release (?P<version>[\d\.]+) '
+        r'\((?P<codename>[^)]+)\)')
+    match = re.match(redhat_regex, redhat_release)
+    if match:
+        group = match.groupdict()
+        group['name'] = group['name'].lower().partition(' linux')[0]
+        if group['name'] == 'red hat enterprise':
+            group['name'] = 'redhat'
+        return {'ID': group['name'], 'VERSION_ID': group['version'],
+                'VERSION_CODENAME': group['codename']}
+    return {}
+
+
 def get_linux_distro():
     distro_name = ''
     distro_version = ''
     flavor = ''
+    os_release = {}
     if os.path.exists('/etc/os-release'):
         os_release = load_shell_content(load_file('/etc/os-release'))
+    if not os_release:
+        os_release = _parse_redhat_release()
+    if os_release:
         distro_name = os_release.get('ID', '')
         distro_version = os_release.get('VERSION_ID', '')
         if 'sles' in distro_name or 'suse' in distro_name:
@@ -594,9 +624,11 @@ def get_linux_distro():
             flavor = os_release.get('VERSION_CODENAME', '')
             if not flavor:
                 match = re.match(r'[^ ]+ \((?P<codename>[^)]+)\)',
-                                 os_release.get('VERSION'))
+                                 os_release.get('VERSION', ''))
                 if match:
                     flavor = match.groupdict()['codename']
+        if distro_name == 'rhel':
+            distro_name = 'redhat'
     else:
         dist = ('', '', '')
         try:
diff --git a/cloudinit/warnings.py b/cloudinit/warnings.py
index f9f7a63..1da90c4 100644
--- a/cloudinit/warnings.py
+++ b/cloudinit/warnings.py
@@ -130,7 +130,7 @@ def show_warning(name, cfg=None, sleep=None, mode=True, **kwargs):
         os.path.join(_get_warn_dir(cfg), name),
         topline + "\n".join(fmtlines) + "\n" + topline)
 
-    LOG.warning(topline + "\n".join(fmtlines) + "\n" + closeline)
+    LOG.warning("%s%s\n%s", topline, "\n".join(fmtlines), closeline)
 
     if sleep:
         LOG.debug("sleeping %d seconds for warning '%s'", sleep, name)
diff --git a/debian/changelog b/debian/changelog
index d6a89e4..05932be 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,19 @@
+cloud-init (18.3-18-g3cee0bf8-0ubuntu1) cosmic; urgency=medium
+
+  * New upstream snapshot.
+    - oracle: fix detect_openstack to report True on OracleCloud.com DMI data
+    - tests: improve LXDInstance trying to workaround or catch bug.
+    - update_metadata re-config on every boot comments and tests not quite
+      right [Mike Gerdts]
+    - tests: Collect build_info from system if available.
+    - pylint: Fix pylint warnings reported in pylint 2.0.0.
+    - get_linux_distro: add support for rhel via redhat-release.
+    - get_linux_distro: add support for centos6 and rawhide flavors of redhat
+    - tools: add '--debug' to tools/net-convert.py
+    - tests: bump the version of paramiko to 2.4.1.
+
+ -- Chad Smith <[e-mail]>  Tue, 31 Jul 2018 12:50:28 -0600
+
 cloud-init (18.3-9-g2e62cb8a-0ubuntu1) cosmic; urgency=medium
 
   * New upstream snapshot.
diff --git a/integration-requirements.txt b/integration-requirements.txt
index 01baebd..f80cb94 100644
--- a/integration-requirements.txt
+++ b/integration-requirements.txt
@@ -9,7 +9,7 @@
 boto3==1.5.9
 
 # ssh communication
-paramiko==2.4.0
+paramiko==2.4.1
 
 # lxd backend
 # 04/03/2018: enables use of lxd 3.0
diff --git a/tests/cloud_tests/platforms/instances.py b/tests/cloud_tests/platforms/instances.py
index 95bc3b1..529e79c 100644
--- a/tests/cloud_tests/platforms/instances.py
+++ b/tests/cloud_tests/platforms/instances.py
@@ -97,7 +97,8 @@ class Instance(TargetBase):
             return self._ssh_client
 
         if not self.ssh_ip or not self.ssh_port:
-            raise ValueError
+            raise ValueError("Cannot ssh_connect, ssh_ip=%s ssh_port=%s" %
+                             (self.ssh_ip, self.ssh_port))
 
         client = paramiko.SSHClient()
         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
diff --git a/tests/cloud_tests/platforms/lxd/instance.py b/tests/cloud_tests/platforms/lxd/instance.py
index d396519..83c97ab 100644
--- a/tests/cloud_tests/platforms/lxd/instance.py
+++ b/tests/cloud_tests/platforms/lxd/instance.py
@@ -12,6 +12,8 @@ from tests.cloud_tests.util import PlatformError
 
 from ..instances import Instance
 
+from pylxd import exceptions as pylxd_exc
+
 
 class LXDInstance(Instance):
     """LXD container backed instance."""
@@ -30,6 +32,9 @@ class LXDInstance(Instance):
         @param config: image config
         @param features: supported feature flags
         """
+        if not pylxd_container:
+            raise ValueError("Invalid value pylxd_container: %s" %
+                             pylxd_container)
         self._pylxd_container = pylxd_container
         super(LXDInstance, self).__init__(
             platform, name, properties, config, features)
@@ -40,9 +45,19 @@ class LXDInstance(Instance):
     @property
     def pylxd_container(self):
         """Property function."""
+        if self._pylxd_container is None:
+            raise RuntimeError(
+                "%s: Attempted use of pylxd_container after deletion." % self)
         self._pylxd_container.sync()
         return self._pylxd_container
 
+    def __str__(self):
+        return (
+            '%s(name=%s) status=%s' %
+            (self.__class__.__name__, self.name,
+             ("deleted" if self._pylxd_container is None else
+              self.pylxd_container.status)))
+
     def _execute(self, command, stdin=None, env=None):
         if env is None:
             env = {}
@@ -165,10 +180,27 @@ class LXDInstance(Instance):
         self.shutdown(wait=wait)
         self.start(wait=wait)
 
-    def shutdown(self, wait=True):
+    def shutdown(self, wait=True, retry=1):
         """Shutdown instance."""
-        if self.pylxd_container.status != 'Stopped':
+        if self.pylxd_container.status == 'Stopped':
+            return
+
+        try:
+            LOG.debug("%s: shutting down (wait=%s)", self, wait)
             self.pylxd_container.stop(wait=wait)
+        except (pylxd_exc.LXDAPIException, pylxd_exc.NotFound) as e:
+            # An exception happens here sometimes (LP: #1783198)
+            # LOG it, and try again.
+            LOG.warning(
+                ("%s: shutdown(retry=%d) caught %s in shutdown "
+                 "(response=%s): %s"),
+                self, retry, e.__class__.__name__, e.response, e)
+            if isinstance(e, pylxd_exc.NotFound):
+                LOG.debug("container_exists(%s) == %s",
+                          self.name, self.platform.container_exists(self.name))
+            if retry == 0:
+                raise e
+            return self.shutdown(wait=wait, retry=retry - 1)
 
     def start(self, wait=True, wait_for_cloud_init=False):
         """Start instance."""
@@ -189,12 +221,14 @@ class LXDInstance(Instance):
 
     def destroy(self):
         """Clean up instance."""
+        LOG.debug("%s: deleting container.", self)
         self.unfreeze()
         self.shutdown()
         self.pylxd_container.delete(wait=True)
+        self._pylxd_container = None
+
         if self.platform.container_exists(self.name):
-            raise OSError('container {} was not properly removed'
-                          .format(self.name))
+            raise OSError('%s: container was not properly removed' % self)
         if self._console_log_file and os.path.exists(self._console_log_file):
             os.unlink(self._console_log_file)
         shutil.rmtree(self.tmpd)
diff --git a/tests/cloud_tests/setup_image.py b/tests/cloud_tests/setup_image.py
index 4e19570..39f4517 100644
--- a/tests/cloud_tests/setup_image.py
+++ b/tests/cloud_tests/setup_image.py
@@ -4,6 +4,7 @@
 
 from functools import partial
 import os
+import yaml
 
 from tests.cloud_tests import LOG
 from tests.cloud_tests import stage, util
@@ -220,7 +221,14 @@ def setup_image(args, image):
     calls = [partial(stage.run_single, desc, partial(func, args, image))
              for name, func, desc in handlers if getattr(args, name, None)]
 
-    LOG.info('setting up %s', image)
+    try:
+        data = yaml.load(image.read_data("/etc/cloud/build.info", decode=True))
+        info = ' '.join(["%s=%s" % (k, data.get(k))
+                         for k in ("build_name", "serial") if k in data])
+    except Exception as e:
+        info = "N/A (%s)" % e
+
+    LOG.info('setting up %s (%s)', image, info)
     res = stage.run_stage(
         'set up for {}'.format(image), calls, continue_after_error=False)
     return res
diff --git a/tests/cloud_tests/testcases.yaml b/tests/cloud_tests/testcases.yaml
index a16d1dd..fb9a5d2 100644
--- a/tests/cloud_tests/testcases.yaml
+++ b/tests/cloud_tests/testcases.yaml
@@ -27,6 +27,10 @@ base_test_data:
         package-versions: |
             #!/bin/sh
             dpkg-query --show
+        build.info: |
+            #!/bin/sh
+            binfo=/etc/cloud/build.info
+            [ -f "$binfo" ] && cat "$binfo" || echo "N/A"
         system.journal.gz: |
             #!/bin/sh
             [ -d /run/systemd ] || { echo "not systemd."; exit 0; }
diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py
index 585acc3..d862f4b 100644
--- a/tests/unittests/test_datasource/test_openstack.py
+++ b/tests/unittests/test_datasource/test_openstack.py
@@ -510,6 +510,24 @@ class TestDetectOpenStack(test_helpers.CiTestCase):
             ds.detect_openstack(),
             'Expected detect_openstack == True on OpenTelekomCloud')
 
+    @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
+    def test_detect_openstack_oraclecloud_chassis_asset_tag(self, m_dmi,
+                                                            m_is_x86):
+        """Return True on OpenStack reporting Oracle cloud asset-tag."""
+        m_is_x86.return_value = True
+
+        def fake_dmi_read(dmi_key):
+            if dmi_key == 'system-product-name':
+                return 'Standard PC (i440FX + PIIX, 1996)'  # No match
+            if dmi_key == 'chassis-asset-tag':
+                return 'OracleCloud.com'
+            assert False, 'Unexpected dmi read of %s' % dmi_key
+
+        m_dmi.side_effect = fake_dmi_read
+        self.assertTrue(
+            ds.detect_openstack(),
+            'Expected detect_openstack == True on OracleCloud.com')
+
     @test_helpers.mock.patch(MOCK_PATH + 'util.get_proc_env')
     @test_helpers.mock.patch(MOCK_PATH + 'util.read_dmi_data')
     def test_detect_openstack_by_proc_1_environ(self, m_dmi, m_proc_env,
diff --git a/tests/unittests/test_rh_subscription.py b/tests/unittests/test_rh_subscription.py
index 2271810..4cd27ee 100644
--- a/tests/unittests/test_rh_subscription.py
+++ b/tests/unittests/test_rh_subscription.py
@@ -8,10 +8,16 @@ import logging
 from cloudinit.config import cc_rh_subscription
 from cloudinit import util
 
-from cloudinit.tests.helpers import TestCase, mock
+from cloudinit.tests.helpers import CiTestCase, mock
 
+SUBMGR = cc_rh_subscription.SubscriptionManager
+SUB_MAN_CLI = 'cloudinit.config.cc_rh_subscription._sub_man_cli'
+
+
[e-mail](SUB_MAN_CLI)
+class GoodTests(CiTestCase):
+    with_logs = True
 
-class GoodTests(TestCase):
     def setUp(self):
         super(GoodTests, self).setUp()
         self.name = "cc_rh_subscription"
@@ -19,7 +25,6 @@ class GoodTests(TestCase):
         self.log = logging.getLogger("good_tests")
         self.args = []
         self.handle = cc_rh_subscription.handle
-        self.SM = cc_rh_subscription.SubscriptionManager
 
         self.config = {'rh_subscription':
                        {'username': '[e-mail]',
@@ -35,55 +40,47 @@ class GoodTests(TestCase):
                              'disable-repo': ['repo4', 'repo5']
                              }}
 
-    def test_already_registered(self):
+    def test_already_registered(self, m_sman_cli):
         '''
         Emulates a system that is already registered. Ensure it gets
         a non-ProcessExecution error from is_registered()
         '''
-        with mock.patch.object(cc_rh_subscription.SubscriptionManager,
-                               '_sub_man_cli') as mockobj:
-            self.SM.log_success = mock.MagicMock()
-            self.handle(self.name, self.config, self.cloud_init,
-                        self.log, self.args)
-            self.assertEqual(self.SM.log_success.call_count, 1)
-            self.assertEqual(mockobj.call_count, 1)
-
-    def test_simple_registration(self):
+        self.handle(self.name, self.config, self.cloud_init,
+                    self.log, self.args)
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assertIn('System is already registered', self.logs.getvalue())
+
+    def test_simple_registration(self, m_sman_cli):
         '''
         Simple registration with username and password
         '''
-        self.SM.log_success = mock.MagicMock()
         reg = "The system has been registered with ID:" \
               " 12345678-abde-abcde-1234-1234567890abc"
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError, (reg, 'bar')]
         self.handle(self.name, self.config, self.cloud_init,
                     self.log, self.args)
-        self.assertIn(mock.call(['identity']),
-                      self.SM._sub_man_cli.call_args_list)
+        self.assertIn(mock.call(['identity']), m_sman_cli.call_args_list)
         self.assertIn(mock.call(['register', '--username=[e-mail]',
                                  '--password=scooby-snacks'],
                                 logstring_val=True),
-                      self.SM._sub_man_cli.call_args_list)
-
-        self.assertEqual(self.SM.log_success.call_count, 1)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 2)
+                      m_sman_cli.call_args_list)
+        self.assertIn('rh_subscription plugin completed successfully',
+                      self.logs.getvalue())
+        self.assertEqual(m_sman_cli.call_count, 2)
 
     @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_getRepos")
-    @mock.patch.object(cc_rh_subscription.SubscriptionManager, "_sub_man_cli")
-    def test_update_repos_disable_with_none(self, m_sub_man_cli, m_get_repos):
+    def test_update_repos_disable_with_none(self, m_get_repos, m_sman_cli):
         cfg = copy.deepcopy(self.config)
         m_get_repos.return_value = ([], ['repo1'])
-        m_sub_man_cli.return_value = (b'', b'')
         cfg['rh_subscription'].update(
             {'enable-repo': ['repo1'], 'disable-repo': None})
         mysm = cc_rh_subscription.SubscriptionManager(cfg)
         self.assertEqual(True, mysm.update_repos())
         m_get_repos.assert_called_with()
-        self.assertEqual(m_sub_man_cli.call_args_list,
+        self.assertEqual(m_sman_cli.call_args_list,
                          [mock.call(['repos', '--enable=repo1'])])
 
-    def test_full_registration(self):
+    def test_full_registration(self, m_sman_cli):
         '''
         Registration with auto-attach, service-level, adding pools,
         and enabling and disabling yum repos
@@ -93,26 +90,28 @@ class GoodTests(TestCase):
         call_lists.append(['repos', '--disable=repo5', '--enable=repo2',
                            '--enable=repo3'])
         call_lists.append(['attach', '--auto', '--servicelevel=self-support'])
-        self.SM.log_success = mock.MagicMock()
         reg = "The system has been registered with ID:" \
               " 12345678-abde-abcde-1234-1234567890abc"
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (reg, 'bar'),
-                         ('Service level set to: self-support', ''),
-                         ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''),
-                         ('Repo ID: repo1\nRepo ID: repo5\n', ''),
-                         ('Repo ID: repo2\nRepo ID: repo3\nRepo ID: '
-                          'repo4', ''),
-                         ('', '')])
+        m_sman_cli.side_effect = [
+            util.ProcessExecutionError,
+            (reg, 'bar'),
+            ('Service level set to: self-support', ''),
+            ('pool1\npool3\n', ''), ('pool2\n', ''), ('', ''),
+            ('Repo ID: repo1\nRepo ID: repo5\n', ''),
+            ('Repo ID: repo2\nRepo ID: repo3\nRepo ID: repo4', ''),
+            ('', '')]
         self.handle(self.name, self.config_full, self.cloud_init,
                     self.log, self.args)
+        self.assertEqual(m_sman_cli.call_count, 9)
         for call in call_lists:
-            self.assertIn(mock.call(call), self.SM._sub_man_cli.call_args_list)
-        self.assertEqual(self.SM.log_success.call_count, 1)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 9)
+            self.assertIn(mock.call(call), m_sman_cli.call_args_list)
+        self.assertIn("rh_subscription plugin completed successfully",
+                      self.logs.getvalue())
 
 
-class TestBadInput(TestCase):
[e-mail](SUB_MAN_CLI)
+class TestBadInput(CiTestCase):
+    with_logs = True
     name = "cc_rh_subscription"
     cloud_init = None
     log = logging.getLogger("bad_tests")
@@ -155,81 +154,81 @@ class TestBadInput(TestCase):
         super(TestBadInput, self).setUp()
         self.handle = cc_rh_subscription.handle
 
-    def test_no_password(self):
-        '''
-        Attempt to register without the password key/value
-        '''
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+    def assert_logged_warnings(self, warnings):
+        logs = self.logs.getvalue()
+        missing = [w for w in warnings if "WARNING: " + w not in logs]
+        self.assertEqual([], missing, "Missing expected warnings.")
+
+    def test_no_password(self, m_sman_cli):
+        '''Attempt to register without the password key/value.'''
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_no_password, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 0)
+        self.assertEqual(m_sman_cli.call_count, 0)
 
-    def test_no_org(self):
-        '''
-        Attempt to register without the org key/value
-        '''
-        self.input_is_missing_data(self.config_no_key)
-
-    def test_service_level_without_auto(self):
-        '''
-        Attempt to register using service-level without the auto-attach key
-        '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+    def test_no_org(self, m_sman_cli):
+        '''Attempt to register without the org key/value.'''
+        m_sman_cli.side_effect = [util.ProcessExecutionError]
+        self.handle(self.name, self.config_no_key, self.cloud_init,
+                    self.log, self.args)
+        m_sman_cli.assert_called_with(['identity'])
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assert_logged_warnings((
+            'Unable to register system due to incomplete information.',
+            'Use either activationkey and org *or* userid and password',
+            'Registration failed or did not run completely',
+            'rh_subscription plugin did not complete successfully'))
+
+    def test_service_level_without_auto(self, m_sman_cli):
+        '''Attempt to register using service-level without auto-attach key.'''
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_service, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 1)
-        self.assertEqual(self.SM.log_warn.call_count, 2)
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assert_logged_warnings((
+            'The service-level key must be used in conjunction with ',
+            'rh_subscription plugin did not complete successfully'))
 
-    def test_pool_not_a_list(self):
+    def test_pool_not_a_list(self, m_sman_cli):
         '''
         Register with pools that are not in the format of a list
         '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_badpool, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 2)
-        self.assertEqual(self.SM.log_warn.call_count, 2)
+        self.assertEqual(m_sman_cli.call_count, 2)
+        self.assert_logged_warnings((
+            'Pools must in the format of a list',
+            'rh_subscription plugin did not complete successfully'))
 
-    def test_repo_not_a_list(self):
+    def test_repo_not_a_list(self, m_sman_cli):
         '''
         Register with repos that are not in the format of a list
         '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_badrepo, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM.log_warn.call_count, 3)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 2)
+        self.assertEqual(m_sman_cli.call_count, 2)
+        self.assert_logged_warnings((
+            'Repo IDs must in the format of a list.',
+            'Unable to add or remove repos',
+            'rh_subscription plugin did not complete successfully'))
 
-    def test_bad_key_value(self):
+    def test_bad_key_value(self, m_sman_cli):
         '''
         Attempt to register with a key that we don't know
         '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError, (self.reg, 'bar')])
+        m_sman_cli.side_effect = [util.ProcessExecutionError,
+                                  (self.reg, 'bar')]
         self.handle(self.name, self.config_badkey, self.cloud_init,
                     self.log, self.args)
-        self.assertEqual(self.SM.log_warn.call_count, 2)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 1)
-
-    def input_is_missing_data(self, config):
-        '''
-        Helper def for tests that having missing information
-        '''
-        self.SM.log_warn = mock.MagicMock()
-        self.SM._sub_man_cli = mock.MagicMock(
-            side_effect=[util.ProcessExecutionError])
-        self.handle(self.name, config, self.cloud_init,
-                    self.log, self.args)
-        self.SM._sub_man_cli.assert_called_with(['identity'])
-        self.assertEqual(self.SM.log_warn.call_count, 4)
-        self.assertEqual(self.SM._sub_man_cli.call_count, 1)
+        self.assertEqual(m_sman_cli.call_count, 1)
+        self.assert_logged_warnings((
+            'fookey is not a valid key for rh_subscription. Valid keys are:',
+            'rh_subscription plugin did not complete successfully'))
 
 # vi: ts=4 expandtab
diff --git a/tools/net-convert.py b/tools/net-convert.py
index 68559cb..d1a4a64 100755
--- a/tools/net-convert.py
+++ b/tools/net-convert.py
@@ -4,11 +4,13 @@
 import argparse
 import json
 import os
+import sys
 import yaml
 
 from cloudinit.sources.helpers import openstack
 
 from cloudinit.net import eni
+from cloudinit import log
 from cloudinit.net import netplan
 from cloudinit.net import network_state
 from cloudinit.net import sysconfig
@@ -29,14 +31,23 @@ def main():
                         metavar="name,mac",
                         action='append',
                         help="interface name to mac mapping")
+    parser.add_argument("--debug", action='store_true',
+                        help='enable debug logging to stderr.')
     parser.add_argument("--output-kind", "-ok",
                         choices=['eni', 'netplan', 'sysconfig'],
                         required=True)
     args = parser.parse_args()
 
+    if not args.directory.endswith("/"):
+        args.directory += "/"
+
     if not os.path.isdir(args.directory):
         os.makedirs(args.directory)
 
+    if args.debug:
+        log.setupBasicLogging(level=log.DEBUG)
+    else:
+        log.setupBasicLogging(level=log.WARN)
     if args.mac:
         known_macs = {}
         for item in args.mac:
@@ -53,8 +64,10 @@ def main():
         pre_ns = yaml.load(net_data)
         if 'network' in pre_ns:
             pre_ns = pre_ns.get('network')
-        print("Input YAML")
-        print(yaml.dump(pre_ns, default_flow_style=False, indent=4))
+        if args.debug:
+            sys.stderr.write('\n'.join(
+                ["Input YAML",
+                 yaml.dump(pre_ns, default_flow_style=False, indent=4), ""]))
         ns = network_state.parse_net_config_data(pre_ns)
     else:
         pre_ns = openstack.convert_net_json(
@@ -65,8 +78,10 @@ def main():
         raise RuntimeError("No valid network_state object created from"
                            "input data")
 
-    print("\nInternal State")
-    print(yaml.dump(ns, default_flow_style=False, indent=4))
+    if args.debug:
+        sys.stderr.write('\n'.join([
+            "", "Internal State",
+            yaml.dump(ns, default_flow_style=False, indent=4), ""]))
     if args.output_kind == "eni":
         r_cls = eni.Renderer
     elif args.output_kind == "netplan":
@@ -75,6 +90,11 @@ def main():
         r_cls = sysconfig.Renderer
 
     r = r_cls()
+    sys.stderr.write(''.join([
+        "Read input format '%s' from '%s'.\n" % (
+            args.kind, args.network_data.name),
+        "Wrote output format '%s' to '%s'\n" % (
+            args.output_kind, args.directory)]) + "\n")
     r.render_network_state(network_state=ns, target=args.directory)