Merge ~xnox/cloud-init:networkd into cloud-init:master

[Dimitri John Ledkov]

azure: discover datasource from networkd lease files

LP: #1718029

azure: discover datasource from networkd lease files

LP: #1718029

[Dimitri John Ledkov (xnox)]

Testing seems all good. In-place test on Azure yields:

2017-10-02 08:06:31,894 - azure.py[DEBUG]: Finding Azure endpoint from networkd...
2017-10-02 08:06:31,895 - util.py[DEBUG]: Reading from /run/systemd/netif/leases/2 (quiet=False)
2017-10-02 08:06:31,895 - util.py[DEBUG]: Read 410 bytes from /run/systemd/netif/leases/2
2017-10-02 08:06:31,896 - azure.py[DEBUG]: Azure endpoint found at 168.63.129.16

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:71c71ae832fc7960597a626e55e4b4f43f59f09b
https://jenkins.ubuntu.com/server/job/cloud-init-ci/364/
Executed test runs:
    SUCCESS: Checkout
    FAILED: Unit & Style Tests

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/364/rebuild

review: Needs Fixing (continuous-integration)

19e4771... by Dimitri John Ledkov on 2017-10-02

CloudStack: discover server address from networkd lease files

LP: #1718029

[Dimitri John Ledkov (xnox)]

CloudStack portion is untested; please provide a cloudstack xenial instance with ssh-import-id lp:xnox

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:19e4771e125793d34e24cd7a35876f19bd4ebe25
https://jenkins.ubuntu.com/server/job/cloud-init-ci/365/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    SUCCESS: MAAS Compatability Testing
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/365/rebuild

review: Approve (continuous-integration)

[Scott Moser (smoser)]

I re-worked a bit at
 https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/331664

[Dimitri John Ledkov (xnox)]

Superseeded by https://code.launchpad.net/~smoser/cloud-init/+git/cloud-init/+merge/331664

review: Needs Resubmitting

Unmerged commits

19e4771... by Dimitri John Ledkov on 2017-10-02

CloudStack: discover server address from networkd lease files

LP: #1718029

9d7548b... by Dimitri John Ledkov on 2017-09-30

azure: discover datasource from networkd lease files

LP: #1718029

diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 7e0f9bb..5efbb99 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -12,6 +12,7 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import glob
 import os
 from socket import inet_ntoa
 from struct import pack
@@ -23,6 +24,9 @@ from cloudinit import sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util
 
+from six.moves.configparser import ConfigParser
+from six.moves import cStringIO as StringIO
+
 LOG = logging.getLogger(__name__)
 
 
@@ -220,16 +224,31 @@ def get_latest_lease(lease_d=None):
     return latest_file
 
 
+def _get_server_address_from_networkd(_leases='/run/systemd/netif/leases/*'):
+    for lease in glob.iglob(_leases):
+        cp = ConfigParser()
+        # pylint: disable=deprecated-method
+        cp.readfp(StringIO('[LEASE]\n' + util.load_file(lease)))
+        return cp.get('LEASE', 'SERVER_ADDRESS')
+    return None
+
+
 def get_vr_address():
     # Get the address of the virtual router via dhcp leases
     # If no virtual router is detected, fallback on default gateway.
     # See http://docs.cloudstack.apache.org/projects/cloudstack-administration/en/4.8/virtual_machines/user-data.html # noqa
+
+    # Try networkd first...
+    latest_address = _get_server_address_from_networkd()
+    if latest_address:
+        return latest_address
+
+    # Try dhcp lease files next...
     lease_file = get_latest_lease()
     if not lease_file:
         LOG.debug("No lease file found, using default gateway")
         return get_default_gateway()
 
-    latest_address = None
     with open(lease_file, "r") as fd:
         for line in fd:
             if "dhcp-server-identifier" in line:
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 28ed0ae..adffe98 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -1,5 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import glob
 import json
 import logging
 import os
@@ -15,6 +16,8 @@ from xml.etree import ElementTree
 
 from cloudinit import util
 
+from six.moves.configparser import ConfigParser
+from six.moves import cStringIO as StringIO
 
 LOG = logging.getLogger(__name__)
 
@@ -239,6 +242,18 @@ class WALinuxAgentShim(object):
         return socket.inet_ntoa(packed_bytes)
 
     @staticmethod
+    def _get_value_from_networkd_leases(_leases='/run/systemd/netif/leases/*'):
+        for lease in glob.iglob(_leases):
+            cp = ConfigParser()
+            try:
+                # pylint: disable=deprecated-method
+                cp.readfp(StringIO('[LEASE]\n' + util.load_file(lease)))
+                return cp.get('LEASE', 'OPTION_245')
+            except Exception:
+                pass
+        return None
+
+    @staticmethod
     def _get_value_from_leases_file(fallback_lease_file):
         leases = []
         content = util.load_file(fallback_lease_file)
@@ -287,12 +302,15 @@ class WALinuxAgentShim(object):
 
     @staticmethod
     def find_endpoint(fallback_lease_file=None):
-        LOG.debug('Finding Azure endpoint...')
         value = None
-        # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
-        # a dhclient exit hook that calls cloud-init-dhclient-hook
-        dhcp_options = WALinuxAgentShim._load_dhclient_json()
-        value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
+        LOG.debug('Finding Azure endpoint from networkd...')
+        value = WALinuxAgentShim._get_value_from_networkd_leases()
+        if value is None:
+            # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
+            # a dhclient exit hook that calls cloud-init-dhclient-hook
+            LOG.debug('Finding Azure endpoint from hook json...')
+            dhcp_options = WALinuxAgentShim._load_dhclient_json()
+            value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
         if value is None:
             # Fallback and check the leases file if unsuccessful
             LOG.debug("Unable to find endpoint in dhclient logs. "
diff --git a/tests/unittests/test_datasource/test_azure_helper.py b/tests/unittests/test_datasource/test_azure_helper.py
index 44b99ec..10cd7f4 100644
--- a/tests/unittests/test_datasource/test_azure_helper.py
+++ b/tests/unittests/test_datasource/test_azure_helper.py
@@ -1,6 +1,8 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 import os
+import shutil
+import tempfile
 
 from cloudinit.sources.helpers import azure as azure_helper
 from cloudinit.tests.helpers import ExitStack, mock, TestCase
@@ -135,6 +137,45 @@ class TestExtractIpAddressFromLeaseValue(TestCase):
             ))
 
 
+class TestExtractIpAddressFromNetworkd(TestCase):
+
+    def setUp(self):
+        super(TestExtractIpAddressFromNetworkd, self).setUp()
+        lease_dir = tempfile.mkdtemp()
+        self.leases = lease_dir + '/*'
+        ifindex = '2'
+        self.lease_file = os.path.join(lease_dir, ifindex)
+        self.addCleanup(shutil.rmtree, lease_dir, ignore_errors=True)
+
+    def test_none(self):
+        value = None
+        self.assertEqual(
+            value,
+            azure_helper.WALinuxAgentShim._get_value_from_networkd_leases(
+                self.leases
+            ))
+
+    def test_typical(self):
+        value = '624c3620'
+        with open(self.lease_file, 'w') as f:
+            f.write('OPTION_245=%s\n' % value)
+        self.assertEqual(
+            value,
+            azure_helper.WALinuxAgentShim._get_value_from_networkd_leases(
+                self.leases
+            ))
+
+    def test_malformed(self):
+        value = None
+        with open(self.lease_file, 'w') as f:
+            f.write('OPTION_246=624c3620\n')
+        self.assertEqual(
+            value,
+            azure_helper.WALinuxAgentShim._get_value_from_networkd_leases(
+                self.leases
+            ))
+
+
 class TestGoalStateParsing(TestCase):
 
     default_parameters = {