Merge ~chad.smith/cloud-init:fix/cent-6-jinja2 into cloud-init:master

[Chad Smith]

Centos 6: Fix jinja rendering and netlink on older python versions.

jinja2.runtime.implements_to_string is not available in older versions
of jinja2. This just catches that error separately from other
jinja support allowing the code to work.

Also fix netlink behavior stuct.unpack methods expect data as a string.

To improve testing and mocks, relocate the following:
 - read_cfg_paths to cloudinit.stages
 - addLogHandlerCLI to cloudinit.log

The testcase to recreate was:
 ./tools/run-container --package --source-package --unittest \
 --artifacts=./rpm/ centos/6

LP: #1795933

see commit message

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:b41878cb80997dafa51fd2f4c73d74bf82387f24
https://jenkins.ubuntu.com/server/job/cloud-init-ci/487/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Unit & Style Tests
    SUCCESS: Ubuntu LTS: Build
    SUCCESS: Ubuntu LTS: Integration
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/cloud-init-ci/487/rebuild

review: Approve (continuous-integration)

[Scott Moser (smoser)]

no mention of the addLogHanderCLI or read_cfg_paths move in commit message.
the rest of it appears to be test or PY26 only, so i'm good with that.

fix your commit message.

[Ryan Harper (raharper)]

I'd like to see this rebased against 18.3 since that's the last PY26 supported release.

review: Needs Fixing

Unmerged commits

b41878c... by Chad Smith on 2018-12-12

on PY26, subytraction is not attempted

5dc81f7... by Chad Smith on 2018-12-12

unittest fixups

c2f465e... by Chad Smith on 2018-12-12

always remember the mock

5f8561a... by Chad Smith on 2018-12-12

lint

e274a19... by Chad Smith on 2018-12-12

devel.read_cfg_paths -> stages and devel.addLogHandlerCLI -> log

4283e1e... by Chad Smith on 2018-12-12

struct.unpack requires str in py26

b7e7db8... by Chad Smith on 2018-12-12

need to mock leaked getuid for root unittest runs

31ef622... by Chad Smith on 2018-12-12

centos: assertraises context_manager.exception is an int on SystemExit instance

69677fc... by Scott Moser on 2018-12-12

Take Chad's good fix

9271323... by Scott Moser on 2018-10-03

Centos 6: Fix jinja rendering on older jinja versions.

jinja2.runtime.implements_to_string is not available in older versions
of jinja2. This just catches that error separately from other
jinja support allowing the code to work.

The testcase to recreate was:
  ./tools/run-container --source-package --package \
     --artifacts=./rpm/ centos/6

diff --git a/cloudinit/cmd/devel/__init__.py b/cloudinit/cmd/devel/__init__.py
index 3ae28b6..e69de29 100644
--- a/cloudinit/cmd/devel/__init__.py
+++ b/cloudinit/cmd/devel/__init__.py
@@ -1,25 +0,0 @@
-# This file is part of cloud-init. See LICENSE file for license information.
-
-"""Common cloud-init devel commandline utility functions."""
-
-
-import logging
-
-from cloudinit import log
-from cloudinit.stages import Init
-
-
-def addLogHandlerCLI(logger, log_level):
-    """Add a commandline logging handler to emit messages to stderr."""
-    formatter = logging.Formatter('%(levelname)s: %(message)s')
-    log.setupBasicLogging(log_level, formatter=formatter)
-    return logger
-
-
-def read_cfg_paths():
-    """Return a Paths object based on the system configuration on disk."""
-    init = Init(ds_deps=[])
-    init.read_cfg()
-    return init.paths
-
-# vi: ts=4 expandtab
diff --git a/cloudinit/cmd/devel/render.py b/cloudinit/cmd/devel/render.py
index 1bc2240..b93abd4 100755
--- a/cloudinit/cmd/devel/render.py
+++ b/cloudinit/cmd/devel/render.py
@@ -9,7 +9,7 @@ import sys
 from cloudinit.handlers.jinja_template import render_jinja_payload_from_file
 from cloudinit import log
 from cloudinit.sources import INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE
-from . import addLogHandlerCLI, read_cfg_paths
+from cloudinit import stages
 
 NAME = 'render'
 
@@ -45,11 +45,11 @@ def handle_args(name, args):
 
     @return 0 on success, 1 on failure.
     """
-    addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
+    log.addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
     if args.instance_data:
         instance_data_fn = args.instance_data
     else:
-        paths = read_cfg_paths()
+        paths = stages.read_cfg_paths()
         uid = os.getuid()
         redacted_data_fn = os.path.join(paths.run_dir, INSTANCE_JSON_FILE)
         if uid == 0:
diff --git a/cloudinit/cmd/devel/tests/test_render.py b/cloudinit/cmd/devel/tests/test_render.py
index 988bba0..19c05c3 100644
--- a/cloudinit/cmd/devel/tests/test_render.py
+++ b/cloudinit/cmd/devel/tests/test_render.py
@@ -7,7 +7,7 @@ from collections import namedtuple
 from cloudinit.cmd.devel import render
 from cloudinit.helpers import Paths
 from cloudinit.sources import INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE
-from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJinja
+from cloudinit.tests.helpers import CiTestCase, PY26, mock, skipUnlessJinja
 from cloudinit.util import ensure_dir, write_file
 
 
@@ -20,6 +20,7 @@ class TestRender(CiTestCase):
     def setUp(self):
         super(TestRender, self).setUp()
         self.tmp = self.tmp_dir()
+        self.add_patch('cloudinit.log.addLogHandlerCLI', 'm_logcli')
 
     def test_handle_args_error_on_missing_user_data(self):
         """When user_data file path does not exist, log an error."""
@@ -28,8 +29,7 @@ class TestRender(CiTestCase):
         write_file(instance_data, '{}')
         args = self.args(
             user_data=absent_file, instance_data=instance_data, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
+        self.assertEqual(1, render.handle_args('anyname', args))
         self.assertIn(
             'Missing user-data file: %s' % absent_file,
             self.logs.getvalue())
@@ -40,8 +40,7 @@ class TestRender(CiTestCase):
         absent_file = self.tmp_path('instance-data', dir=self.tmp)
         args = self.args(
             user_data=user_data, instance_data=absent_file, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
+        self.assertEqual(1, render.handle_args('anyname', args))
         self.assertIn(
             'Missing instance-data.json file: %s' % absent_file,
             self.logs.getvalue())
@@ -52,12 +51,11 @@ class TestRender(CiTestCase):
         run_dir = self.tmp_path('run_dir', dir=self.tmp)
         ensure_dir(run_dir)
         paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
+        self.add_patch('cloudinit.stages.read_cfg_paths', 'm_paths')
         self.m_paths.return_value = paths
         args = self.args(
             user_data=user_data, instance_data=None, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
+        self.assertEqual(1, render.handle_args('anyname', args))
         json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
         self.assertIn(
             'Missing instance-data.json file: %s' % json_file,
@@ -69,14 +67,13 @@ class TestRender(CiTestCase):
         run_dir = self.tmp_path('run_dir', dir=self.tmp)
         ensure_dir(run_dir)
         paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
+        self.add_patch('cloudinit.stages.read_cfg_paths', 'm_paths')
         self.m_paths.return_value = paths
         args = self.args(
             user_data=user_data, instance_data=None, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            with mock.patch('os.getuid') as m_getuid:
-                m_getuid.return_value = 0
-                self.assertEqual(1, render.handle_args('anyname', args))
+        with mock.patch('os.getuid') as m_getuid:
+            m_getuid.return_value = 0
+            self.assertEqual(1, render.handle_args('anyname', args))
         json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
         json_sensitive = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE)
         self.assertIn(
@@ -95,15 +92,14 @@ class TestRender(CiTestCase):
         json_sensitive = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE)
         write_file(json_sensitive, '{"my-var": "jinja worked"}')
         paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.devel.render.read_cfg_paths', 'm_paths')
+        self.add_patch('cloudinit.stages.read_cfg_paths', 'm_paths')
         self.m_paths.return_value = paths
         args = self.args(
             user_data=user_data, instance_data=None, debug=False)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-                with mock.patch('os.getuid') as m_getuid:
-                    m_getuid.return_value = 0
-                    self.assertEqual(0, render.handle_args('anyname', args))
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            with mock.patch('os.getuid') as m_getuid:
+                m_getuid.return_value = 0
+                self.assertEqual(0, render.handle_args('anyname', args))
         self.assertIn('rendering: jinja worked', m_stdout.getvalue())
 
     @skipUnlessJinja()
@@ -115,13 +111,10 @@ class TestRender(CiTestCase):
         write_file(instance_data, '{"my-var": "jinja worked"}')
         args = self.args(
             user_data=user_data, instance_data=instance_data, debug=True)
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_console_err:
-            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-                self.assertEqual(0, render.handle_args('anyname', args))
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(0, render.handle_args('anyname', args))
         self.assertIn(
             'DEBUG: Converted jinja variables\n{', self.logs.getvalue())
-        self.assertIn(
-            'DEBUG: Converted jinja variables\n{', m_console_err.getvalue())
         self.assertEqual('rendering: jinja worked', m_stdout.getvalue())
 
     @skipUnlessJinja()
@@ -133,12 +126,16 @@ class TestRender(CiTestCase):
         write_file(instance_data, '{"my-var": "jinja worked"}')
         args = self.args(
             user_data=user_data, instance_data=instance_data, debug=True)
-        with mock.patch('sys.stderr', new_callable=StringIO):
-            self.assertEqual(1, render.handle_args('anyname', args))
-        self.assertIn(
-            'WARNING: Ignoring jinja template for %s: Undefined jinja'
-            ' variable: "my-var". Jinja tried subtraction. Perhaps you meant'
-            ' "my_var"?' % user_data,
-            self.logs.getvalue())
+        self.assertEqual(1, render.handle_args('anyname', args))
+        if PY26:
+            msg = (
+                'WARNING: Ignoring jinja template for %s: \'var\' is'
+                ' undefined' % user_data)
+        else:
+            msg = (
+                'WARNING: Ignoring jinja template for %s: Undefined jinja'
+                ' variable: "my-var". Jinja tried subtraction. Perhaps you'
+                ' meant "my_var"?' % user_data)
+        self.assertIn(msg, self.logs.getvalue())
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/cmd/query.py b/cloudinit/cmd/query.py
index 1d888b9..07d9d5f 100644
--- a/cloudinit/cmd/query.py
+++ b/cloudinit/cmd/query.py
@@ -10,10 +10,10 @@ import sys
 
 from cloudinit.handlers.jinja_template import (
     convert_jinja_instance_data, render_jinja_payload)
-from cloudinit.cmd.devel import addLogHandlerCLI, read_cfg_paths
 from cloudinit import log
 from cloudinit.sources import (
     INSTANCE_JSON_FILE, INSTANCE_JSON_SENSITIVE_FILE, REDACT_SENSITIVE_VALUE)
+from cloudinit import stages
 from cloudinit import util
 
 NAME = 'query'
@@ -69,7 +69,7 @@ def get_parser(parser=None):
 def handle_args(name, args):
     """Handle calls to 'cloud-init query' as a subcommand."""
     paths = None
-    addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
+    log.addLogHandlerCLI(LOG, log.DEBUG if args.debug else log.WARNING)
     if not any([args.list_keys, args.varname, args.format, args.dump_all]):
         LOG.error(
             'Expected one of the options: --all, --format,'
@@ -79,7 +79,7 @@ def handle_args(name, args):
 
     uid = os.getuid()
     if not all([args.instance_data, args.user_data, args.vendor_data]):
-        paths = read_cfg_paths()
+        paths = stages.read_cfg_paths()
     if args.instance_data:
         instance_data_fn = args.instance_data
     else:
diff --git a/cloudinit/cmd/tests/test_clean.py b/cloudinit/cmd/tests/test_clean.py
index 5a3ec3b..266db32 100644
--- a/cloudinit/cmd/tests/test_clean.py
+++ b/cloudinit/cmd/tests/test_clean.py
@@ -169,7 +169,7 @@ class TestClean(CiTestCase):
                  'sys.argv': {'new': ['clean', '--logs']}},
                 clean.main)
 
-        self.assertEqual(0, context_manager.exception.code)
+        self.assertEqual('0', str(context_manager.exception))
         self.assertFalse(
             os.path.exists(self.log1), 'Unexpected log {0}'.format(self.log1))
 
diff --git a/cloudinit/cmd/tests/test_cloud_id.py b/cloudinit/cmd/tests/test_cloud_id.py
index 7373817..3998272 100644
--- a/cloudinit/cmd/tests/test_cloud_id.py
+++ b/cloudinit/cmd/tests/test_cloud_id.py
@@ -49,7 +49,7 @@ class TestCloudId(CiTestCase):
             with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
                 with self.assertRaises(SystemExit) as context_manager:
                     cloud_id.main()
-        self.assertEqual(1, context_manager.exception.code)
+        self.assertEqual('1', str(context_manager.exception))
         self.assertIn(
             "ERROR: File not found '%s'" % self.instance_data,
             m_stderr.getvalue())
@@ -62,7 +62,7 @@ class TestCloudId(CiTestCase):
             with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
                 with self.assertRaises(SystemExit) as context_manager:
                     cloud_id.main()
-        self.assertEqual(1, context_manager.exception.code)
+        self.assertEqual('1', str(context_manager.exception))
         self.assertIn(
             "ERROR: File '%s' is not valid json." % self.instance_data,
             m_stderr.getvalue())
@@ -77,7 +77,7 @@ class TestCloudId(CiTestCase):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 with self.assertRaises(SystemExit) as context_manager:
                     cloud_id.main()
-        self.assertEqual(0, context_manager.exception.code)
+        self.assertEqual('0', str(context_manager.exception))
         self.assertEqual("mycloud\n", m_stdout.getvalue())
 
     def test_cloud_id_long_name_from_instance_data(self):
@@ -90,7 +90,7 @@ class TestCloudId(CiTestCase):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 with self.assertRaises(SystemExit) as context_manager:
                     cloud_id.main()
-        self.assertEqual(0, context_manager.exception.code)
+        self.assertEqual('0', str(context_manager.exception))
         self.assertEqual("mycloud\tsomereg\n", m_stdout.getvalue())
 
     def test_cloud_id_lookup_from_instance_data_region(self):
@@ -104,7 +104,7 @@ class TestCloudId(CiTestCase):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 with self.assertRaises(SystemExit) as context_manager:
                     cloud_id.main()
-        self.assertEqual(0, context_manager.exception.code)
+        self.assertEqual('0', str(context_manager.exception))
         self.assertEqual("aws-china\tcn-north-1\n", m_stdout.getvalue())
 
     def test_cloud_id_lookup_json_instance_data_adds_cloud_id_to_json(self):
@@ -121,7 +121,7 @@ class TestCloudId(CiTestCase):
             with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
                 with self.assertRaises(SystemExit) as context_manager:
                     cloud_id.main()
-        self.assertEqual(0, context_manager.exception.code)
+        self.assertEqual('0', str(context_manager.exception))
         self.assertEqual(expected + '\n', m_stdout.getvalue())
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/cmd/tests/test_query.py b/cloudinit/cmd/tests/test_query.py
index 28738b1..3b32bd3 100644
--- a/cloudinit/cmd/tests/test_query.py
+++ b/cloudinit/cmd/tests/test_query.py
@@ -27,21 +27,20 @@ class TestQuery(CiTestCase):
         super(TestQuery, self).setUp()
         self.tmp = self.tmp_dir()
         self.instance_data = self.tmp_path('instance-data', dir=self.tmp)
+        self.add_patch('cloudinit.log.addLogHandlerCLI', 'm_logcli')
 
     def test_handle_args_error_on_missing_param(self):
         """Error when missing required parameters and print usage."""
         args = self.args(
             debug=False, dump_all=False, format=None, instance_data=None,
             list_keys=False, user_data=None, vendor_data=None, varname=None)
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-                self.assertEqual(1, query.handle_args('anyname', args))
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            self.assertEqual(1, query.handle_args('anyname', args))
         expected_error = (
             'ERROR: Expected one of the options: --all, --format, --list-keys'
             ' or varname\n')
         self.assertIn(expected_error, self.logs.getvalue())
         self.assertIn('usage: query', m_stdout.getvalue())
-        self.assertIn(expected_error, m_stderr.getvalue())
 
     def test_handle_args_error_on_missing_instance_data(self):
         """When instance_data file path does not exist, log an error."""
@@ -49,14 +48,10 @@ class TestQuery(CiTestCase):
         args = self.args(
             debug=False, dump_all=True, format=None, instance_data=absent_fn,
             list_keys=False, user_data='ud', vendor_data='vd', varname=None)
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-            self.assertEqual(1, query.handle_args('anyname', args))
+        self.assertEqual(1, query.handle_args('anyname', args))
         self.assertIn(
             'ERROR: Missing instance-data file: %s' % absent_fn,
             self.logs.getvalue())
-        self.assertIn(
-            'ERROR: Missing instance-data file: %s' % absent_fn,
-            m_stderr.getvalue())
 
     def test_handle_args_error_when_no_read_permission_instance_data(self):
         """When instance_data file is unreadable, log an error."""
@@ -65,16 +60,12 @@ class TestQuery(CiTestCase):
         args = self.args(
             debug=False, dump_all=True, format=None, instance_data=noread_fn,
             list_keys=False, user_data='ud', vendor_data='vd', varname=None)
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-            with mock.patch('cloudinit.cmd.query.util.load_file') as m_load:
-                m_load.side_effect = OSError(errno.EACCES, 'Not allowed')
-                self.assertEqual(1, query.handle_args('anyname', args))
+        with mock.patch('cloudinit.cmd.query.util.load_file') as m_load:
+            m_load.side_effect = OSError(errno.EACCES, 'Not allowed')
+            self.assertEqual(1, query.handle_args('anyname', args))
         self.assertIn(
             "ERROR: No read permission on '%s'. Try sudo" % noread_fn,
             self.logs.getvalue())
-        self.assertIn(
-            "ERROR: No read permission on '%s'. Try sudo" % noread_fn,
-            m_stderr.getvalue())
 
     def test_handle_args_defaults_instance_data(self):
         """When no instance_data argument, default to configured run_dir."""
@@ -84,17 +75,13 @@ class TestQuery(CiTestCase):
         run_dir = self.tmp_path('run_dir', dir=self.tmp)
         ensure_dir(run_dir)
         paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths')
+        self.add_patch('cloudinit.stages.read_cfg_paths', 'm_paths')
         self.m_paths.return_value = paths
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-            self.assertEqual(1, query.handle_args('anyname', args))
+        self.assertEqual(1, query.handle_args('anyname', args))
         json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
         self.assertIn(
             'ERROR: Missing instance-data file: %s' % json_file,
             self.logs.getvalue())
-        self.assertIn(
-            'ERROR: Missing instance-data file: %s' % json_file,
-            m_stderr.getvalue())
 
     def test_handle_args_root_fallsback_to_instance_data(self):
         """When no instance_data argument, root falls back to redacted json."""
@@ -104,18 +91,17 @@ class TestQuery(CiTestCase):
         run_dir = self.tmp_path('run_dir', dir=self.tmp)
         ensure_dir(run_dir)
         paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths')
+        self.add_patch('cloudinit.stages.read_cfg_paths', 'm_paths')
         self.m_paths.return_value = paths
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-            with mock.patch('os.getuid') as m_getuid:
-                m_getuid.return_value = 0
-                self.assertEqual(1, query.handle_args('anyname', args))
+        with mock.patch('os.getuid') as m_getuid:
+            m_getuid.return_value = 0
+            self.assertEqual(1, query.handle_args('anyname', args))
         json_file = os.path.join(run_dir, INSTANCE_JSON_FILE)
         sensitive_file = os.path.join(run_dir, INSTANCE_JSON_SENSITIVE_FILE)
         self.assertIn(
             'WARNING: Missing root-readable %s. Using redacted %s instead.' % (
                 sensitive_file, json_file),
-            m_stderr.getvalue())
+            self.logs.getvalue())
 
     def test_handle_args_root_uses_instance_sensitive_data(self):
         """When no instance_data argument, root uses semsitive json."""
@@ -128,7 +114,7 @@ class TestQuery(CiTestCase):
         write_file(sensitive_file, '{"my-var": "it worked"}')
         ensure_dir(run_dir)
         paths = Paths({'run_dir': run_dir})
-        self.add_patch('cloudinit.cmd.query.read_cfg_paths', 'm_paths')
+        self.add_patch('cloudinit.stages.read_cfg_paths', 'm_paths')
         self.m_paths.return_value = paths
         args = self.args(
             debug=False, dump_all=True, format=None, instance_data=None,
@@ -150,7 +136,9 @@ class TestQuery(CiTestCase):
             instance_data=self.instance_data, list_keys=False,
             user_data='ud', vendor_data='vd', varname=None)
         with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-            self.assertEqual(0, query.handle_args('anyname', args))
+            with mock.patch('os.getuid') as m_getuid:
+                m_getuid.return_value = 100
+                self.assertEqual(0, query.handle_args('anyname', args))
         self.assertEqual(
             '{\n "my_var": "it worked",\n "userdata": "<%s> file:ud",\n'
             ' "vendordata": "<%s> file:vd"\n}\n' % (
@@ -177,7 +165,9 @@ class TestQuery(CiTestCase):
             instance_data=self.instance_data, user_data='ud', vendor_data='vd',
             list_keys=False, varname='v1.key_2')
         with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-            self.assertEqual(0, query.handle_args('anyname', args))
+            with mock.patch('os.getuid') as m_getuid:
+                m_getuid.return_value = 100
+                self.assertEqual(0, query.handle_args('anyname', args))
         self.assertEqual('value-2\n', m_stdout.getvalue())
 
     def test_handle_args_returns_standardized_vars_to_top_level_aliases(self):
@@ -206,7 +196,9 @@ class TestQuery(CiTestCase):
             instance_data=self.instance_data, user_data='ud', vendor_data='vd',
             list_keys=False, varname=None)
         with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-            self.assertEqual(0, query.handle_args('anyname', args))
+            with mock.patch('os.getuid') as m_getuid:
+                m_getuid.return_value = 100
+                self.assertEqual(0, query.handle_args('anyname', args))
         self.assertEqual(expected, m_stdout.getvalue())
 
     def test_handle_args_list_keys_sorts_top_level_keys_when_no_varname(self):
@@ -221,7 +213,9 @@ class TestQuery(CiTestCase):
             instance_data=self.instance_data, list_keys=True, user_data='ud',
             vendor_data='vd', varname=None)
         with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-            self.assertEqual(0, query.handle_args('anyname', args))
+            with mock.patch('os.getuid') as m_getuid:
+                m_getuid.return_value = 100
+                self.assertEqual(0, query.handle_args('anyname', args))
         self.assertEqual(expected, m_stdout.getvalue())
 
     def test_handle_args_list_keys_sorts_nested_keys_when_varname(self):
@@ -236,7 +230,9 @@ class TestQuery(CiTestCase):
             instance_data=self.instance_data, list_keys=True,
             user_data='ud', vendor_data='vd', varname='v1')
         with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
-            self.assertEqual(0, query.handle_args('anyname', args))
+            with mock.patch('os.getuid') as m_getuid:
+                m_getuid.return_value = 100
+                self.assertEqual(0, query.handle_args('anyname', args))
         self.assertEqual(expected, m_stdout.getvalue())
 
     def test_handle_args_list_keys_errors_when_varname_is_not_a_dict(self):
@@ -250,10 +246,11 @@ class TestQuery(CiTestCase):
             debug=False, dump_all=False, format=None,
             instance_data=self.instance_data, list_keys=True, user_data='ud',
             vendor_data='vd',  varname='top')
-        with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr:
-            with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+        with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout:
+            with mock.patch('os.getuid') as m_getuid:
+                m_getuid.return_value = 100
                 self.assertEqual(1, query.handle_args('anyname', args))
         self.assertEqual('', m_stdout.getvalue())
-        self.assertIn(expected_error, m_stderr.getvalue())
+        self.assertIn(expected_error, self.logs.getvalue())
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/cmd/tests/test_status.py b/cloudinit/cmd/tests/test_status.py
index aded858..9f742df 100644
--- a/cloudinit/cmd/tests/test_status.py
+++ b/cloudinit/cmd/tests/test_status.py
@@ -386,7 +386,7 @@ class TestStatus(CiTestCase):
                      '_is_cloudinit_disabled': (False, ''),
                      'Init': {'side_effect': self.init_class}},
                     status.main)
-        self.assertEqual(0, context_manager.exception.code)
+        self.assertEqual('0', str(context_manager.exception))
         self.assertEqual('status: running\n', m_stdout.getvalue())
 
 # vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index dcd2645..22ecacd 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -457,4 +457,11 @@ class DefaultingConfigParser(RawConfigParser):
 def identity(object):
     return object
 
+
+def implement_unicode_from_str(object):
+    """Add __unicode__ method to object from __str__ behavior."""
+    object.__unicode__ = object.__str__
+    object.__str__ = lambda x: x.__unicode().encode('utf-8')
+    return object
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/log.py b/cloudinit/log.py
index 5ae312b..f63190a 100644
--- a/cloudinit/log.py
+++ b/cloudinit/log.py
@@ -126,6 +126,13 @@ def getLogger(name='cloudinit'):
     return logging.getLogger(name)
 
 
+def addLogHandlerCLI(logger, log_level):
+    """Add a commandline logging handler to emit messages to stderr."""
+    formatter = logging.Formatter('%(levelname)s: %(message)s')
+    setupBasicLogging(log_level, formatter=formatter)
+    return logger
+
+
 # Fixes this annoyance...
 # No handlers could be found for logger XXX annoying output...
 try:
diff --git a/cloudinit/sources/helpers/netlink.py b/cloudinit/sources/helpers/netlink.py
index d377ae3..89b4e6e 100644
--- a/cloudinit/sources/helpers/netlink.py
+++ b/cloudinit/sources/helpers/netlink.py
@@ -97,6 +97,8 @@ def get_netlink_msg_header(data):
     assert (data is not None), ("data is none")
     assert (len(data) >= NLMSGHDR_SIZE), (
         "data is smaller than netlink message header")
+    if util.PY26:
+        data = str(data)
     msg_len, msg_type, flags, seq, pid = struct.unpack(NLMSGHDR_FMT,
                                                        data[:MSG_TYPE_OFFSET])
     LOG.debug("Got netlink msg of type %d", msg_type)
@@ -140,6 +142,8 @@ def unpack_rta_attr(data, offset):
         "rta offset is less than expected length")
     length = rta_type = 0
     attr_data = None
+    if util.PY26:
+        data = str(data)
     try:
         length = struct.unpack_from("H", data, offset=offset)[0]
         rta_type = struct.unpack_from("H", data, offset=offset+2)[0]
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 8a06412..badc883 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -889,6 +889,13 @@ def fetch_base_config():
         ], reverse=True)
 
 
+def read_cfg_paths():
+    """Return a Paths object based on the system configuration on disk."""
+    init = Init(ds_deps=[])
+    init.read_cfg()
+    return init.paths
+
+
 def _pkl_store(obj, fname):
     try:
         pk_contents = pickle.dumps(obj)
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index b668674..85a5b0f 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -13,6 +13,9 @@
 import collections
 import re
 
+from cloudinit import log as logging
+from cloudinit import type_utils as tu
+from cloudinit import util
 
 try:
     from Cheetah.Template import Template as CTemplate
@@ -21,19 +24,24 @@ except (ImportError, AttributeError):
     CHEETAH_AVAILABLE = False
 
 try:
-    from jinja2.runtime import implements_to_string
     from jinja2 import Template as JTemplate
     from jinja2 import DebugUndefined as JUndefined
     JINJA_AVAILABLE = True
 except (ImportError, AttributeError):
-    from cloudinit.helpers import identity
-    implements_to_string = identity
     JINJA_AVAILABLE = False
     JUndefined = object
 
-from cloudinit import log as logging
-from cloudinit import type_utils as tu
-from cloudinit import util
+if JINJA_AVAILABLE:
+    try:
+        # centos 6 jinja2 (2.2.1) works other than missing implements_to_string
+        from jinja2.runtime import implements_to_string
+    except ImportError:
+        if not hasattr(JUndefined, '__unicode__'):
+            from cloudinit.helpers import identity
+            implements_to_string = identity
+        else:
+            from cloudinit.helpers import implement_unicode_from_str
+            implements_to_string = implement_unicode_from_str
 
 
 LOG = logging.getLogger(__name__)
@@ -47,7 +55,11 @@ class UndefinedJinjaVariable(JUndefined):
     """Class used to represent any undefined jinja template varible."""
 
     def __str__(self):
-        return u'%s%s' % (MISSING_JINJA_PREFIX, self._undefined_name)
+        if hasattr(self, '_undefined_name'):
+            hint = getattr(self, '_undefined_name')
+        else:
+            hint = getattr(self, '_undefined_hint')
+        return u'%s%s' % (MISSING_JINJA_PREFIX, hint)
 
     def __sub__(self, other):
         other = str(other).replace(MISSING_JINJA_PREFIX, '')
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index b92ffc7..b95de01 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -197,7 +197,7 @@ class TestJinjaTemplatePartHandler(CiTestCase):
         script_file = os.path.join(script_handler.script_dir, 'part01')
         self.assertEqual(
             'Cannot render jinja template vars. Instance data not yet present'
-            ' at {}/instance-data.json'.format(
+            ' at {0}/instance-data.json'.format(
                 self.run_dir), str(context_manager.exception))
         self.assertFalse(
             os.path.exists(script_file),
@@ -246,7 +246,7 @@ class TestJinjaTemplatePartHandler(CiTestCase):
             frequency='freq', headers='headers')
         script_file = os.path.join(script_handler.script_dir, 'part01')
         self.assertNotIn(
-            'Instance data not yet present at {}/instance-data.json'.format(
+            'Instance data not yet present at {0}/instance-data.json'.format(
                 self.run_dir),
             self.logs.getvalue())
         self.assertEqual(
diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py
index 1bad07f..c8e5968 100644
--- a/tests/unittests/test_handler/test_schema.py
+++ b/tests/unittests/test_handler/test_schema.py
@@ -350,7 +350,7 @@ class MainTest(CiTestCase):
                         m_stderr:
                     with self.assertRaises(SystemExit) as context_manager:
                         main()
-        self.assertEqual(1, context_manager.exception.code)
+        self.assertEqual('1', str(context_manager.exception))
         self.assertEqual(
             'Expected either --config-file argument or --doc\n',
             m_stderr.getvalue())
@@ -364,7 +364,7 @@ class MainTest(CiTestCase):
                         m_stderr:
                     with self.assertRaises(SystemExit) as context_manager:
                         main()
-        self.assertEqual(1, context_manager.exception.code)
+        self.assertEqual('1', str(context_manager.exception))
         self.assertEqual(
             'Configfile NOT_A_FILE does not exist\n',
             m_stderr.getvalue())