Merge lp:~smoser/curtin/simplestreams into lp:~matsubara/curtin/simplestreams

[Scott Moser]

[Scott Moser (smoser)]

A few cleanups. A few added features.

[Diogo Matsubara (matsubara)]

Changes looks good to me. Thanks for the patch. I'll incorporate them into my branch.

review: Approve

=== modified file 'Makefile'
--- Makefile	2015-11-10 17:24:36 +0000
+++ Makefile	2015-11-10 19:41:00 +0000
@@ -1,6 +1,8 @@
 TOP := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 CWD := $(shell pwd)
 PYTHON ?= python3
+CURTIN_VMTEST_IMAGE_SYNC ?= False
+export CURTIN_VMTEST_IMAGE_SYNC
 
 build:
 
@@ -30,7 +32,7 @@
 
 # By default don't sync images when running all tests.
 vmtest:
-	CURTIN_VMTEST_IMAGE_SYNC=False nosetests3 $(noseopts) tests/vmtests
+	nosetests3 $(noseopts) tests/vmtests
 
 vmtest-deps:
 	@$(CWD)/tools/vmtest-system-setup

=== modified file 'tests/vmtests/__init__.py'
--- tests/vmtests/__init__.py	2015-11-10 17:24:36 +0000
+++ tests/vmtests/__init__.py	2015-11-10 19:41:00 +0000
@@ -15,12 +15,13 @@
 
 from .helpers import check_call
 
-IMAGE_SRC_URL = (
+IMAGE_SRC_URL = os.environ.get(
+    'IMAGE_SRC_URL',
     "http://maas.ubuntu.com/images/ephemeral-v2/daily/streams/v1/index.sjson")
+
 IMAGE_DIR = os.environ.get("IMAGE_DIR", "/srv/images")
 DEFAULT_SSTREAM_OPTS = [
-    '--max=1',
-    '--keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg']
+    '--max=1', '--keyring=/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg']
 DEFAULT_FILTERS = ['arch=amd64', 'item_name=root-image.gz']
 
 
@@ -73,7 +74,17 @@
 
     def sync_images(self, filters=DEFAULT_FILTERS):
         """Sync MAAS images from source_url simplestreams."""
-        cmd = ['sstream-mirror'] + DEFAULT_SSTREAM_OPTS + [
+        try:
+            out = subprocess.check_output(
+                ['sstream-mirror', '--help'], stderr=subprocess.STDOUT)
+            if isinstance(out, bytes):
+                out = out.decode()
+            if '--progress' in out:
+                progress = ['--progress']
+        except subprocess.CalledProcessError:
+            progress = []
+
+        cmd = ['sstream-mirror'] + DEFAULT_SSTREAM_OPTS + progress + [
             self.source_url, self.base_dir] + filters
         logger.debug('Syncing images {}'.format(cmd))
         out = subprocess.check_output(cmd)
@@ -81,15 +92,14 @@
 
     def get_image(self, release, arch, filters=None):
         """Return local path for root image, kernel and initrd."""
-        if not filters:
-            filters = (
-                'release={release} krel={release} arch={arch}'.format(
-                release=release, arch=arch))
+        if filters is None:
+            filters = [
+                'release=%s' % release, 'krel=%s' % release, 'arch=%s' % arch]
         # Query local sstream for compressed root image.
         logger.debug(
-            'Query simplestreams for root image: ' + filters)
+            'Query simplestreams for root image: %s', filters)
         cmd = ['sstream-query'] + DEFAULT_SSTREAM_OPTS + [
-            self.url, 'item_name=root-image.gz' ] + filters.split()
+            self.url, 'item_name=root-image.gz'] + filters
         logger.debug(" ".join(cmd))
         out = subprocess.check_output(cmd)
         logger.debug(out)
@@ -97,7 +107,7 @@
         # If there's output, ImageStore.sync decides if images should be
         # synced or not.
         if not out or self.sync:
-            self.sync_images(filters=filters.split())
+            self.sync_images(filters=filters)
             out = subprocess.check_output(cmd)
         sstream_data = ast.literal_eval(bytes.decode(out))
         root_image_gz = urllib.parse.urlsplit(sstream_data['item_url']).path
@@ -107,7 +117,7 @@
         with open(root_image_gz, "rb") as fp:
             checksum = hashlib.sha256(fp.read()).hexdigest()
         if checksum != sstream_data['sha256']:
-            self.sync_images(filters=filters.split())
+            self.sync_images(filters=filters)
             out = subprocess.check_output(cmd)
             sstream_data = ast.literal_eval(bytes.decode(out))
             root_image_gz = urllib.parse.urlsplit(
@@ -119,8 +129,8 @@
         initrd_path = os.path.join(image_dir, 'root-image-initrd')
         # Check if we need to unpack things from the compressed image.
         if (not os.path.exists(root_image_path) or
-            not os.path.exists(kernel_path) or
-            not os.path.exists(initrd_path)):
+                not os.path.exists(kernel_path) or
+                not os.path.exists(initrd_path)):
             self.convert_image(root_image_gz)
         return (root_image_path, kernel_path, initrd_path)
 
@@ -171,7 +181,7 @@
                               stdout=DEVNULL, stderr=subprocess.STDOUT)
 
     def __del__(self):
-        if (os.getenv('CURTIN_VMTEST_KEEP_DATA', "false") != "true"):
+        if get_env_var_bool('CURTIN_VMTEST_KEEP_DATA', False):
             # remove tempdir
             shutil.rmtree(self.tmpdir)
 
@@ -187,7 +197,7 @@
         # get boot img
         image_store = ImageStore(IMAGE_SRC_URL, IMAGE_DIR)
         # Disable sync if env var is set.
-        if os.getenv('CURTIN_VMTEST_IMAGE_SYNC', False):
+        if get_env_var_bool('CURTIN_VMTEST_IMAGE_SYNC', False):
             logger.debug("Images not synced.")
             image_store.sync = False
         (boot_img, boot_kernel, boot_initrd) = image_store.get_image(
@@ -472,3 +482,18 @@
         logger.debug('Cloud config archive content (pre-json):' + part)
         parts.append({'content': part, 'type': 'text/x-shellscript'})
     return '#cloud-config-archive\n' + json.dumps(parts, indent=1)
+
+
+def get_env_var_bool(envname, default=False):
+    """get a boolean environment variable.
+
+    If environment variable is not set, use default.
+    False values are case insensitive 'false', '0', ''."""
+    if not isinstance(default, bool):
+        raise ValueError("default '%s' for '%s' is not a boolean" %
+                         (default, envname))
+    val = os.environ.get(envname)
+    if val is None:
+        return default
+
+    return val.lower() not in ("false", "0", "")

=== modified file 'tools/vmtest-system-setup'
--- tools/vmtest-system-setup	2015-09-21 16:19:01 +0000
+++ tools/vmtest-system-setup	2015-11-10 19:41:00 +0000
@@ -5,6 +5,10 @@
 fail() { [ $# -eq 0 ] || error "$@"; exit 2; }
 
 rel="$(lsb_release -sc)"
+case "$(uname -m)" in
+  i?86|x86_64) qemu="qemu-system-x86";;
+  ppc*) qemu="qemu-system-ppc";;
+esac
 DEPS=(
   cloud-image-utils
   make
@@ -12,6 +16,7 @@
   python3-nose
   python3-yaml
   simplestreams
+  $qemu
   ubuntu-cloudimage-keyring
 )