Merge lp:~cjwatson/ubuntu-system-image/cdimage-custom into lp:~registry/ubuntu-system-image/client

[Colin Watson]

Add a new cdimage-custom generator.

Add a new cdimage-custom generator.

This is basically just a clone-and-hack of cdimage-ubuntu, simplified somewhat. It goes with recent changes to ubuntu-cdimage, and is all with the aim of being able to fix bug 1367332 (moving some click packages to /custom) in a single step for the community Ubuntu images.

Unmerged revisions

246. By Colin Watson on 2014-10-10

Add a new cdimage-custom generator.

245. By Stéphane Graber on 2014-09-26

Drop system/android/cache/recovery from core image.

244. By Stéphane Graber on 2014-09-10

Hash http filepaths by default using a combination of the URL and version string, update code to pass current pep-8 test.

243. By Stéphane Graber on 2014-08-27

Fix variable names conflicts.

242. By Stéphane Graber on 2014-08-27

Use a comma as the separator to avoid ini parsing errors.

241. By Stéphane Graber on 2014-08-27

Add support for device overrides.

240. By Stéphane Graber on 2014-08-05

Add device name to version tarball.

239. By Stéphane Graber on 2014-08-05

Fix incorrect path for download cache in core image

238. By Stéphane Graber on 2014-08-04

Add /android/cache/recovery to core builds for now.

237. By Stéphane Graber on 2014-07-31

Skip android bits for non-touch

=== added file '.bzrignore'
--- .bzrignore	1970-01-01 00:00:00 +0000
+++ .bzrignore	2014-10-10 11:11:17 +0000
@@ -0,0 +1,9 @@
+etc/config
+lib/phablet/__pycache__
+secret/gpg/keyrings/*
+secret/gpg/keys/*
+secret/ssh/*
+tests/coverage
+tests/keys/*
+www/*
+state/*

=== renamed file '.bzrignore' => '.bzrignore.moved'
=== added file 'README'
--- README	1970-01-01 00:00:00 +0000
+++ README	2014-10-10 11:11:17 +0000
@@ -0,0 +1,12 @@
+Runtime dependencies:
+ - pxz | xz-utils
+ - python3, python3-gpgme | python, python-gpgme
+ - e2fsprogs
+ - android-tools-fsutils
+ - abootimg
+
+Test dependencies:
+ - python-mock, python3-mock
+ - python-coverage, python3-coverage
+ - pep8
+ - pyflakes3, pyflakes

=== added directory 'bin'
=== added file 'bin/copy-image'
--- bin/copy-image	1970-01-01 00:00:00 +0000
+++ bin/copy-image	2014-10-10 11:11:17 +0000
@@ -0,0 +1,308 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import os
+import sys
+sys.path.insert(0, os.path.join(sys.path[0], os.pardir, "lib"))
+
+from systemimage import config, generators, tools, tree
+
+import argparse
+import fcntl
+import logging
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="image copier")
+    parser.add_argument("source_channel", metavar="SOURCE-CHANNEL")
+    parser.add_argument("destination_channel", metavar="DESTINATION-CHANNEL")
+    parser.add_argument("device", metavar="DEVICE")
+    parser.add_argument("version", metavar="VERSION", type=int)
+    parser.add_argument("-k", "--keep-version", action="store_true",
+                        help="Keep the original verison number")
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+
+    args = parser.parse_args()
+
+    # Setup logging
+    formatter = logging.Formatter(
+        "%(asctime)s %(levelname)s %(message)s")
+
+    levels = {1: logging.ERROR,
+              2: logging.WARNING,
+              3: logging.INFO,
+              4: logging.DEBUG}
+
+    if args.verbose > 0:
+        stdoutlogger = logging.StreamHandler(sys.stdout)
+        stdoutlogger.setFormatter(formatter)
+        logging.root.setLevel(levels[min(4, args.verbose)])
+        logging.root.addHandler(stdoutlogger)
+    else:
+        logging.root.addHandler(logging.NullHandler())
+
+    # Load the configuration
+    conf = config.Config()
+
+    # Try to acquire a global lock
+    lock_file = os.path.join(conf.state_path, "global.lock")
+    lock_fd = open(lock_file, 'w')
+
+    try:
+        fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except IOError:
+        print("Something else holds the global lock. exiting.")
+        sys.exit(0)
+
+    # Load the tree
+    pub = tree.Tree(conf)
+
+    # Do some checks
+    if args.source_channel not in pub.list_channels():
+        parser.error("Invalid source channel: %s" % args.source_channel)
+
+    if args.destination_channel not in pub.list_channels():
+        parser.error("Invalid destination channel: %s" %
+                     args.destination_channel)
+
+    if args.device not in pub.list_channels()[args.source_channel]['devices']:
+        parser.error("Invalid device for source channel: %s" %
+                     args.device)
+
+    if args.device not in \
+            pub.list_channels()[args.destination_channel]['devices']:
+        parser.error("Invalid device for destination channel: %s" %
+                     args.device)
+
+    if "alias" in pub.list_channels()[args.source_channel] and \
+            pub.list_channels()[args.source_channel]['alias'] \
+            != args.source_channel:
+        parser.error("Source channel is an alias.")
+
+    if "alias" in pub.list_channels()[args.destination_channel] and \
+            pub.list_channels()[args.destination_channel]['alias'] \
+            != args.destination_channel:
+        parser.error("Destination channel is an alias.")
+
+    if "redirect" in pub.list_channels()[args.source_channel]:
+        parser.error("Source channel is a redirect.")
+
+    if "redirect" in pub.list_channels()[args.destination_channel]:
+        parser.error("Destination channel is a redirect.")
+
+    source_device = pub.get_device(args.source_channel, args.device)
+    destination_device = pub.get_device(args.destination_channel, args.device)
+
+    if args.keep_version:
+        images = [image for image in destination_device.list_images()
+                  if image['version'] == args.version]
+        if images:
+            parser.error("Version number is already used: %s" % args.version)
+
+    # Assign a new version number
+    new_version = args.version
+    if not args.keep_version:
+        # Find the next available version
+        new_version = 1
+        for image in destination_device.list_images():
+            if image['version'] >= new_version:
+                new_version = image['version'] + 1
+    logging.debug("Version for next image: %s" % new_version)
+
+    # Extract the build we want to copy
+    images = [image for image in source_device.list_images()
+              if image['type'] == "full" and image['version'] == args.version]
+    if not images:
+        parser.error("Can't find version: %s" % args.version)
+    source_image = images[0]
+
+    # Extract the list of existing full images
+    full_images = {image['version']: image
+                   for image in destination_device.list_images()
+                   if image['type'] == "full"}
+
+    # Check that the last full and the new image aren't one and the same
+    source_files = [entry['path'].split("/")[-1]
+                    for entry in source_image['files']
+                    if not entry['path'].split("/")[-1].startswith("version-")]
+    destination_files = []
+    if full_images:
+        latest_full = sorted(full_images.values(),
+                             key=lambda image: image['version'])[-1]
+        destination_files = [entry['path'].split("/")[-1]
+                             for entry in latest_full['files']
+                             if not entry['path'].split(
+                                 "/")[-1].startswith("version-")]
+    if source_files == destination_files:
+        parser.error("Source image is already latest full in "
+                     "destination channel.")
+
+    # Generate a list of required deltas
+    delta_base = []
+
+    if args.destination_channel in conf.channels:
+        for base_channel in conf.channels[args.destination_channel].deltabase:
+            # Skip missing channels
+            if base_channel not in pub.list_channels():
+                continue
+
+            # Skip missing devices
+            if args.device not in (pub.list_channels()
+                                   [base_channel]['devices']):
+                continue
+
+            # Extract the latest full image
+            base_device = pub.get_device(base_channel, args.device)
+            base_images = sorted([image
+                                  for image in base_device.list_images()
+                                  if image['type'] == "full"],
+                                 key=lambda image: image['version'])
+
+            # Check if the version is valid and add it
+            if base_images and base_images[-1]['version'] in full_images:
+                if (full_images[base_images[-1]['version']]
+                        not in delta_base):
+                    delta_base.append(full_images
+                                      [base_images[-1]['version']])
+                    logging.debug("Source version for delta: %s" %
+                                  base_images[-1]['version'])
+
+    # Create new empty entries
+    new_images = {'full': {'files': []}}
+    for delta in delta_base:
+        new_images["delta_%s" % delta['version']] = {'files': []}
+
+    # Extract current version_detail and files
+    version_detail = ""
+    for entry in source_image['files']:
+        path = os.path.realpath("%s/%s" % (conf.publish_path, entry['path']))
+
+        filename = path.split("/")[-1]
+
+        # Look for version-X.tar.xz
+        if filename == "version-%s.tar.xz" % args.version:
+            # Extract the metadata
+            if os.path.exists(path.replace(".tar.xz", ".json")):
+                with open(path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+                    if "channel.ini" in metadata:
+                        version_detail = metadata['channel.ini'].get(
+                            "version_detail", None)
+        else:
+            new_images['full']['files'].append(path)
+    logging.debug("Source version_detail is: %s" % version_detail)
+
+    # Generate new version tarball
+    environment = {}
+    environment['channel_name'] = args.destination_channel
+    environment['device'] = destination_device
+    environment['device_name'] = args.device
+    environment['version'] = new_version
+    environment['version_detail'] = [entry
+                                     for entry in version_detail.split(",")
+                                     if not entry.startswith("version=")]
+    environment['new_files'] = new_images['full']['files']
+
+    logging.info("Generating new version tarball for '%s' (%s)"
+                 % (new_version, "," % environment['version_detail']))
+    version_path = generators.generate_file(conf, "version", [], environment)
+    if version_path:
+        new_images['full']['files'].append(version_path)
+
+    # Generate deltas
+    for abspath in new_images['full']['files']:
+        prefix = abspath.split("/")[-1].rsplit("-", 1)[0]
+        for delta in delta_base:
+            # Extract the source
+            src_path = None
+            for file_dict in delta['files']:
+                if (file_dict['path'].split("/")[-1]
+                        .startswith(prefix)):
+                    src_path = "%s/%s" % (conf.publish_path,
+                                          file_dict['path'])
+                    break
+
+            # Check that it's not the current file
+            if src_path:
+                src_path = os.path.realpath(src_path)
+
+                # FIXME: the keyring- is a big hack...
+                if src_path == abspath and "keyring-" not in src_path:
+                    continue
+
+                # Generators are allowed to return None when no delta
+                # exists at all.
+                logging.info("Generating delta from '%s' for '%s'" %
+                             (delta['version'],
+                              prefix))
+                delta_path = generators.generate_delta(conf, src_path,
+                                                       abspath)
+            else:
+                delta_path = abspath
+
+            if not delta_path:
+                continue
+
+            # Get the full and relative paths
+            delta_abspath, delta_relpath = tools.expand_path(
+                delta_path, conf.publish_path)
+
+            new_images['delta_%s' % delta['version']]['files'] \
+                .append(delta_abspath)
+
+    # Add full image
+    logging.info("Publishing new image '%s' (%s) with %s files."
+                 % (new_version, ",".join(environment['version_detail']),
+                    len(new_images['full']['files'])))
+    destination_device.create_image("full", new_version,
+                                    ",".join(environment['version_detail']),
+                                    new_images['full']['files'])
+
+    # Add delta images
+    for delta in delta_base:
+        files = new_images["delta_%s" % delta['version']]['files']
+        logging.info("Publishing new delta from '%s' (%s)"
+                     " to '%s' (%s) with %s files" %
+                     (delta['version'], delta.get("description", ""),
+                      new_version, ",".join(environment['version_detail']),
+                      len(files)))
+
+        destination_device.create_image(
+            "delta", new_version,
+            ",".join(environment['version_detail']),
+            files,
+            base=delta['version'])
+
+    # Expire images
+    if args.destination_channel in conf.channels:
+        if conf.channels[args.destination_channel].fullcount > 0:
+            logging.info("Expiring old images")
+            destination_device.expire_images(
+                conf.channels[args.destination_channel].fullcount)
+
+    # Sync all channel aliases
+    logging.info("Syncing any existing alias")
+    pub.sync_aliases(args.destination_channel)
+
+    # Remove any orphaned file
+    logging.info("Removing orphaned files from the pool")
+    pub.cleanup_tree()
+
+    # Sync the mirrors
+    logging.info("Triggering a mirror sync")
+    tools.sync_mirrors(conf)

=== added file 'bin/generate-keyrings'
--- bin/generate-keyrings	1970-01-01 00:00:00 +0000
+++ bin/generate-keyrings	2014-10-10 11:11:17 +0000
@@ -0,0 +1,87 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+import time
+sys.path.insert(0, 'lib')
+
+from systemimage import config
+from systemimage import gpg
+from systemimage import tools
+
+conf = config.Config()
+
+# archive-master keyring
+if os.path.exists(os.path.join(conf.gpg_key_path, "archive-master")):
+    archive_master = gpg.Keyring(conf, "archive-master")
+    archive_master.set_metadata("archive-master")
+    archive_master.import_keys(os.path.join(conf.gpg_key_path,
+                                            "archive-master"))
+    path = archive_master.generate_tarball()
+    tools.xz_compress(path)
+    os.remove(path)
+    gpg.sign_file(conf, "archive-master", "%s.xz" % path)
+
+# image-master keyring
+if os.path.exists(os.path.join(conf.gpg_key_path, "image-master")) and \
+        os.path.exists(os.path.join(conf.gpg_key_path, "archive-master")):
+    image_master = gpg.Keyring(conf, "image-master")
+    image_master.set_metadata("image-master")
+    image_master.import_keys(os.path.join(conf.gpg_key_path, "image-master"))
+    path = image_master.generate_tarball()
+    tools.xz_compress(path)
+    os.remove(path)
+    gpg.sign_file(conf, "archive-master", "%s.xz" % path)
+
+# image-signing keyring
+if os.path.exists(os.path.join(conf.gpg_key_path, "image-signing")) and \
+        os.path.exists(os.path.join(conf.gpg_key_path, "image-master")):
+    image_signing = gpg.Keyring(conf, "image-signing")
+    image_signing.set_metadata("image-signing",
+                               int(time.strftime("%s",
+                                                 time.localtime())) + 63072000)
+    image_signing.import_keys(os.path.join(conf.gpg_key_path, "image-signing"))
+    path = image_signing.generate_tarball()
+    tools.xz_compress(path)
+    os.remove(path)
+    gpg.sign_file(conf, "image-master", "%s.xz" % path)
+
+# device-signing keyring
+if os.path.exists(os.path.join(conf.gpg_key_path, "device-signing")) and \
+        os.path.exists(os.path.join(conf.gpg_key_path, "image-signing")):
+    device_signing = gpg.Keyring(conf, "device-signing")
+    device_signing.set_metadata("device-signing",
+                                int(time.strftime("%s",
+                                                  time.localtime())) + 2678400)
+    device_signing.import_keys(os.path.join(conf.gpg_key_path,
+                                            "device-signing"))
+    path = device_signing.generate_tarball()
+    tools.xz_compress(path)
+    os.remove(path)
+    gpg.sign_file(conf, "image-signing", "%s.xz" % path)
+
+# blacklist keyring
+if os.path.exists(os.path.join(conf.gpg_key_path, "blacklist")) and \
+        os.path.exists(os.path.join(conf.gpg_key_path, "image-master")):
+    blacklist = gpg.Keyring(conf, "blacklist")
+    blacklist.set_metadata("blacklist")
+    path = blacklist.generate_tarball()
+    tools.xz_compress(path)
+    os.remove(path)
+    gpg.sign_file(conf, "image-master", "%s.xz" % path)

=== added file 'bin/generate-keys'
--- bin/generate-keys	1970-01-01 00:00:00 +0000
+++ bin/generate-keys	2014-10-10 11:11:17 +0000
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Canonical Ltd.
+# Author: Timothy Chavez <[e-mail]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, 'lib')
+from systemimage import config
+from systemimage.gpg import generate_signing_key
+
+
+KEYS = {
+    "archive-master": ("{0} Archive Master key", 0),
+    "image-master": ("{0} Image Master key", 0),
+    "device-signing": ("{0} Device Signing key", "2y"),
+    "image-signing": ("{0} Image Signing key", "2y")
+}
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate signing keya.')
+    parser.add_argument("--email", dest="email", required=True,
+                        help="An email address to associate with the keys")
+    parser.add_argument("--prefix", dest="prefix", required=True,
+                        help="A prefix to include in the key name")
+    args = parser.parse_args()
+
+    conf = config.Config()
+
+    print("I: Generating signing keys...")
+
+    for key_id, (key_name, key_expiry) in KEYS.iteritems():
+        key_path = os.path.join(conf.gpg_key_path, key_id)
+        if os.path.exists(key_path):
+            print("W: The key \"{0}\" already exists".format(key_id))
+            continue
+        os.makedirs(key_path)
+        generate_signing_key(
+            key_path, key_name.format(args.prefix), args.email, key_expiry)
+
+    print("I: Done")
+
+
+if __name__ == "__main__":
+    main()

=== added file 'bin/import-images'
--- bin/import-images	1970-01-01 00:00:00 +0000
+++ bin/import-images	2014-10-10 11:11:17 +0000
@@ -0,0 +1,305 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+sys.path.insert(0, os.path.join(sys.path[0], os.pardir, "lib"))
+
+from systemimage import config, generators, tools, tree
+
+import argparse
+import fcntl
+import logging
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="image importer")
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+    args = parser.parse_args()
+
+    # Setup logging
+    formatter = logging.Formatter(
+        "%(asctime)s %(levelname)s %(message)s")
+
+    levels = {1: logging.ERROR,
+              2: logging.WARNING,
+              3: logging.INFO,
+              4: logging.DEBUG}
+
+    if args.verbose > 0:
+        stdoutlogger = logging.StreamHandler(sys.stdout)
+        stdoutlogger.setFormatter(formatter)
+        logging.root.setLevel(levels[min(4, args.verbose)])
+        logging.root.addHandler(stdoutlogger)
+    else:
+        logging.root.addHandler(logging.NullHandler())
+
+    # Load the configuration
+    conf = config.Config()
+
+    # Try to acquire a global lock
+    lock_file = os.path.join(conf.state_path, "global.lock")
+    lock_fd = open(lock_file, 'w')
+
+    try:
+        fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except IOError:
+        logging.info("Something else holds the global lock. exiting.")
+        sys.exit(0)
+
+    # Load the tree
+    pub = tree.Tree(conf)
+
+    # Iterate through the channels
+    for channel_name, channel in conf.channels.items():
+        # We're only interested in automated channels
+        if channel.type != "auto":
+            logging.debug("Skipping non-auto channel: %s" % channel_name)
+            continue
+
+        logging.info("Processing channel: %s" % channel_name)
+
+        # Check the channel exists
+        if channel_name not in pub.list_channels():
+            logging.error("Invalid channel name: %s" % channel_name)
+            continue
+
+        # Iterate through the devices
+        for device_name in pub.list_channels()[channel_name]['devices']:
+            logging.info("Processing device: %s" % device_name)
+
+            device = pub.get_device(channel_name, device_name)
+
+            # Extract last full version
+            full_images = {image['version']: image
+                           for image in device.list_images()
+                           if image['type'] == "full"}
+
+            last_full = None
+            if full_images:
+                last_full = sorted(full_images.values(),
+                                   key=lambda image: image['version'])[-1]
+                logging.debug("Last full image: %s" % last_full['version'])
+            else:
+                logging.debug("This is the first full image.")
+
+            # Extract all delta base versions
+            delta_base = []
+
+            for base_channel in channel.deltabase:
+                # Skip missing channels
+                if base_channel not in pub.list_channels():
+                    logging.warn("Invalid base channel: %s" % base_channel)
+                    continue
+
+                # Skip missing devices
+                if device_name not in (pub.list_channels()
+                                       [base_channel]['devices']):
+                    logging.warn("Missing device in base channel: %s in %s" %
+                                 (device_name, base_channel))
+                    continue
+
+                # Extract the latest full image
+                base_device = pub.get_device(base_channel, device_name)
+                base_images = sorted([image
+                                      for image in base_device.list_images()
+                                      if image['type'] == "full"],
+                                     key=lambda image: image['version'])
+
+                # Check if the version is valid and add it
+                if base_images and base_images[-1]['version'] in full_images:
+                    if (full_images[base_images[-1]['version']]
+                            not in delta_base):
+                        delta_base.append(full_images
+                                          [base_images[-1]['version']])
+                        logging.debug("Source version for delta: %s" %
+                                      base_images[-1]['version'])
+
+            # Allocate new version number
+            new_version = channel.versionbase
+            if last_full:
+                new_version = last_full['version'] + 1
+            logging.debug("Version for next image: %s" % new_version)
+
+            # And the list used to generate version_detail
+            version_detail = []
+
+            # And a list of new files
+            new_files = []
+
+            # Keep track of what files we've processed
+            processed_files = []
+
+            # Create new empty entries
+            new_images = {}
+            new_images['full'] = {'files': []}
+            for delta in delta_base:
+                new_images["delta_%s" % delta['version']] = {'files': []}
+
+            # Iterate through the files
+            for file_entry in channel.files:
+                # Deal with device specific overrides
+                if "," in file_entry['name']:
+                    file_name, file_device = file_entry['name'].split(',', 1)
+                    if file_device != device_name:
+                        logging.debug("Skipping '%s' because the device name"
+                                      "doesn't match" % file_entry['name'])
+                        continue
+                else:
+                    file_name = file_entry['name']
+
+                if file_name in processed_files:
+                    logging.debug("Skipping '%s' because a more specific"
+                                  "generator was already called."
+                                  % file_entry['name'])
+                    continue
+
+                processed_files.append(file_name)
+
+                # Generate the environment
+                environment = {}
+                environment['channel_name'] = channel_name
+                environment['device'] = device
+                environment['device_name'] = device_name
+                environment['version'] = new_version
+                environment['version_detail'] = version_detail
+                environment['new_files'] = new_files
+
+                # Call file generator
+                logging.info("Calling '%s' generator for a new file"
+                             % file_entry['generator'])
+                path = generators.generate_file(conf,
+                                                file_entry['generator'],
+                                                file_entry['arguments'],
+                                                environment)
+
+                # Generators are allowed to return None when no build
+                # exists at all. This cancels the whole image.
+                if not path:
+                    new_files = []
+                    logging.info("No image will be produced because the "
+                                 "'%s' generator returned None" %
+                                 file_entry['generator'])
+                    break
+
+                # Get the full and relative paths
+                abspath, relpath = tools.expand_path(path, conf.publish_path)
+                urlpath = "/%s" % "/".join(relpath.split(os.sep))
+
+                # FIXME: Extract the prefix, used later for matching between
+                #        full images. This forces a specific filename format.
+                prefix = abspath.split("/")[-1].rsplit("-", 1)[0]
+
+                # Add the file to the full image
+                new_images['full']['files'].append(abspath)
+
+                # Check if same as current
+                new_file = True
+                if last_full:
+                    for file_dict in last_full['files']:
+                        if file_dict['path'] == urlpath:
+                            new_file = False
+                            break
+
+                if new_file:
+                    logging.info("New file from '%s': %s" %
+                                 (file_entry['generator'], relpath))
+                    new_files.append(abspath)
+                else:
+                    logging.info("File from '%s' is already current" %
+                                 (file_entry['generator']))
+
+                # Generate deltas
+                for delta in delta_base:
+                    # Extract the source
+                    src_path = None
+                    for file_dict in delta['files']:
+                        if (file_dict['path'].split("/")[-1]
+                                .startswith(prefix)):
+                            src_path = "%s/%s" % (conf.publish_path,
+                                                  file_dict['path'])
+                            break
+
+                    # Check that it's not the current file
+                    if src_path:
+                        src_path = os.path.realpath(src_path)
+
+                        # FIXME: the keyring- is a big hack...
+                        if src_path == abspath and "keyring-" not in src_path:
+                            continue
+
+                        # Generators are allowed to return None when no delta
+                        # exists at all.
+                        logging.info("Generating delta from '%s' for '%s'" %
+                                     (delta['version'],
+                                      file_entry['generator']))
+                        delta_path = generators.generate_delta(conf, src_path,
+                                                               abspath)
+                    else:
+                        delta_path = abspath
+
+                    if not delta_path:
+                        continue
+
+                    # Get the full and relative paths
+                    delta_abspath, delta_relpath = tools.expand_path(
+                        delta_path, conf.publish_path)
+
+                    new_images['delta_%s' % delta['version']]['files'] \
+                        .append(delta_abspath)
+
+            # Check if we've got a new image
+            if len(new_files):
+                # Publish full image
+                logging.info("Publishing new image '%s' (%s) with %s files."
+                             % (new_version,
+                                ",".join(environment['version_detail']),
+                                len(new_images['full']['files'])))
+                device.create_image("full", new_version,
+                                    ",".join(environment['version_detail']),
+                                    new_images['full']['files'])
+                # Publish deltas
+                for delta in delta_base:
+                    files = new_images["delta_%s" % delta['version']]['files']
+                    logging.info("Publishing new delta from '%s' (%s)"
+                                 " to '%s' (%s) with %s files" %
+                                 (delta['version'],
+                                  delta.get("description", ""),
+                                  new_version,
+                                  ",".join(environment['version_detail']),
+                                  len(files)))
+                    device.create_image(
+                        "delta", new_version,
+                        ",".join(environment['version_detail']), files,
+                        base=delta['version'])
+
+            # Expire images
+            if channel.fullcount > 0:
+                logging.info("Expiring old images")
+                device.expire_images(channel.fullcount)
+
+        # Sync all channel aliases
+        logging.info("Syncing any existing alias")
+        pub.sync_aliases(channel_name)
+
+    # Remove any orphaned file
+    logging.info("Removing orphaned files from the pool")
+    pub.cleanup_tree()
+
+    # Sync the mirrors
+    logging.info("Triggering a mirror sync")
+    tools.sync_mirrors(conf)

=== added file 'bin/set-phased-percentage'
--- bin/set-phased-percentage	1970-01-01 00:00:00 +0000
+++ bin/set-phased-percentage	2014-10-10 11:11:17 +0000
@@ -0,0 +1,90 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+sys.path.insert(0, os.path.join(sys.path[0], os.pardir, "lib"))
+
+from systemimage import config, tools, tree
+
+import argparse
+import logging
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="set phased percentage")
+    parser.add_argument("channel", metavar="CHANNEL")
+    parser.add_argument("device", metavar="DEVICE")
+    parser.add_argument("version", metavar="VERSION", type=int)
+    parser.add_argument("percentage", metavar="PERCENTAGE", type=int)
+    parser.add_argument("--verbose", "-v", action="count")
+
+    args = parser.parse_args()
+
+    # Setup logging
+    formatter = logging.Formatter(
+        "%(asctime)s %(levelname)s %(message)s")
+
+    levels = {1: logging.ERROR,
+              2: logging.WARNING,
+              3: logging.INFO,
+              4: logging.DEBUG}
+
+    if args.verbose > 0:
+        stdoutlogger = logging.StreamHandler(sys.stdout)
+        stdoutlogger.setFormatter(formatter)
+        logging.root.setLevel(levels[min(4, args.verbose)])
+        logging.root.addHandler(stdoutlogger)
+    else:
+        logging.root.addHandler(logging.NullHandler())
+
+    # Load the configuration
+    conf = config.Config()
+
+    # Load the tree
+    pub = tree.Tree(conf)
+
+    # Do some checks
+    if args.channel not in pub.list_channels():
+        parser.error("Invalid channel: %s" % args.channel)
+
+    if args.device not in pub.list_channels()[args.channel]['devices']:
+        parser.error("Invalid device for source channel: %s" %
+                     args.device)
+
+    if args.percentage < 0 or args.percentage > 100:
+        parser.error("Invalid value: %s" % args.percentage)
+
+    if "alias" in pub.list_channels()[args.channel] and \
+            pub.list_channels()[args.channel]['alias'] != args.channel:
+        parser.error("Channel is an alias.")
+
+    if "redirect" in pub.list_channels()[args.channel]:
+        parser.error("Channel is a redirect.")
+
+    dev = pub.get_device(args.channel, args.device)
+    logging.info("Setting phased-percentage of '%s' to %s%%" %
+                 (args.version, args.percentage))
+    dev.set_phased_percentage(args.version, args.percentage)
+
+    # Sync all channel aliases
+    logging.info("Syncing any existing alias")
+    pub.sync_aliases(args.channel)
+
+    # Sync the mirrors
+    logging.info("Triggering a mirror sync")
+    tools.sync_mirrors(conf)

=== added file 'bin/si-shell'
--- bin/si-shell	1970-01-01 00:00:00 +0000
+++ bin/si-shell	2014-10-10 11:11:17 +0000
@@ -0,0 +1,79 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import code
+import logging
+import os
+import sys
+sys.path.insert(0, os.path.join(sys.path[0], os.pardir, "lib"))
+
+from systemimage import config, tree
+
+import argparse
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description="system-image shell")
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+
+    args = parser.parse_args()
+
+    # Setup logging
+    formatter = logging.Formatter(
+        "%(asctime)s %(levelname)s %(message)s")
+
+    levels = {1: logging.ERROR,
+              2: logging.WARNING,
+              3: logging.INFO,
+              4: logging.DEBUG}
+
+    if args.verbose > 0:
+        stdoutlogger = logging.StreamHandler(sys.stdout)
+        stdoutlogger.setFormatter(formatter)
+        stdoutlogger.setLevel(levels[min(4, args.verbose)])
+        logging.root.addHandler(stdoutlogger)
+    else:
+        logging.root.addHandler(logging.NullHandler())
+
+    # Load the configuration
+    conf = config.Config()
+
+    # Load the tree
+    pub = tree.Tree(conf)
+
+    # Start the shell
+    banner = """Welcome to the system-image shell.
+The configuration is available as: conf
+The system-image tree is availabe as: pub
+"""
+
+    class CompleterConsole(code.InteractiveConsole):
+        def __init__(self):
+            local = {'conf': conf,
+                     'pub': pub}
+            code.InteractiveConsole.__init__(self, locals=local)
+            try:
+                import readline
+            except ImportError:
+                print('I: readline module not available.')
+            else:
+                import rlcompleter
+                rlcompleter  # Silence pyflakes
+                readline.parse_and_bind("tab: complete")
+
+    console = CompleterConsole()
+    console.interact(banner)

=== added directory 'etc'
=== added file 'etc/config.example'
--- etc/config.example	1970-01-01 00:00:00 +0000
+++ etc/config.example	2014-10-10 11:11:17 +0000
@@ -0,0 +1,48 @@
+[global]
+base_path = /some/fs/path
+channels = trusty, trusty-proposed, trusty-customized
+gpg_key_path = secret/gpg/keys/
+gpg_keyring_path = secret/gpg/keyrings/
+publish_path = www/
+state_path = state/
+mirrors = a, b
+public_fqdn = system-image.example.net
+public_http_port = 80
+public_https_port = 443
+
+[channel_trusty]
+type = manual
+versionbase = 1
+fullcount = 10
+
+[channel_trusty-proposed]
+type = auto
+versionbase = 1
+fullcount = 20
+deltabase = trusty, trusty-proposed
+files = ubuntu, device, version
+file_ubuntu = cdimage-ubuntu;daily-preinstalled;trusty,import=any
+file_device = cdimage-device;daily-preinstalled;trusty,import=any
+file_version = version
+
+[channel_trusty-customized]
+type = auto
+versionbase = 1
+fullcount = 15
+files = ubuntu, device, custom, version
+file_ubuntu = system-image;trusty;file=ubuntu
+file_device = system-image;trusty;file=device
+file_custom = http;http://www.example.net/custom/custom.tar.xz;name=custom,monitor=http://www.example.net/custom/build_number
+file_version = version
+
+[mirror_default]
+ssh_user = mirror
+ssh_key = secret/ssh/mirror
+ssh_port = 22
+ssh_command = sync-mirror
+
+[mirror_a]
+ssh_host = a.example.com
+
+[mirror_b]
+ssh_host = b.example.com

=== added directory 'lib'
=== added directory 'lib/systemimage'
=== added file 'lib/systemimage/__init__.py'
=== added file 'lib/systemimage/config.py'
--- lib/systemimage/config.py	1970-01-01 00:00:00 +0000
+++ lib/systemimage/config.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+try:
+    from configparser import ConfigParser
+except ImportError:  # pragma: no cover
+    from ConfigParser import ConfigParser
+
+
+def parse_config(path):
+    config = {}
+
+    configp = ConfigParser()
+    try:
+        configp.read(path)
+    except:
+        return config
+
+    for section in configp.sections():
+        config_section = {}
+        for option in configp.options(section):
+            value = configp.get(section, option)
+            if ", " in value:
+                value = [entry.strip('"').strip()
+                         for entry in value.split(", ")]
+            else:
+                value = value.strip('"').strip()
+            config_section[option] = value
+        config[section] = config_section
+
+    return config
+
+
+class Config:
+    def __init__(self, path=None):
+        if not path:
+            path = "%s/etc/config" % os.environ.get("SYSTEM_IMAGE_ROOT",
+                                                    os.getcwd())
+            if not os.path.exists(path):
+                path = os.path.realpath(os.path.join(os.path.dirname(__file__),
+                                                     "../../etc/config"))
+
+        self.load_config(path)
+
+    def load_config(self, path):
+        if not os.path.exists(path):
+            raise Exception("Configuration file doesn't exist: %s" % path)
+
+        # Read the config
+        config = parse_config(path)
+
+        if 'global' not in config:
+            config['global'] = {}
+
+        # Set defaults
+        self.base_path = config['global'].get(
+            "base_path", os.environ.get("SYSTEM_IMAGE_ROOT", os.getcwd()))
+
+        self.gpg_key_path = config['global'].get(
+            "gpg_key_path", os.path.join(self.base_path,
+                                         "secret", "gpg", "keys"))
+        if not self.gpg_key_path.startswith("/"):
+            self.gpg_key_path = os.path.join(self.base_path, self.gpg_key_path)
+
+        self.gpg_keyring_path = config['global'].get(
+            "gpg_keyring_path", os.path.join(self.base_path,
+                                             "secret", "gpg", "keyrings"))
+        if not self.gpg_keyring_path.startswith("/"):
+            self.gpg_keyring_path = os.path.join(self.base_path,
+                                                 self.gpg_keyring_path)
+
+        self.publish_path = config['global'].get(
+            "publish_path", os.path.join(self.base_path, "www"))
+        if not self.publish_path.startswith("/"):
+            self.publish_path = os.path.join(self.base_path, self.publish_path)
+
+        self.state_path = config['global'].get(
+            "state_path", os.path.join(self.base_path, "state"))
+        if not self.state_path.startswith("/"):
+            self.state_path = os.path.join(self.base_path, self.state_path)
+
+        # Export some more keys as-is
+        for key in ("public_fqdn", "public_http_port", "public_https_port"):
+            if key not in config['global']:
+                continue
+
+            setattr(self, key, config['global'][key])
+
+        # Parse the mirror configuration
+        self.mirrors = {}
+        if "mirrors" in config['global']:
+            if not isinstance(config['global']['mirrors'], list):
+                config['global']['mirrors'] = [config['global']['mirrors']]
+
+            if len(config['global']['mirrors']) != 0:
+                if "mirror_default" not in config:
+                    raise KeyError("Missing mirror_default section.")
+
+                for key in ("ssh_user", "ssh_key", "ssh_port", "ssh_command"):
+                    if key not in config['mirror_default']:
+                        raise KeyError("Missing key in mirror_default: %s" %
+                                       key)
+
+                for entry in config['global']['mirrors']:
+                    dict_entry = "mirror_%s" % entry
+                    if dict_entry not in config:
+                        raise KeyError("Missing mirror section: %s" %
+                                       dict_entry)
+
+                    mirror = type("Mirror", (object,), {})
+
+                    if "ssh_host" not in config[dict_entry]:
+                        raise KeyError("Missing key in %s: ssh_host" %
+                                       dict_entry)
+                    else:
+                        mirror.ssh_host = config[dict_entry]['ssh_host']
+
+                    mirror.ssh_user = config[dict_entry].get(
+                        "ssh_user", config['mirror_default']['ssh_user'])
+                    mirror.ssh_key = config[dict_entry].get(
+                        "ssh_key", config['mirror_default']['ssh_key'])
+                    if not mirror.ssh_key.startswith("/"):
+                        mirror.ssh_key = os.path.join(self.base_path,
+                                                      mirror.ssh_key)
+                    mirror.ssh_port = int(config[dict_entry].get(
+                        "ssh_port", config['mirror_default']['ssh_port']))
+                    mirror.ssh_command = config[dict_entry].get(
+                        "ssh_command", config['mirror_default']['ssh_command'])
+
+                    self.mirrors[entry] = mirror
+
+        # Parse the channel configuration
+        self.channels = {}
+        if "channels" in config['global']:
+            if not isinstance(config['global']['channels'], list):
+                config['global']['channels'] = \
+                    [config['global']['channels']]
+
+            if len(config['global']['channels']) != 0:
+                for entry in config['global']['channels']:
+                    dict_entry = "channel_%s" % entry
+                    if dict_entry not in config:
+                        raise KeyError("Missing channel section: %s" %
+                                       dict_entry)
+
+                    channel = type("Channel", (object,), {})
+
+                    channel.versionbase = int(config[dict_entry].get(
+                        'versionbase', 1))
+
+                    channel.type = config[dict_entry].get(
+                        "type", "manual")
+
+                    channel.fullcount = int(config[dict_entry].get(
+                        "fullcount", 0))
+
+                    channel.deltabase = [entry]
+                    if "deltabase" in config[dict_entry]:
+                        if isinstance(config[dict_entry]["deltabase"],
+                                      list):
+                            channel.deltabase = \
+                                config[dict_entry]["deltabase"]
+                        else:
+                            channel.deltabase = \
+                                [config[dict_entry]["deltabase"]]
+
+                    # Parse the file list
+                    files = config[dict_entry].get("files", [])
+                    if isinstance(files, str):
+                        files = [files]
+
+                    channel.files = []
+                    for file_entry in files:
+                        if "file_%s" % file_entry not in config[dict_entry]:
+                            raise KeyError("Missing file entry: %s" %
+                                           "file_%s" % file_entry)
+
+                        fields = (config[dict_entry]
+                                  ["file_%s" % file_entry].split(";"))
+
+                        file_dict = {}
+                        file_dict['name'] = file_entry
+                        file_dict['generator'] = fields[0]
+                        file_dict['arguments'] = []
+                        if len(fields) > 1:
+                            file_dict['arguments'] = fields[1:]
+
+                        channel.files.append(file_dict)
+
+                    self.channels[entry] = channel

=== added file 'lib/systemimage/diff.py'
--- lib/systemimage/diff.py	1970-01-01 00:00:00 +0000
+++ lib/systemimage/diff.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import tarfile
+import time
+
+from io import BytesIO
+
+
+def compare_files(fd_source, fd_target):
+    """
+        Compare two files.
+
+        Returns True if their content matches.
+        Returns False if they don't match.
+        Returns None if the files can't be compared.
+    """
+
+    if fd_source == fd_target:
+        return True
+
+    if not fd_source or not fd_target:
+        return False
+
+    return fd_source.read() == fd_target.read()
+
+
+def list_tarfile(tarfile):
+    """
+        Walk through a tarfile and generate a list of the content.
+
+        Returns a tuple containing a set and a dict.
+        The set is typically used for simple diffs between tarballs.
+        The dict is used to easily grab the details of a specific entry.
+    """
+
+    set_content = set()
+    dict_content = {}
+
+    for entry in tarfile:
+        if entry.isdir():
+            set_content.add((entry.path, 'dir', None))
+            dict_content[entry.path] = ('dir', None)
+        else:
+            fhash = ("%s" % entry.mode,
+                     "%s" % entry.devmajor,
+                     "%s" % entry.devminor,
+                     "%s" % entry.type.decode('utf-8'),
+                     "%s" % entry.uid,
+                     "%s" % entry.gid,
+                     "%s" % entry.size,
+                     "%s" % entry.mtime)
+
+            set_content.add((entry.path, 'file', fhash))
+            dict_content[entry.path] = ('file', fhash)
+
+    return (set_content, dict_content)
+
+
+class ImageDiff:
+    source_content = None
+    target_content = None
+    diff = None
+
+    def __init__(self, source, target):
+        self.source_file = tarfile.open(source, 'r:')
+        self.target_file = tarfile.open(target, 'r:')
+
+    def scan_content(self, image):
+        """
+            Scan the content of an image and return the image tuple.
+            This also caches the content for further use.
+        """
+
+        if image not in ("source", "target"):
+            raise KeyError("Invalid image '%s'." % image)
+
+        image_file = getattr(self, "%s_file" % image)
+
+        content = list_tarfile(image_file)
+
+        setattr(self, "%s_content" % image, content)
+        return content
+
+    def compare_images(self):
+        """
+            Compare the file listing of two images and return a set.
+            This also caches the diff for further use.
+
+            The set contains tuples of (path, changetype).
+        """
+        if not self.source_content:
+            self.scan_content("source")
+
+        if not self.target_content:
+            self.scan_content("target")
+
+        # Find the changes in the two trees
+        changes = set()
+        for change in self.source_content[0] \
+                .symmetric_difference(self.target_content[0]):
+            if change[0] not in self.source_content[1]:
+                changetype = "add"
+            elif change[0] not in self.target_content[1]:
+                changetype = "del"
+            else:
+                changetype = "mod"
+            changes.add((change[0], changetype))
+
+        # Ignore files that only vary in mtime
+        # (separate loop to run after de-dupe)
+        for change in sorted(changes):
+            if change[1] == "mod":
+                fstat_source = self.source_content[1][change[0]][1]
+                fstat_target = self.target_content[1][change[0]][1]
+
+                # Skip differences between directories and files
+                if not fstat_source or not fstat_target:  # pragma: no cover
+                    continue
+
+                # Deal with switched hardlinks
+                if (fstat_source[0:2] == fstat_target[0:2] and
+                        fstat_source[3] != fstat_target[3] and
+                        (fstat_source[3] == "1" or fstat_target[3] == "1") and
+                        fstat_source[4:5] == fstat_target[4:5] and
+                        fstat_source[7] == fstat_target[7]):
+                    source_file = self.source_file.getmember(change[0])
+                    target_file = self.target_file.getmember(change[0])
+                    if compare_files(
+                            self.source_file.extractfile(change[0]),
+                            self.target_file.extractfile(change[0])):
+                        changes.remove(change)
+                        continue
+
+                # Deal with regular files
+                if fstat_source[0:7] == fstat_target[0:7]:
+                    source_file = self.source_file.getmember(change[0])
+                    target_file = self.target_file.getmember(change[0])
+
+                    if (source_file.linkpath
+                            and source_file.linkpath == target_file.linkpath):
+                        changes.remove(change)
+                        continue
+
+                    if (source_file.isfile() and target_file.isfile()
+                            and compare_files(
+                                self.source_file.extractfile(change[0]),
+                                self.target_file.extractfile(change[0]))):
+                        changes.remove(change)
+                        continue
+
+        self.diff = changes
+        return changes
+
+    def print_changes(self):
+        """
+            Simply print the list of changes.
+        """
+
+        if not self.diff:
+            self.compare_images()
+
+        for change in sorted(self.diff):
+            print(" - %s (%s)" % (change[0], change[1]))
+
+    def generate_diff_tarball(self, path):
+        """
+            Generate a tarball containing all files that are
+            different between the source and target iamge as well
+            as a file listing all removals.
+        """
+
+        if not self.diff:
+            self.compare_images()
+
+        output = tarfile.open(path, 'w:')
+
+        # List both deleted files and modified files in the removal list
+        # that's needed to allow file type change (e.g. directory to symlink)
+        removed_files_list = [entry[0] for entry in self.diff
+                              if entry[1] in ("del", "mod")]
+
+        removed_files = "\n".join(removed_files_list)
+        removed_files = "%s\n" % removed_files.encode('utf-8')
+
+        removals = tarfile.TarInfo()
+        removals.name = "removed"
+        removals.size = len(removed_files)
+        removals.mtime = int(time.strftime("%s", time.localtime()))
+        removals.uname = "root"
+        removals.gname = "root"
+
+        output.addfile(removals, BytesIO(removed_files.encode('utf-8')))
+
+        # Copy all the added and modified
+        added = []
+        for name, action in sorted(self.diff):
+            if action == 'del':
+                continue
+
+            if name in added:
+                continue
+
+            newfile = self.target_file.getmember(name)
+            if newfile.islnk():
+                if newfile.linkname.startswith("system/"):
+                    targetfile_path = newfile.linkname
+                else:
+                    targetfile_path = os.path.normpath(os.path.join(
+                        os.path.dirname(newfile.name), newfile.linkname))
+
+                targetfile = self.target_file.getmember(targetfile_path)
+
+                if ((targetfile_path, 'add') in self.diff or
+                        (targetfile_path, 'mod') in self.diff) and \
+                        targetfile_path not in added:
+                    fileptr = self.target_file.extractfile(targetfile)
+                    output.addfile(targetfile, fileptr)
+                    added.append(targetfile.name)
+
+            fileptr = None
+            if newfile.isfile():
+                fileptr = self.target_file.extractfile(name)
+            output.addfile(newfile, fileobj=fileptr)
+            added.append(newfile.name)
+
+        output.close()

=== added file 'lib/systemimage/generators.py'
--- lib/systemimage/generators.py	1970-01-01 00:00:00 +0000
+++ lib/systemimage/generators.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,1173 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from hashlib import sha256
+from systemimage import diff, gpg, tree, tools
+import json
+import os
+import socket
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import time
+
+try:
+    from urllib.request import urlopen, urlretrieve
+except ImportError:  # pragma: no cover
+    from urllib import urlopen, urlretrieve
+
+# Global
+CACHE = {}
+
+
+def root_ownership(tarinfo):
+    tarinfo.mode = 0o644
+    tarinfo.mtime = int(time.strftime("%s", time.localtime()))
+    tarinfo.uname = "root"
+    tarinfo.gname = "root"
+    return tarinfo
+
+
+def unpack_arguments(arguments):
+    """
+        Takes a string representing comma separate key=value options and
+        returns a dict.
+    """
+    arg_dict = {}
+
+    for option in arguments.split(","):
+        fields = option.split("=")
+        if len(fields) != 2:
+            continue
+
+        arg_dict[fields[0]] = fields[1]
+
+    return arg_dict
+
+
+def generate_delta(conf, source_path, target_path):
+    """
+        Take two .tar.xz file and generate a third file, stored in the pool.
+        The path to the pool file is then returned and <path>.asc is also
+        generated using the default signing key.
+    """
+    source_filename = source_path.split("/")[-1].replace(".tar.xz", "")
+    target_filename = target_path.split("/")[-1].replace(".tar.xz", "")
+
+    # FIXME: This is a bit of an hack, it'd be better not to have to hardcode
+    #        that kind of stuff...
+    if (source_filename.startswith("version-")
+            and target_filename.startswith("version-")):
+        return target_path
+
+    if (source_filename.startswith("keyring-")
+            and target_filename.startswith("keyring-")):
+        return target_path
+
+    # Now for everything else
+    path = os.path.realpath(os.path.join(conf.publish_path, "pool",
+                                         "%s.delta-%s.tar.xz" %
+                                         (target_filename, source_filename)))
+
+    # Return pre-existing entries
+    if os.path.exists(path):
+        return path
+
+    # Create the pool if it doesn't exist
+    if not os.path.exists(os.path.join(conf.publish_path, "pool")):
+        os.makedirs(os.path.join(conf.publish_path, "pool"))
+
+    # Generate the diff
+    tempdir = tempfile.mkdtemp()
+    tools.xz_uncompress(source_path, os.path.join(tempdir, "source.tar"))
+    tools.xz_uncompress(target_path, os.path.join(tempdir, "target.tar"))
+
+    imagediff = diff.ImageDiff(os.path.join(tempdir, "source.tar"),
+                               os.path.join(tempdir, "target.tar"))
+
+    imagediff.generate_diff_tarball(os.path.join(tempdir, "output.tar"))
+    tools.xz_compress(os.path.join(tempdir, "output.tar"), path)
+    shutil.rmtree(tempdir)
+
+    # Sign the result
+    gpg.sign_file(conf, "image-signing", path)
+
+    # Generate the metadata file
+    metadata = {}
+    metadata['generator'] = "delta"
+    metadata['source'] = {}
+    metadata['target'] = {}
+
+    if os.path.exists(source_path.replace(".tar.xz", ".json")):
+        with open(source_path.replace(".tar.xz", ".json"), "r") as fd:
+            metadata['source'] = json.loads(fd.read())
+
+    if os.path.exists(target_path.replace(".tar.xz", ".json")):
+        with open(target_path.replace(".tar.xz", ".json"), "r") as fd:
+            metadata['target'] = json.loads(fd.read())
+
+    with open(path.replace(".tar.xz", ".json"), "w+") as fd:
+        fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
+                                     indent=4, separators=(',', ': ')))
+    gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
+
+    return path
+
+
+def generate_file(conf, generator, arguments, environment):
+    """
+        Dispatcher for the various generators and importers.
+        It calls the right generator and signs the generated file
+        before returning the path.
+    """
+
+    if generator == "version":
+        path = generate_file_version(conf, arguments, environment)
+    elif generator == "cdimage-device":
+        path = generate_file_cdimage_device(conf, arguments, environment)
+    elif generator == "cdimage-ubuntu":
+        path = generate_file_cdimage_ubuntu(conf, arguments, environment)
+    elif generator == "cdimage-custom":
+        path = generate_file_cdimage_custom(conf, arguments, environment)
+    elif generator == "http":
+        path = generate_file_http(conf, arguments, environment)
+    elif generator == "keyring":
+        path = generate_file_keyring(conf, arguments, environment)
+    elif generator == "system-image":
+        path = generate_file_system_image(conf, arguments, environment)
+    elif generator == "remote-system-image":
+        path = generate_file_remote_system_image(conf, arguments, environment)
+    else:
+        raise Exception("Invalid generator: %s" % generator)
+
+    return path
+
+
+def generate_file_cdimage_device(conf, arguments, environment):
+    """
+        Scan a cdimage tree for new device files.
+    """
+
+    # We need at least a path and a series
+    if len(arguments) < 2:
+        return None
+
+    # Read the arguments
+    cdimage_path = arguments[0]
+    series = arguments[1]
+
+    options = {}
+    if len(arguments) > 2:
+        options = unpack_arguments(arguments[2])
+
+    boot_arch = "armhf"
+    recovery_arch = "armel"
+    system_arch = "armel"
+    if environment['device_name'] in ("generic_x86", "generic_i386"):
+        boot_arch = "i386"
+        recovery_arch = "i386"
+        system_arch = "i386"
+    elif environment['device_name'] in ("generic_amd64",):
+        boot_arch = "amd64"
+        recovery_arch = "amd64"
+        system_arch = "amd64"
+
+    # Check that the directory exists
+    if not os.path.exists(cdimage_path):
+        return None
+
+    versions = sorted([version for version in os.listdir(cdimage_path)
+                       if version not in ("pending", "current")],
+                      reverse=True)
+
+    for version in versions:
+        # Skip directory without checksums
+        if not os.path.exists(os.path.join(cdimage_path, version,
+                                           "SHA256SUMS")):
+            continue
+
+        # Check for all the needed files
+        boot_path = os.path.join(cdimage_path, version,
+                                 "%s-preinstalled-boot-%s+%s.img" %
+                                 (series, boot_arch,
+                                  environment['device_name']))
+        if not os.path.exists(boot_path):
+            continue
+
+        recovery_path = os.path.join(cdimage_path, version,
+                                     "%s-preinstalled-recovery-%s+%s.img" %
+                                     (series, recovery_arch,
+                                      environment['device_name']))
+        if not os.path.exists(recovery_path):
+            continue
+
+        system_path = os.path.join(cdimage_path, version,
+                                   "%s-preinstalled-system-%s+%s.img" %
+                                   (series, system_arch,
+                                    environment['device_name']))
+        if not os.path.exists(system_path):
+            continue
+
+        # Check if we should only import tested images
+        if options.get("import", "any") == "good":
+            if not os.path.exists(os.path.join(cdimage_path, version,
+                                               ".marked_good")):
+                continue
+
+        # Set the version_detail string
+        version_detail = "device=%s" % version
+
+        # Extract the hashes
+        boot_hash = None
+        recovery_hash = None
+        system_hash = None
+        with open(os.path.join(cdimage_path, version,
+                               "SHA256SUMS"), "r") as fd:
+            for line in fd:
+                line = line.strip()
+                if line.endswith(boot_path.split("/")[-1]):
+                    boot_hash = line.split()[0]
+                elif line.endswith(recovery_path.split("/")[-1]):
+                    recovery_hash = line.split()[0]
+                elif line.endswith(system_path.split("/")[-1]):
+                    system_hash = line.split()[0]
+
+                if boot_hash and recovery_hash and system_hash:
+                    break
+
+        if not boot_hash or not recovery_hash or not system_hash:
+            continue
+
+        hash_string = "%s/%s/%s" % (boot_hash, recovery_hash, system_hash)
+        global_hash = sha256(hash_string.encode('utf-8')).hexdigest()
+
+        # Generate the path
+        path = os.path.join(conf.publish_path, "pool",
+                            "device-%s.tar.xz" % global_hash)
+
+        # Return pre-existing entries
+        if os.path.exists(path):
+            # Get the real version number (in case it got copied)
+            if os.path.exists(path.replace(".tar.xz", ".json")):
+                with open(path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    version_detail = metadata['version_detail']
+
+            environment['version_detail'].append(version_detail)
+            return path
+
+        temp_dir = tempfile.mkdtemp()
+
+        # Generate a new tarball
+        target_tarball = tarfile.open(os.path.join(temp_dir, "target.tar"),
+                                      "w:")
+
+        # system image
+        # # convert to raw image
+        system_img = os.path.join(temp_dir, "system.img")
+        with open(os.path.devnull, "w") as devnull:
+            subprocess.call(["simg2img", system_path, system_img],
+                            stdout=devnull)
+
+        # # shrink to minimal size
+        with open(os.path.devnull, "w") as devnull:
+            subprocess.call(["resize2fs", "-M", system_img],
+                            stdout=devnull, stderr=devnull)
+
+        # # include in tarball
+        target_tarball.add(system_img,
+                           arcname="system/var/lib/lxc/android/system.img",
+                           filter=root_ownership)
+
+        # boot image
+        target_tarball.add(boot_path, arcname="partitions/boot.img",
+                           filter=root_ownership)
+
+        # recovery image
+        target_tarball.add(recovery_path,
+                           arcname="partitions/recovery.img",
+                           filter=root_ownership)
+
+        target_tarball.close()
+
+        # Create the pool if it doesn't exist
+        if not os.path.exists(os.path.join(conf.publish_path, "pool")):
+            os.makedirs(os.path.join(conf.publish_path, "pool"))
+
+        # Compress the target tarball and sign it
+        tools.xz_compress(os.path.join(temp_dir, "target.tar"), path)
+        gpg.sign_file(conf, "image-signing", path)
+
+        # Generate the metadata file
+        metadata = {}
+        metadata['generator'] = "cdimage-device"
+        metadata['version'] = version
+        metadata['version_detail'] = version_detail
+        metadata['series'] = series
+        metadata['device'] = environment['device_name']
+        metadata['boot_path'] = boot_path
+        metadata['boot_checksum'] = boot_hash
+        metadata['recovery_path'] = recovery_path
+        metadata['recovery_checksum'] = recovery_hash
+        metadata['system_path'] = system_path
+        metadata['system_checksum'] = system_hash
+
+        with open(path.replace(".tar.xz", ".json"), "w+") as fd:
+            fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
+                                         indent=4, separators=(',', ': ')))
+        gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
+
+        # Cleanup
+        shutil.rmtree(temp_dir)
+
+        environment['version_detail'].append(version_detail)
+        return path
+
+    return None
+
+
+def generate_file_cdimage_ubuntu(conf, arguments, environment):
+    """
+        Scan a cdimage tree for new ubuntu files.
+    """
+
+    # We need at least a path and a series
+    if len(arguments) < 2:
+        return None
+
+    # Read the arguments
+    cdimage_path = arguments[0]
+    series = arguments[1]
+
+    options = {}
+    if len(arguments) > 2:
+        options = unpack_arguments(arguments[2])
+
+    arch = "armhf"
+    if environment['device_name'] in ("generic_x86", "generic_i386"):
+        arch = "i386"
+    elif environment['device_name'] in ("generic_amd64",):
+        arch = "amd64"
+
+    # Check that the directory exists
+    if not os.path.exists(cdimage_path):
+        return None
+
+    versions = sorted([version for version in os.listdir(cdimage_path)
+                       if version not in ("pending", "current")],
+                      reverse=True)
+
+    for version in versions:
+        # Skip directory without checksums
+        if not os.path.exists(os.path.join(cdimage_path, version,
+                                           "SHA256SUMS")):
+            continue
+
+        # Check for the rootfs
+        rootfs_path = os.path.join(cdimage_path, version,
+                                   "%s-preinstalled-%s-%s.tar.gz" %
+                                   (series, options.get("product", "touch"),
+                                    arch))
+        if not os.path.exists(rootfs_path):
+            continue
+
+        # Check if we should only import tested images
+        if options.get("import", "any") == "good":
+            if not os.path.exists(os.path.join(cdimage_path, version,
+                                               ".marked_good")):
+                continue
+
+        # Set the version_detail string
+        version_detail = "ubuntu=%s" % version
+
+        # Extract the hash
+        rootfs_hash = None
+        with open(os.path.join(cdimage_path, version,
+                               "SHA256SUMS"), "r") as fd:
+            for line in fd:
+                line = line.strip()
+                if line.endswith(rootfs_path.split("/")[-1]):
+                    rootfs_hash = line.split()[0]
+                    break
+
+        if not rootfs_hash:
+            continue
+
+        # Generate the path
+        path = os.path.join(conf.publish_path, "pool",
+                            "ubuntu-%s.tar.xz" % rootfs_hash)
+
+        # Return pre-existing entries
+        if os.path.exists(path):
+            # Get the real version number (in case it got copied)
+            if os.path.exists(path.replace(".tar.xz", ".json")):
+                with open(path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    version_detail = metadata['version_detail']
+
+            environment['version_detail'].append(version_detail)
+            return path
+
+        temp_dir = tempfile.mkdtemp()
+
+        # Unpack the source tarball
+        tools.gzip_uncompress(rootfs_path, os.path.join(temp_dir,
+                                                        "source.tar"))
+
+        # Generate a new shifted tarball
+        source_tarball = tarfile.open(os.path.join(temp_dir, "source.tar"),
+                                      "r:")
+        target_tarball = tarfile.open(os.path.join(temp_dir, "target.tar"),
+                                      "w:")
+
+        added = []
+        for entry in source_tarball:
+            # FIXME: Will need to be done on the real rootfs
+            # Skip some files
+            if entry.name in ("SWAP.swap", "etc/mtab"):
+                continue
+
+            fileptr = None
+            if entry.isfile():
+                try:
+                    fileptr = source_tarball.extractfile(entry.name)
+                except KeyError:  # pragma: no cover
+                    pass
+
+            # Update hardlinks to point to the right target
+            if entry.islnk():
+                entry.linkname = "system/%s" % entry.linkname
+
+            entry.name = "system/%s" % entry.name
+            target_tarball.addfile(entry, fileobj=fileptr)
+            added.append(entry.name)
+
+        if options.get("product", "touch") == "touch":
+            # FIXME: Will need to be done on the real rootfs
+            # Add some symlinks and directories
+            # # /android
+            new_file = tarfile.TarInfo()
+            new_file.type = tarfile.DIRTYPE
+            new_file.name = "system/android"
+            new_file.mode = 0o755
+            new_file.mtime = int(time.strftime("%s", time.localtime()))
+            new_file.uname = "root"
+            new_file.gname = "root"
+            target_tarball.addfile(new_file)
+
+            # # Android partitions
+            for android_path in ("cache", "data", "factory", "firmware",
+                                 "persist", "system"):
+                new_file = tarfile.TarInfo()
+                new_file.type = tarfile.SYMTYPE
+                new_file.name = "system/%s" % android_path
+                new_file.linkname = "/android/%s" % android_path
+                new_file.mode = 0o755
+                new_file.mtime = int(time.strftime("%s", time.localtime()))
+                new_file.uname = "root"
+                new_file.gname = "root"
+                target_tarball.addfile(new_file)
+
+            # # /vendor
+            new_file = tarfile.TarInfo()
+            new_file.type = tarfile.SYMTYPE
+            new_file.name = "system/vendor"
+            new_file.linkname = "/android/system/vendor"
+            new_file.mode = 0o755
+            new_file.mtime = int(time.strftime("%s", time.localtime()))
+            new_file.uname = "root"
+            new_file.gname = "root"
+            target_tarball.addfile(new_file)
+
+        # # /userdata
+        new_file = tarfile.TarInfo()
+        new_file.type = tarfile.DIRTYPE
+        new_file.name = "system/userdata"
+        new_file.mode = 0o755
+        new_file.mtime = int(time.strftime("%s", time.localtime()))
+        new_file.uname = "root"
+        new_file.gname = "root"
+        target_tarball.addfile(new_file)
+
+        # # /etc/mtab
+        new_file = tarfile.TarInfo()
+        new_file.type = tarfile.SYMTYPE
+        new_file.name = "system/etc/mtab"
+        new_file.linkname = "/proc/mounts"
+        new_file.mode = 0o444
+        new_file.mtime = int(time.strftime("%s", time.localtime()))
+        new_file.uname = "root"
+        new_file.gname = "root"
+        target_tarball.addfile(new_file)
+
+        # # /lib/modules
+        new_file = tarfile.TarInfo()
+        new_file.type = tarfile.DIRTYPE
+        new_file.name = "system/lib/modules"
+        new_file.mode = 0o755
+        new_file.mtime = int(time.strftime("%s", time.localtime()))
+        new_file.uname = "root"
+        new_file.gname = "root"
+        target_tarball.addfile(new_file)
+
+        source_tarball.close()
+        target_tarball.close()
+
+        # Create the pool if it doesn't exist
+        if not os.path.exists(os.path.join(conf.publish_path, "pool")):
+            os.makedirs(os.path.join(conf.publish_path, "pool"))
+
+        # Compress the target tarball and sign it
+        tools.xz_compress(os.path.join(temp_dir, "target.tar"), path)
+        gpg.sign_file(conf, "image-signing", path)
+
+        # Generate the metadata file
+        metadata = {}
+        metadata['generator'] = "cdimage-ubuntu"
+        metadata['version'] = version
+        metadata['version_detail'] = version_detail
+        metadata['series'] = series
+        metadata['rootfs_path'] = rootfs_path
+        metadata['rootfs_checksum'] = rootfs_hash
+
+        with open(path.replace(".tar.xz", ".json"), "w+") as fd:
+            fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
+                                         indent=4, separators=(',', ': ')))
+        gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
+
+        # Cleanup
+        shutil.rmtree(temp_dir)
+
+        environment['version_detail'].append(version_detail)
+        return path
+
+    return None
+
+
+def generate_file_cdimage_custom(conf, arguments, environment):
+    """
+        Scan a cdimage tree for new custom files.
+    """
+
+    # We need at least a path and a series
+    if len(arguments) < 2:
+        return None
+
+    # Read the arguments
+    cdimage_path = arguments[0]
+    series = arguments[1]
+
+    options = {}
+    if len(arguments) > 2:
+        options = unpack_arguments(arguments[2])
+
+    arch = "armhf"
+    if environment['device_name'] in ("generic_x86", "generic_i386"):
+        arch = "i386"
+    elif environment['device_name'] in ("generic_amd64",):
+        arch = "amd64"
+
+    # Check that the directory exists
+    if not os.path.exists(cdimage_path):
+        return None
+
+    versions = sorted([version for version in os.listdir(cdimage_path)
+                       if version not in ("pending", "current")],
+                      reverse=True)
+
+    for version in versions:
+        # Skip directory without checksums
+        if not os.path.exists(os.path.join(cdimage_path, version,
+                                           "SHA256SUMS")):
+            continue
+
+        # Check for the custom tarball
+        custom_path = os.path.join(cdimage_path, version,
+                                   "%s-preinstalled-%s-%s.custom.tar.gz" %
+                                   (series, options.get("product", "touch"),
+                                    arch))
+        if not os.path.exists(custom_path):
+            continue
+
+        # Check if we should only import tested images
+        if options.get("import", "any") == "good":
+            if not os.path.exists(os.path.join(cdimage_path, version,
+                                               ".marked_good")):
+                continue
+
+        # Set the version_detail string
+        version_detail = "custom=%s" % version
+
+        # Extract the hash
+        custom_hash = None
+        with open(os.path.join(cdimage_path, version,
+                               "SHA256SUMS"), "r") as fd:
+            for line in fd:
+                line = line.strip()
+                if line.endswith(custom_path.split("/")[-1]):
+                    custom_hash = line.split()[0]
+                    break
+
+        if not custom_hash:
+            continue
+
+        # Generate the path
+        path = os.path.join(conf.publish_path, "pool",
+                            "custom-%s.tar.xz" % custom_hash)
+
+        # Return pre-existing entries
+        if os.path.exists(path):
+            # Get the real version number (in case it got copied)
+            if os.path.exists(path.replace(".tar.xz", ".json")):
+                with open(path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    version_detail = metadata['version_detail']
+
+            environment['version_detail'].append(version_detail)
+            return path
+
+        temp_dir = tempfile.mkdtemp()
+
+        # Unpack the source tarball
+        tools.gzip_uncompress(custom_path, os.path.join(temp_dir,
+                                                        "source.tar"))
+
+        # Create the pool if it doesn't exist
+        if not os.path.exists(os.path.join(conf.publish_path, "pool")):
+            os.makedirs(os.path.join(conf.publish_path, "pool"))
+
+        # Compress the target tarball and sign it
+        tools.xz_compress(os.path.join(temp_dir, "source.tar"), path)
+        gpg.sign_file(conf, "image-signing", path)
+
+        # Generate the metadata file
+        metadata = {}
+        metadata['generator'] = "cdimage-custom"
+        metadata['version'] = version
+        metadata['version_detail'] = version_detail
+        metadata['series'] = series
+        metadata['custom_path'] = custom_path
+        metadata['custom_checksum'] = custom_hash
+
+        with open(path.replace(".tar.xz", ".json"), "w+") as fd:
+            fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
+                                         indent=4, separators=(',', ': ')))
+        gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
+
+        # Cleanup
+        shutil.rmtree(temp_dir)
+
+        environment['version_detail'].append(version_detail)
+        return path
+
+    return None
+
+
+def generate_file_http(conf, arguments, environment):
+    """
+        Grab, cache and returns a file using http/https.
+    """
+
+    # We need at least a URL
+    if len(arguments) == 0:
+        return None
+
+    # Read the arguments
+    url = arguments[0]
+
+    options = {}
+    if len(arguments) > 1:
+        options = unpack_arguments(arguments[1])
+
+    path = None
+    version = None
+
+    if "http_%s" % url in CACHE:
+        version = CACHE['http_%s' % url]
+
+    # Get the version/build number
+    if "monitor" in options or version:
+        if not version:
+            # Grab the current version number
+            old_timeout = socket.getdefaulttimeout()
+            socket.setdefaulttimeout(5)
+            try:
+                version = urlopen(options['monitor']).read().strip()
+            except socket.timeout:
+                return None
+            except IOError:
+                return None
+            socket.setdefaulttimeout(old_timeout)
+
+            # Validate the version number
+            if not version or len(version.split("\n")) > 1:
+                return None
+
+            # Push the result in the cache
+            CACHE['http_%s' % url] = version
+
+        # Set version_detail
+        version_detail = "%s=%s" % (options.get("name", "http"), version)
+
+        # FIXME: can be dropped once all the non-hased tarballs are gone
+        old_path = os.path.realpath(os.path.join(conf.publish_path, "pool",
+                                                 "%s-%s.tar.xz" %
+                                                 (options.get("name", "http"),
+                                                  version)))
+        if os.path.exists(old_path):
+            # Get the real version number (in case it got copied)
+            if os.path.exists(old_path.replace(".tar.xz", ".json")):
+                with open(old_path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    version_detail = metadata['version_detail']
+
+            environment['version_detail'].append(version_detail)
+            return old_path
+
+        # Build the path, hasing together the URL and version
+        hash_string = "%s:%s" % (url, version)
+        global_hash = sha256(hash_string.encode('utf-8')).hexdigest()
+        path = os.path.realpath(os.path.join(conf.publish_path, "pool",
+                                             "%s-%s.tar.xz" %
+                                             (options.get("name", "http"),
+                                              global_hash)))
+
+        # Return pre-existing entries
+        if os.path.exists(path):
+            # Get the real version number (in case it got copied)
+            if os.path.exists(path.replace(".tar.xz", ".json")):
+                with open(path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    version_detail = metadata['version_detail']
+
+            environment['version_detail'].append(version_detail)
+            return path
+
+    # Grab the real thing
+    tempdir = tempfile.mkdtemp()
+    old_timeout = socket.getdefaulttimeout()
+    socket.setdefaulttimeout(5)
+    try:
+        urlretrieve(url, os.path.join(tempdir, "download"))
+    except socket.timeout:
+        shutil.rmtree(tempdir)
+        return None
+    except IOError:
+        shutil.rmtree(tempdir)
+        return None
+    socket.setdefaulttimeout(old_timeout)
+
+    # Hash it if we don't have a version number
+    if not version:
+        # Hash the file
+        with open(os.path.join(tempdir, "download"), "rb") as fd:
+            version = sha256(fd.read()).hexdigest()
+
+        # Set version_detail
+        version_detail = "%s=%s" % (options.get("name", "http"), version)
+
+        # Push the result in the cache
+        CACHE['http_%s' % url] = version
+
+        # Build the path
+        path = os.path.realpath(os.path.join(conf.publish_path, "pool",
+                                             "%s-%s.tar.xz" %
+                                             (options.get("name", "http"),
+                                              version)))
+        # Return pre-existing entries
+        if os.path.exists(path):
+            # Get the real version number (in case it got copied)
+            if os.path.exists(path.replace(".tar.xz", ".json")):
+                with open(path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    version_detail = metadata['version_detail']
+
+            environment['version_detail'].append(version_detail)
+            shutil.rmtree(tempdir)
+            return path
+
+    # Create the pool if it doesn't exist
+    if not os.path.exists(os.path.join(conf.publish_path, "pool")):
+        os.makedirs(os.path.join(conf.publish_path, "pool"))
+
+    # Move the file to the pool and sign it
+    shutil.move(os.path.join(tempdir, "download"), path)
+    gpg.sign_file(conf, "image-signing", path)
+
+    # Generate the metadata file
+    metadata = {}
+    metadata['generator'] = "http"
+    metadata['version'] = version
+    metadata['version_detail'] = version_detail
+    metadata['url'] = url
+
+    with open(path.replace(".tar.xz", ".json"), "w+") as fd:
+        fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
+                                     indent=4, separators=(',', ': ')))
+    gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
+
+    # Cleanup
+    shutil.rmtree(tempdir)
+
+    environment['version_detail'].append(version_detail)
+    return path
+
+
+def generate_file_keyring(conf, arguments, environment):
+    """
+        Generate a keyring tarball or return a pre-existing one.
+    """
+
+    # Don't generate keyring tarballs when nothing changed
+    if len(environment['new_files']) == 0:
+        return None
+
+    # We need a keyring name
+    if len(arguments) == 0:
+        return None
+
+    # Read the arguments
+    keyring_name = arguments[0]
+    keyring_path = os.path.join(conf.gpg_keyring_path, keyring_name)
+
+    # Fail on missing keyring
+    if not os.path.exists("%s.tar.xz" % keyring_path) or \
+            not os.path.exists("%s.tar.xz.asc" % keyring_path):
+        return None
+
+    with open("%s.tar.xz" % keyring_path, "rb") as fd:
+        hash_tarball = sha256(fd.read()).hexdigest()
+
+    with open("%s.tar.xz.asc" % keyring_path, "rb") as fd:
+        hash_signature = sha256(fd.read()).hexdigest()
+
+    hash_string = "%s/%s" % (hash_tarball, hash_signature)
+    global_hash = sha256(hash_string.encode('utf-8')).hexdigest()
+
+    # Build the path
+    path = os.path.realpath(os.path.join(conf.publish_path, "pool",
+                                         "keyring-%s.tar.xz" %
+                                         global_hash))
+
+    # Set the version_detail string
+    environment['version_detail'].append("keyring=%s" % keyring_name)
+
+    # Don't bother re-generating a file if it already exists
+    if os.path.exists(path):
+        return path
+
+    # Create temporary directory
+    tempdir = tempfile.mkdtemp()
+
+    # Generate the tarball
+    tarball = tarfile.open(os.path.join(tempdir, "output.tar"), "w:")
+    tarball.add("%s.tar.xz" % keyring_path,
+                arcname="/system/etc/system-image/archive-master.tar.xz",
+                filter=root_ownership)
+    tarball.add("%s.tar.xz.asc" % keyring_path,
+                arcname="/system/etc/system-image/archive-master.tar.xz.asc",
+                filter=root_ownership)
+    tarball.close()
+
+    # Create the pool if it doesn't exist
+    if not os.path.exists(os.path.join(conf.publish_path, "pool")):
+        os.makedirs(os.path.join(conf.publish_path, "pool"))
+
+    # Compress and sign it
+    tools.xz_compress(os.path.join(tempdir, "output.tar"), path)
+    gpg.sign_file(conf, "image-signing", path)
+
+    # Generate the metadata file
+    metadata = {}
+    metadata['generator'] = "keyring"
+    metadata['version'] = global_hash
+    metadata['version_detail'] = "keyring=%s" % keyring_name
+    metadata['path'] = keyring_path
+
+    with open(path.replace(".tar.xz", ".json"), "w+") as fd:
+        fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
+                                     indent=4, separators=(',', ': ')))
+    gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
+
+    # Cleanup
+    shutil.rmtree(tempdir)
+
+    return path
+
+
+def generate_file_remote_system_image(conf, arguments, environment):
+    """
+        Import files from a remote system-image server
+    """
+
+    # We need at least a channel name and a file prefix
+    if len(arguments) < 3:
+        return None
+
+    # Read the arguments
+    base_url = arguments[0]
+    channel_name = arguments[1]
+    prefix = arguments[2]
+
+    options = {}
+    if len(arguments) > 3:
+        options = unpack_arguments(arguments[3])
+
+    device_name = environment['device_name']
+    if 'device' in options:
+        device_name = options['device']
+
+    # Fetch and validate the remote channels.json
+    old_timeout = socket.getdefaulttimeout()
+    socket.setdefaulttimeout(5)
+    try:
+        channel_json = json.loads(urlopen("%s/channels.json" %
+                                          base_url).read().decode().strip())
+    except socket.timeout:
+        return None
+    except IOError:
+        return None
+    socket.setdefaulttimeout(old_timeout)
+
+    if channel_name not in channel_json:
+        return None
+
+    if "devices" not in channel_json[channel_name]:
+        return None
+
+    if device_name not in channel_json[channel_name]['devices']:
+        return None
+
+    if "index" not in (channel_json[channel_name]['devices']
+                       [device_name]):
+        return None
+
+    index_url = "%s/%s" % (base_url, channel_json[channel_name]['devices']
+                           [device_name]['index'])
+
+    # Fetch and validate the remote index.json
+    old_timeout = socket.getdefaulttimeout()
+    socket.setdefaulttimeout(5)
+    try:
+        index_json = json.loads(urlopen(index_url).read().decode())
+    except socket.timeout:
+        return None
+    except IOError:
+        return None
+    socket.setdefaulttimeout(old_timeout)
+
+    # Grab the list of full images
+    full_images = sorted([image for image in index_json['images']
+                          if image['type'] == "full"],
+                         key=lambda image: image['version'])
+
+    # No images
+    if not full_images:
+        return None
+
+    # Found an image, so let's try to find a match
+    for file_entry in full_images[-1]['files']:
+        file_name = file_entry['path'].split("/")[-1]
+        file_prefix = file_name.rsplit("-", 1)[0]
+        if file_prefix == prefix:
+            path = os.path.realpath("%s/%s" % (conf.publish_path,
+                                               file_entry['path']))
+            if os.path.exists(path):
+                return path
+
+            # Create the target if needed
+            if not os.path.exists(os.path.dirname(path)):
+                os.makedirs(os.path.dirname(path))
+
+            # Grab the file
+            file_url = "%s/%s" % (base_url, file_entry['path'])
+            socket.setdefaulttimeout(5)
+            try:
+                urlretrieve(file_url, path)
+            except socket.timeout:
+                if os.path.exists(path):
+                    os.remove(path)
+                return None
+            except IOError:
+                if os.path.exists(path):
+                    os.remove(path)
+                return None
+            socket.setdefaulttimeout(old_timeout)
+
+            if "keyring" in options:
+                if not tools.repack_recovery_keyring(conf, path,
+                                                     options['keyring']):
+                    if os.path.exists(path):
+                        os.remove(path)
+                    return None
+
+            gpg.sign_file(conf, "image-signing", path)
+
+            # Attempt to grab an associated json
+            socket.setdefaulttimeout(5)
+            json_path = path.replace(".tar.xz", ".json")
+            json_url = file_url.replace(".tar.xz", ".json")
+            try:
+                urlretrieve(json_url, json_path),
+            except socket.timeout:
+                if os.path.exists(json_path):
+                    os.remove(json_path)
+            except IOError:
+                if os.path.exists(json_path):
+                    os.remove(json_path)
+            socket.setdefaulttimeout(old_timeout)
+
+            if os.path.exists(json_path):
+                gpg.sign_file(conf, "image-signing", json_path)
+                with open(json_path, "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    environment['version_detail'].append(
+                        metadata['version_detail'])
+
+            return path
+
+    return None
+
+
+def generate_file_system_image(conf, arguments, environment):
+    """
+        Copy a file from another channel.
+    """
+
+    # We need at least a channel name and a file prefix
+    if len(arguments) < 2:
+        return None
+
+    # Read the arguments
+    channel_name = arguments[0]
+    prefix = arguments[1]
+
+    # Run some checks
+    pub = tree.Tree(conf)
+    if channel_name not in pub.list_channels():
+        return None
+
+    if (not environment['device_name'] in
+            pub.list_channels()[channel_name]['devices']):
+        return None
+
+    # Try to find the file
+    device = pub.get_device(channel_name, environment['device_name'])
+
+    full_images = sorted([image for image in device.list_images()
+                          if image['type'] == "full"],
+                         key=lambda image: image['version'])
+
+    # No images
+    if not full_images:
+        return None
+
+    # Found an image, so let's try to find a match
+    for file_entry in full_images[-1]['files']:
+        file_name = file_entry['path'].split("/")[-1]
+        file_prefix = file_name.rsplit("-", 1)[0]
+        if file_prefix == prefix:
+            path = os.path.realpath("%s/%s" % (conf.publish_path,
+                                               file_entry['path']))
+
+            if os.path.exists(path.replace(".tar.xz", ".json")):
+                with open(path.replace(".tar.xz", ".json"), "r") as fd:
+                    metadata = json.loads(fd.read())
+
+                if "version_detail" in metadata:
+                    environment['version_detail'].append(
+                        metadata['version_detail'])
+
+            return path
+
+    return None
+
+
+def generate_file_version(conf, arguments, environment):
+    """
+        Generate a version tarball or return a pre-existing one.
+    """
+
+    # Don't generate version tarballs when nothing changed
+    if len(environment['new_files']) == 0:
+        return None
+
+    path = os.path.realpath(os.path.join(environment['device'].path,
+                            "version-%s.tar.xz" % environment['version']))
+
+    # Set the version_detail string
+    environment['version_detail'].append("version=%s" % environment['version'])
+
+    # Don't bother re-generating a file if it already exists
+    if os.path.exists(path):
+        return path
+
+    # Generate version_detail
+    version_detail = ",".join(environment['version_detail'])
+
+    # Create temporary directory
+    tempdir = tempfile.mkdtemp()
+
+    # Generate the tarball
+    tools.generate_version_tarball(
+        conf, environment['channel_name'], environment['device_name'],
+        str(environment['version']),
+        os.path.join(tempdir, "version"), version_detail=version_detail)
+
+    # Create the pool if it doesn't exist
+    if not os.path.exists(os.path.join(environment['device'].path)):
+        os.makedirs(os.path.join(environment['device'].path))
+
+    # Compress and sign it
+    tools.xz_compress(os.path.join(tempdir, "version"), path)
+    gpg.sign_file(conf, "image-signing", path)
+
+    # Generate the metadata file
+    metadata = {}
+    metadata['generator'] = "version"
+    metadata['version'] = environment['version']
+    metadata['version_detail'] = "version=%s" % environment['version']
+    metadata['channel.ini'] = {}
+    metadata['channel.ini']['channel'] = environment['channel_name']
+    metadata['channel.ini']['device'] = environment['device_name']
+    metadata['channel.ini']['version'] = str(environment['version'])
+    metadata['channel.ini']['version_detail'] = version_detail
+
+    with open(path.replace(".tar.xz", ".json"), "w+") as fd:
+        fd.write("%s\n" % json.dumps(metadata, sort_keys=True,
+                                     indent=4, separators=(',', ': ')))
+    gpg.sign_file(conf, "image-signing", path.replace(".tar.xz", ".json"))
+
+    # Cleanup
+    shutil.rmtree(tempdir)
+
+    return path

=== added file 'lib/systemimage/gpg.py'
--- lib/systemimage/gpg.py	1970-01-01 00:00:00 +0000
+++ lib/systemimage/gpg.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,239 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import gpgme
+import os
+import tarfile
+
+from io import BytesIO
+
+
+def generate_signing_key(keyring_path, key_name, key_email, key_expiry):
+    """
+        Generate a new 2048bit RSA signing key.
+    """
+
+    if not os.path.isdir(keyring_path):
+        raise Exception("Keyring path doesn't exist: %s" % keyring_path)
+
+    key_params = """<GnupgKeyParms format="internal">
+Key-Type: RSA
+Key-Length: 2048
+Key-Usage: sign
+Name-Real: %s
+Name-Email: %s
+Expire-Date: %s
+</GnupgKeyParms>
+""" % (key_name, key_email, key_expiry)
+
+    os.environ['GNUPGHOME'] = keyring_path
+
+    ctx = gpgme.Context()
+    result = ctx.genkey(key_params)
+    key = ctx.get_key(result.fpr, True)
+    [uid] = key.uids
+
+    return uid
+
+
+def sign_file(config, key, path, destination=None, detach=True, armor=True):
+    """
+        Sign a file and publish the signature.
+        The key parameter must be a valid key under config.gpg_key_path.
+        The path must be that of a valid file.
+        The destination defaults to <path>.gpg (non-armored) or
+        <path>.asc (armored).
+        The detach and armor parameters respectively control the use of
+        detached signatures and base64 armoring.
+    """
+
+    key_path = "%s/%s" % (config.gpg_key_path, key)
+
+    if not os.path.isdir(key_path):
+        raise IndexError("Invalid GPG key name '%s'." % key)
+
+    if not os.path.isfile(path):
+        raise Exception("Invalid path '%s'." % path)
+
+    if not destination:
+        if armor:
+            destination = "%s.asc" % path
+        elif detach:
+            destination = "%s.sig" % path
+        else:
+            destination = "%s.gpg" % path
+
+    if os.path.exists(destination):
+        raise Exception("destination already exists.")
+
+    os.environ['GNUPGHOME'] = key_path
+
+    # Create a GPG context, assuming no passphrase
+    ctx = gpgme.Context()
+    ctx.armor = armor
+    [key] = ctx.keylist()
+    ctx.signers = [key]
+
+    with open(path, "rb") as fd_in, open(destination, "wb+") as fd_out:
+        if detach:
+            retval = ctx.sign(fd_in, fd_out, gpgme.SIG_MODE_DETACH)
+        else:
+            retval = ctx.sign(fd_in, fd_out, gpgme.SIG_MODE_NORMAL)
+
+    return retval
+
+
+class Keyring:
+    """
+        Represents a keyring, let's you list/add/remove keys and change
+        some of the keyring properties (type, expiration, target hardware)
+    """
+
+    keyring_name = None
+    keyring_type = None
+    keyring_expiry = None
+    keyring_model = None
+    keyring_path = None
+
+    def __init__(self, config, keyring_name):
+        keyring_path = "%s/%s" % (config.gpg_keyring_path, keyring_name)
+
+        if not os.path.isdir(keyring_path):
+            os.makedirs(keyring_path)
+
+        self.keyring_name = keyring_name
+        self.keyring_path = keyring_path
+
+        if os.path.exists("%s/keyring.json" % keyring_path):
+            with open("%s/keyring.json" % keyring_path, "r") as fd:
+                keyring_json = json.loads(fd.read())
+
+            self.keyring_type = keyring_json.get('type', None)
+            self.keyring_expiry = keyring_json.get('expiry', None)
+            self.keyring_model = keyring_json.get('model', None)
+        else:
+            open("%s/pubring.gpg" % keyring_path, "w+").close()
+
+    def generate_tarball(self, destination=None):
+        """
+            Generate a tarball of the keyring and its json metadata.
+            Returns the path to the tarball.
+        """
+
+        if not destination:
+            destination = "%s.tar" % self.keyring_path
+
+        if os.path.isfile(destination):
+            os.remove(destination)
+
+        tarball = tarfile.open(destination, "w:")
+        tarball.add("%s/keyring.json" % self.keyring_path,
+                    arcname="keyring.json")
+        tarball.add("%s/pubring.gpg" % self.keyring_path,
+                    arcname="keyring.gpg")
+        tarball.close()
+
+        return destination
+
+    def set_metadata(self, keyring_type, keyring_expiry=None,
+                     keyring_model=None):
+        """
+            Generate a new keyring.json file.
+        """
+
+        keyring_json = {}
+        if keyring_type:
+            self.keyring_type = keyring_type
+            keyring_json['type'] = keyring_type
+
+        if keyring_expiry:
+            self.keyring_expiry = keyring_expiry
+            keyring_json['expiry'] = keyring_expiry
+
+        if keyring_model:
+            self.keyring_model = keyring_model
+            keyring_json['model'] = keyring_model
+
+        with open("%s/keyring.json" % self.keyring_path, "w+") as fd:
+            fd.write("%s\n" % json.dumps(keyring_json, sort_keys=True,
+                                         indent=4, separators=(',', ': ')))
+
+    def list_keys(self):
+        os.environ['GNUPGHOME'] = self.keyring_path
+
+        keys = []
+
+        ctx = gpgme.Context()
+        for key in ctx.keylist():
+            keys.append((key.subkeys[0].keyid, key.subkeys[0].length,
+                        [uid.uid for uid in key.uids]))
+
+        return keys
+
+    def export_key(self, path, key, armor=True):
+        os.environ['GNUPGHOME'] = self.keyring_path
+
+        ctx = gpgme.Context()
+        ctx.armor = armor
+
+        gpg_key = ctx.get_key(key)
+
+        with open(path, "wb+") as fd:
+            for subkey in gpg_key.subkeys:
+                ctx.export(str(subkey.keyid), fd)
+
+    def import_key(self, path, armor=True):
+        os.environ['GNUPGHOME'] = self.keyring_path
+
+        ctx = gpgme.Context()
+        ctx.armor = armor
+
+        with open(path, "rb") as fd:
+            ctx.import_(fd)
+
+    def import_keys(self, path):
+        """
+            Import all the keys from the specified keyring.
+        """
+
+        os.environ['GNUPGHOME'] = path
+
+        ctx = gpgme.Context()
+
+        keys = []
+        for key in list(ctx.keylist()):
+            for subkey in key.subkeys:
+                content = BytesIO()
+                ctx.export(str(subkey.keyid), content)
+                keys.append(content)
+
+        os.environ['GNUPGHOME'] = self.keyring_path
+        ctx = gpgme.Context()
+
+        for key in keys:
+            key.seek(0)
+            ctx.import_(key)
+
+    def del_key(self, key):
+        os.environ['GNUPGHOME'] = self.keyring_path
+
+        ctx = gpgme.Context()
+
+        gpg_key = ctx.get_key(key)
+
+        ctx.delete(gpg_key)

=== added file 'lib/systemimage/tools.py'
--- lib/systemimage/tools.py	1970-01-01 00:00:00 +0000
+++ lib/systemimage/tools.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,367 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from io import BytesIO
+
+import gzip
+import os
+import re
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import time
+
+
+def expand_path(path, base="/"):
+    """
+        Takes a path and returns a tuple containing the absolute path
+        and a relative path (relative to base).
+    """
+
+    if path.startswith(base):
+        path = re.sub('^%s' % re.escape(base), "", path)
+
+    if path.startswith(os.sep):
+        relpath = path[1:]
+    else:
+        relpath = path
+
+    abspath = os.path.realpath(os.path.join(base, relpath))
+
+    return abspath, relpath
+
+
+# Imported from cdimage.osextras
+def find_on_path(command):
+    """Is command on the executable search path?"""
+
+    if 'PATH' not in os.environ:
+        return False
+    path = os.environ['PATH']
+    for element in path.split(os.pathsep):
+        if not element:
+            continue
+        filename = os.path.join(element, command)
+        if os.path.isfile(filename) and os.access(filename, os.X_OK):
+            return True
+    return False
+
+
+def generate_version_tarball(config, channel, device, version, path,
+                             build_path="system/etc/ubuntu-build",
+                             channel_path="system/etc/system-image/"
+                                          "channel.ini",
+                             version_detail=None,
+                             channel_target=None):
+    """
+        Generates a tarball which contains two files
+        (build_path and channel_path).
+        The first contains the build id, the second a .ini config file.
+        The resulting tarball is written at the provided location (path).
+    """
+
+    tarball = tarfile.open(path, 'w:')
+
+    version_file = tarfile.TarInfo()
+    version_file.size = len(version) + 1
+    version_file.mtime = int(time.strftime("%s", time.localtime()))
+    version_file.name = build_path
+
+    # Append a line break
+    version += "\n"
+
+    tarball.addfile(version_file, BytesIO(version.encode('utf-8')))
+
+    http_port = config.public_http_port
+    https_port = config.public_https_port
+
+    if http_port == 0:
+        http_port = "disabled"
+
+    if https_port == 0:
+        https_port = "disabled"
+
+    channel = """[service]
+base: %s
+http_port: %s
+https_port: %s
+channel: %s
+device: %s
+build_number: %s
+""" % (config.public_fqdn, http_port, https_port,
+       channel, device, version.strip())
+
+    if channel_target:
+        channel += "channel_target: %s\n" % channel_target
+
+    if version_detail:
+        channel += "version_detail: %s\n" % version_detail
+
+    channel_file = tarfile.TarInfo()
+    channel_file.size = len(channel)
+    channel_file.mtime = int(time.strftime("%s", time.localtime()))
+    channel_file.name = channel_path
+
+    tarball.addfile(channel_file, BytesIO(channel.encode('utf-8')))
+
+    tarball.close()
+
+
+def gzip_compress(path, destination=None, level=9):
+    """
+        Compress a file (path) using gzip.
+        By default, creates a .gz version of the file in the same directory.
+        An alternate destination path may be provided.
+        The compress level is 9 by default but can be overriden.
+    """
+
+    if not destination:
+        destination = "%s.gz" % path
+
+    if os.path.exists(destination):
+        raise Exception("destination already exists.")
+
+    uncompressed = open(path, "rb")
+    compressed = gzip.open(destination, "wb+", level)
+    compressed.writelines(uncompressed)
+    compressed.close()
+    uncompressed.close()
+
+    return destination
+
+
+def gzip_uncompress(path, destination=None):
+    """
+        Uncompress a file (path) using gzip.
+        By default, uses the source path without the .gz prefix as the target.
+        An alternate destination path may be provided.
+    """
+
+    if not destination and path[-3:] != ".gz":
+        raise Exception("unspecified destination and path doesn't end"
+                        " with .gz")
+
+    if not destination:
+        destination = path[:-3]
+
+    if os.path.exists(destination):
+        raise Exception("destination already exists.")
+
+    compressed = gzip.open(path, "rb")
+    uncompressed = open(destination, "wb+")
+    uncompressed.writelines(compressed)
+    uncompressed.close()
+    compressed.close()
+
+    return destination
+
+
+def xz_compress(path, destination=None, level=9):
+    """
+        Compress a file (path) using xz.
+        By default, creates a .xz version of the file in the same directory.
+        An alternate destination path may be provided.
+        The compress level is 9 by default but can be overriden.
+    """
+
+    # NOTE: Once we can drop support for < 3.3, the new lzma module can be used
+
+    if not destination:
+        destination = "%s.xz" % path
+
+    if os.path.exists(destination):
+        raise Exception("destination already exists.")
+
+    if find_on_path("pxz"):
+        xz_command = "pxz"
+    else:
+        xz_command = "xz"
+
+    with open(destination, "wb+") as fd:
+        retval = subprocess.call([xz_command, '-z', '-%s' % level, '-c', path],
+                                 stdout=fd)
+    return retval
+
+
+def xz_uncompress(path, destination=None):
+    """
+        Uncompress a file (path) using xz.
+        By default, uses the source path without the .xz prefix as the target.
+        An alternate destination path may be provided.
+    """
+
+    # NOTE: Once we can drop support for < 3.3, the new lzma module can be used
+
+    if not destination and path[-3:] != ".xz":
+        raise Exception("unspecified destination and path doesn't end"
+                        " with .xz")
+
+    if not destination:
+        destination = path[:-3]
+
+    if os.path.exists(destination):
+        raise Exception("destination already exists.")
+
+    with open(destination, "wb+") as fd:
+        retval = subprocess.call(['xz', '-d', '-c', path],
+                                 stdout=fd)
+
+    return retval
+
+
+def trigger_mirror(host, port, username, key, command):
+    return subprocess.call(['ssh',
+                            '-i', key,
+                            '-l', username,
+                            '-p', str(port),
+                            host,
+                            command])
+
+
+def sync_mirrors(config):
+    for mirror in sorted(config.mirrors.values(),
+                         key=lambda mirror: mirror.ssh_host):
+        trigger_mirror(mirror.ssh_host, mirror.ssh_port, mirror.ssh_user,
+                       mirror.ssh_key, mirror.ssh_command)
+
+
+def repack_recovery_keyring(conf, path, keyring_name):
+    tempdir = tempfile.mkdtemp()
+
+    xz_uncompress(path, os.path.join(tempdir, "input.tar"))
+
+    input_tarball = tarfile.open(os.path.join(tempdir, "input.tar"), "r:")
+
+    # Make sure the partition is in there
+    if "partitions/recovery.img" not in input_tarball.getnames():
+        shutil.rmtree(tempdir)
+        return False
+
+    input_tarball.extract("partitions/recovery.img", tempdir)
+
+    # Extract the content of the .img
+    os.mkdir(os.path.join(tempdir, "img"))
+    old_pwd = os.getcwd()
+    os.chdir(os.path.join(tempdir, "img"))
+    cmd = ["abootimg",
+           "-x", os.path.join(tempdir, "partitions", "recovery.img")]
+
+    with open(os.path.devnull, "w") as devnull:
+        subprocess.call(cmd, stdout=devnull, stderr=devnull)
+
+    os.chdir(old_pwd)
+
+    # Extract the content of the initrd
+    os.mkdir(os.path.join(tempdir, "initrd"))
+    state_path = os.path.join(tempdir, "fakeroot_state")
+    old_pwd = os.getcwd()
+    os.chdir(os.path.join(tempdir, "initrd"))
+
+    gzip_uncompress(os.path.join(tempdir, "img", "initrd.img"),
+                    os.path.join(tempdir, "img", "initrd"))
+
+    with open(os.path.join(tempdir, "img", "initrd"), "rb") as fd:
+        with open(os.path.devnull, "w") as devnull:
+            subprocess.call(['fakeroot', '-s', state_path, 'cpio', '-i'],
+                            stdin=fd, stdout=devnull, stderr=devnull)
+
+    os.chdir(old_pwd)
+
+    # Swap the files
+    keyring_path = os.path.join(conf.gpg_keyring_path, "archive-master")
+
+    shutil.copy("%s.tar.xz" % keyring_path,
+                os.path.join(tempdir, "initrd", "etc", "system-image",
+                             "archive-master.tar.xz"))
+
+    shutil.copy("%s.tar.xz.asc" % keyring_path,
+                os.path.join(tempdir, "initrd", "etc", "system-image",
+                             "archive-master.tar.xz.asc"))
+
+    # Re-generate the initrd
+    old_pwd = os.getcwd()
+    os.chdir(os.path.join(tempdir, "initrd"))
+
+    find = subprocess.Popen(["find", "."], stdout=subprocess.PIPE)
+    with open(os.path.join(tempdir, "img", "initrd"), "w+") as fd:
+        with open(os.path.devnull, "w") as devnull:
+            subprocess.call(['fakeroot', '-i', state_path, 'cpio',
+                             '-o', '--format=newc'],
+                            stdin=find.stdout,
+                            stdout=fd,
+                            stderr=devnull)
+
+    os.chdir(old_pwd)
+
+    os.rename(os.path.join(tempdir, "img", "initrd.img"),
+              os.path.join(tempdir, "img", "initrd.img.bak"))
+    gzip_compress(os.path.join(tempdir, "img", "initrd"),
+                  os.path.join(tempdir, "img", "initrd.img"))
+
+    # Rewrite bootimg.cfg
+    content = ""
+    with open(os.path.join(tempdir, "img", "bootimg.cfg"), "r") as source:
+        for line in source:
+            if line.startswith("bootsize"):
+                line = "bootsize=0x900000\n"
+            content += line
+
+    with open(os.path.join(tempdir, "img", "bootimg.cfg"), "w+") as dest:
+        dest.write(content)
+
+    # Update the partition image
+    with open(os.path.devnull, "w") as devnull:
+        subprocess.call(['abootimg', '-u',
+                         os.path.join(tempdir, "partitions", "recovery.img"),
+                         "-f", os.path.join(tempdir, "img", "bootimg.cfg")],
+                        stdout=devnull, stderr=devnull)
+
+    # Update the partition image
+    with open(os.path.devnull, "w") as devnull:
+        subprocess.call(['abootimg', '-u',
+                         os.path.join(tempdir, "partitions", "recovery.img"),
+                         "-r", os.path.join(tempdir, "img", "initrd.img")],
+                        stdout=devnull, stderr=devnull)
+
+    # Generate a new tarball
+    output_tarball = tarfile.open(os.path.join(tempdir, "output.tar"), "w:")
+    for entry in input_tarball:
+        fileptr = None
+        if entry.isfile():
+            try:
+                if entry.name == "partitions/recovery.img":
+                    with open(os.path.join(tempdir, "partitions",
+                                           "recovery.img"), "rb") as fd:
+                        fileptr = BytesIO(fd.read())
+                        entry.size = os.stat(
+                            os.path.join(tempdir, "partitions",
+                                         "recovery.img")).st_size
+                else:
+                    fileptr = input_tarball.extractfile(entry.name)
+            except KeyError:  # pragma: no cover
+                pass
+
+        output_tarball.addfile(entry, fileobj=fileptr)
+    output_tarball.close()
+
+    os.remove(path)
+    xz_compress(os.path.join(tempdir, "output.tar"), path)
+
+    shutil.rmtree(tempdir)
+
+    return True

=== added file 'lib/systemimage/tree.py'
--- lib/systemimage/tree.py	1970-01-01 00:00:00 +0000
+++ lib/systemimage/tree.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,999 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import copy
+import json
+import os
+import shutil
+import time
+
+from contextlib import contextmanager
+from hashlib import sha256
+from systemimage import gpg, tools
+
+
+# Context managers
+@contextmanager
+def channels_json(config, path, commit=False):
+    """
+        Context function (to be used with "with") that will open a
+        channels.json file, parse it, validate it and return the
+        decoded version.
+
+        If commit is True, the file will then be updated (or created) on
+        exit.
+    """
+
+    # If the file doesn't exist, just yield an empty dict
+    json_content = {}
+    if os.path.exists(path):
+        with open(path, "r") as fd:
+            content = fd.read()
+            if content:
+                json_content = json.loads(content)
+
+    # Validation
+    if not isinstance(json_content, dict):
+        raise TypeError("Invalid channels.json, not a dict.")
+
+    if commit:
+        orig_json_content = copy.deepcopy(json_content)
+
+    # Yield the decoded value and save on exit
+    try:
+        yield json_content
+    finally:
+        if commit and (orig_json_content != json_content or
+                       not os.path.exists(path)):
+            new_path = "%s.new" % path
+            with open(new_path, "w+") as fd:
+                fd.write("%s\n" % json.dumps(json_content, sort_keys=True,
+                                             indent=4, separators=(',', ': ')))
+
+            # Move the signature
+            gpg.sign_file(config, "image-signing", new_path)
+            if os.path.exists("%s.asc" % path):
+                os.remove("%s.asc" % path)
+            os.rename("%s.asc" % new_path, "%s.asc" % path)
+
+            # Move the index
+            if os.path.exists(path):
+                os.remove(path)
+            os.rename(new_path, path)
+
+
+@contextmanager
+def index_json(config, path, commit=False):
+    """
+        Context function (to be used with "with") that will open an
+        index.json file, parse it, validate it and return the
+        decoded version.
+
+        If commit is True, the file will then be updated (or created) on
+        exit.
+    """
+
+    # If the file doesn't exist, just yield an empty dict
+    json_content = {}
+    json_content['global'] = {}
+    json_content['images'] = []
+
+    if os.path.exists(path):
+        with open(path, "r") as fd:
+            content = fd.read()
+            if content:
+                json_content = json.loads(content)
+
+    # Validation
+    if not isinstance(json_content, dict):
+        raise TypeError("Invalid index.json, not a dict.")
+
+    if commit:
+        orig_json_content = copy.deepcopy(json_content)
+
+    # Yield the decoded value and save on exit
+    try:
+        yield json_content
+    finally:
+        # Remove any invalid attribute
+        versions = sorted({image['version']
+                           for image in json_content['images']})
+        if versions:
+            last_version = versions[-1]
+
+            # Remove phased-percentage from any old image
+            for image in json_content['images']:
+                if image['version'] != last_version and \
+                        "phased-percentage" in image:
+                    image.pop("phased-percentage")
+
+        # Save to disk
+        if commit and (orig_json_content != json_content or
+                       not os.path.exists(path)):
+            json_content['global']['generated_at'] = time.strftime(
+                "%a %b %d %H:%M:%S UTC %Y", time.gmtime())
+
+            new_path = "%s.new" % path
+            with open(new_path, "w+") as fd:
+                fd.write("%s\n" % json.dumps(json_content, sort_keys=True,
+                                             indent=4, separators=(',', ': ')))
+
+            # Move the signature
+            gpg.sign_file(config, "image-signing", new_path)
+            if os.path.exists("%s.asc" % path):
+                os.remove("%s.asc" % path)
+            os.rename("%s.asc" % new_path, "%s.asc" % path)
+
+            # Move the index
+            if os.path.exists(path):
+                os.remove(path)
+            os.rename(new_path, path)
+
+
+class Tree:
+    def __init__(self, config, path=None):
+        if not path:
+            path = config.publish_path
+
+        if not os.path.isdir(path):
+            raise Exception("Invalid path: %s" % path)
+
+        self.config = config
+        self.path = path
+        self.indexpath = os.path.join(path, "channels.json")
+
+    def __list_existing(self):
+        """
+            Returns a set of all files present in the tree and a set of
+            empty directories that can be removed.
+        """
+
+        existing_files = set()
+        empty_dirs = set()
+
+        for dirpath, dirnames, filenames in os.walk(self.path):
+            if dirpath == os.path.join(self.path, "gpg"):
+                continue
+
+            if not filenames and not dirnames:
+                empty_dirs.add(dirpath)
+
+            for entry in filenames:
+                existing_files.add(os.path.join(dirpath, entry))
+
+        return (existing_files, empty_dirs)
+
+    def __list_referenced(self):
+        """
+            Returns a set of all files that are referenced by the
+            various indexes and should be present in the tree.
+        """
+
+        listed_files = set()
+        listed_files.add(os.path.join(self.path, "channels.json"))
+        listed_files.add(os.path.join(self.path, "channels.json.asc"))
+
+        for channel, metadata in self.list_channels().items():
+            devices = metadata['devices']
+            for device in devices:
+                if 'keyring' in devices[device]:
+                    listed_files.add(os.path.join(
+                        self.path, devices[device]['keyring']['path'][1:]))
+                    listed_files.add(os.path.join(
+                        self.path,
+                        devices[device]['keyring']['signature'][1:]))
+
+                device_entry = self.get_device(channel, device)
+
+                listed_files.add(os.path.join(device_entry.path, "index.json"))
+                listed_files.add(os.path.join(device_entry.path,
+                                              "index.json.asc"))
+
+                for image in device_entry.list_images():
+                    for entry in image['files']:
+                        listed_files.add(os.path.join(self.path,
+                                                      entry['path'][1:]))
+                        listed_files.add(os.path.join(self.path,
+                                                      entry['signature'][1:]))
+
+        return listed_files
+
+    def change_channel_alias(self, channel_name, target_name):
+        """
+            Change the target of an alias.
+        """
+
+        with channels_json(self.config, self.indexpath) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            if "alias" not in channels[channel_name] or \
+                    channels[channel_name]['alias'] == channel_name:
+                raise KeyError("Channel isn't an alias: %s" % channel_name)
+
+            if target_name not in channels:
+                raise KeyError("Couldn't find target channel: %s" %
+                               target_name)
+
+        self.remove_channel(channel_name)
+        self.create_channel_alias(channel_name, target_name)
+
+        return True
+
+    def cleanup_tree(self):
+        """
+            Remove any orphaned file from the tree.
+        """
+
+        for entry in self.list_orphaned_files():
+            if os.path.isdir(entry):
+                os.rmdir(entry)
+            else:
+                os.remove(entry)
+
+        return True
+
+    def create_channel(self, channel_name):
+        """
+            Creates a new channel entry in the tree.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name in channels:
+                raise KeyError("Channel already exists: %s" % channel_name)
+
+            channels[channel_name] = {'devices': {}}
+
+        return True
+
+    def create_channel_alias(self, channel_name, target_name):
+        """
+            Creates a new channel as an alias for an existing one.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name in channels:
+                raise KeyError("Channel already exists: %s" % channel_name)
+
+            if target_name not in channels:
+                raise KeyError("Couldn't find target channel: %s" %
+                               target_name)
+
+            channels[channel_name] = {'devices': {},
+                                      'alias': target_name}
+
+        return self.sync_alias(channel_name)
+
+    def create_channel_redirect(self, channel_name, target_name):
+        """
+            Creates a new channel redirect.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name in channels:
+                raise KeyError("Channel already exists: %s" % channel_name)
+
+            if target_name not in channels:
+                raise KeyError("Couldn't find target channel: %s" %
+                               target_name)
+
+            channels[channel_name] = dict(channels[target_name])
+            channels[channel_name]['redirect'] = target_name
+
+        self.hide_channel(channel_name)
+
+        return True
+
+    def create_device(self, channel_name, device_name, keyring_path=None):
+        """
+            Creates a new device entry in the tree.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            if device_name in channels[channel_name]['devices']:
+                raise KeyError("Device already exists: %s" % device_name)
+
+            device_path = os.path.join(self.path, channel_name, device_name)
+            if not os.path.exists(device_path):
+                os.makedirs(device_path)
+
+            # Create an empty index if it doesn't exist, if it does,
+            # just validate it
+            with index_json(self.config, os.path.join(device_path,
+                                                      "index.json"), True):
+                pass
+
+            device = {}
+            device['index'] = "/%s/%s/index.json" % (channel_name, device_name)
+
+            channels[channel_name]['devices'][device_name] = device
+
+        if keyring_path:
+            self.set_device_keyring(channel_name, device_name, keyring_path)
+
+        self.sync_aliases(channel_name)
+        self.sync_redirects(channel_name)
+
+        return True
+
+    def generate_index(self, magic=False):
+        """
+            Re-generate the channels.json file based on the current content of
+            the tree.
+
+            This function is only present for emergency purposes and will
+            completely rebuild the tree based on what's on the filesystem,
+            looking into some well known locations to guess things like device
+            keyring paths.
+
+            Call this function with confirm="I know what I'm doing" to actually
+            trigger it.
+        """
+
+        if magic != "I know what I'm doing":
+            raise Exception("Invalid magic value, please read the help.")
+
+        if os.path.exists(self.indexpath):
+            os.remove(self.indexpath)
+
+        for channel_name in [entry for entry in os.listdir(self.path)
+                             if os.path.isdir(os.path.join(self.path,
+                                                           entry))
+                             and entry not in ('gpg',)]:
+            self.create_channel(channel_name)
+
+            for device_name in os.listdir(os.path.join(self.path,
+                                                       channel_name)):
+
+                path = os.path.join(self.path, channel_name, device_name)
+                if not os.path.exists(os.path.join(path, "index.json")):
+                    continue
+
+                keyring_path = os.path.join(path, "device.tar.xz")
+                if (os.path.exists(keyring_path)
+                        and os.path.exists("%s.asc" % keyring_path)):
+                    self.create_device(channel_name, device_name, keyring_path)
+                else:
+                    self.create_device(channel_name, device_name)
+
+        return True
+
+    def get_device(self, channel_name, device_name):
+        """
+            Returns a Device instance.
+        """
+
+        with channels_json(self.config, self.indexpath) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            if device_name not in channels[channel_name]['devices']:
+                raise KeyError("Couldn't find device: %s" % device_name)
+
+            device_path = os.path.dirname(channels[channel_name]['devices']
+                                          [device_name]['index'])
+
+            return Device(self.config, os.path.normpath("%s/%s" % (self.path,
+                          device_path)))
+
+    def hide_channel(self, channel_name):
+        """
+            Hide a channel from the client's list.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            channels[channel_name]['hidden'] = True
+
+        return True
+
+    def list_channels(self):
+        """
+            Returns a dict of all existing channels and devices for each of
+            those.
+            This is simply a decoded version of channels.json
+        """
+
+        with channels_json(self.config, self.indexpath) as channels:
+            return channels
+
+    def list_missing_files(self):
+        """
+            Returns a list of absolute paths that should exist but aren't
+            present on the filesystem.
+        """
+
+        all_files, empty_dirs = self.__list_existing()
+        referenced_files = self.__list_referenced()
+
+        return sorted(referenced_files - all_files)
+
+    def list_orphaned_files(self):
+        """
+            Returns a list of absolute paths to files that are present in the
+            tree but aren't referenced anywhere.
+        """
+
+        orphaned_files = set()
+
+        all_files, empty_dirs = self.__list_existing()
+        referenced_files = self.__list_referenced()
+
+        orphaned_files.update(all_files - referenced_files)
+        orphaned_files.update(empty_dirs)
+
+        for entry in list(orphaned_files):
+            if entry.endswith(".json"):
+                tarname = entry.replace(".json", ".tar.xz")
+                if tarname in referenced_files:
+                    orphaned_files.remove(entry)
+
+            if entry.endswith(".json.asc"):
+                tarname = entry.replace(".json.asc", ".tar.xz")
+                if tarname in referenced_files:
+                    orphaned_files.remove(entry)
+
+        return sorted(orphaned_files)
+
+    def publish_keyring(self, keyring_name):
+        """
+            Publish the keyring under gpg/
+        """
+
+        gpg_path = os.path.join(self.config.publish_path, "gpg")
+
+        if not os.path.exists(gpg_path):
+            os.mkdir(gpg_path)
+
+        keyring_path = os.path.join(self.config.gpg_keyring_path, keyring_name)
+
+        if not os.path.exists("%s.tar.xz" % keyring_path):
+            raise Exception("Missing keyring: %s.tar.xz" % keyring_path)
+
+        if not os.path.exists("%s.tar.xz.asc" % keyring_path):
+            raise Exception("Missing keyring signature: %s.tar.xz.asc" %
+                            keyring_path)
+
+        shutil.copy("%s.tar.xz" % keyring_path, gpg_path)
+        shutil.copy("%s.tar.xz.asc" % keyring_path, gpg_path)
+
+        return True
+
+    def remove_channel(self, channel_name):
+        """
+            Remove a channel and everything it contains.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            channel_path = os.path.join(self.path, channel_name)
+            if os.path.exists(channel_path) and \
+               "alias" not in channels[channel_name] and \
+               "redirect" not in channels[channel_name]:
+                shutil.rmtree(channel_path)
+            channels.pop(channel_name)
+
+        return True
+
+    def remove_device(self, channel_name, device_name):
+        """
+            Remove a device and everything it contains.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            if device_name not in channels[channel_name]['devices']:
+                raise KeyError("Couldn't find device: %s" % device_name)
+
+            device_path = os.path.join(self.path, channel_name, device_name)
+            if os.path.exists(device_path):
+                shutil.rmtree(device_path)
+            channels[channel_name]['devices'].pop(device_name)
+
+        self.sync_aliases(channel_name)
+        self.sync_redirects(channel_name)
+
+        return True
+
+    def rename_channel(self, old_name, new_name):
+        """
+            Rename a channel.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if old_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % old_name)
+
+            if new_name in channels:
+                raise KeyError("Channel already exists: %s" % new_name)
+
+            old_channel_path = os.path.join(self.path, old_name)
+            new_channel_path = os.path.join(self.path, new_name)
+            if "redirect" not in channels[old_name]:
+                if os.path.exists(new_channel_path):
+                    raise Exception("Channel path already exists: %s" %
+                                    new_channel_path)
+
+                if not os.path.exists(os.path.dirname(new_channel_path)):
+                    os.makedirs(os.path.dirname(new_channel_path))
+                if os.path.exists(old_channel_path):
+                    os.rename(old_channel_path, new_channel_path)
+
+            channels[new_name] = dict(channels[old_name])
+
+            if "redirect" not in channels[new_name]:
+                for device_name in channels[new_name]['devices']:
+                    index_path = "/%s/%s/index.json" % (new_name, device_name)
+                    channels[new_name]['devices'][device_name]['index'] = \
+                        index_path
+
+                    with index_json(self.config, "%s/%s" %
+                                    (self.path, index_path), True) as index:
+                        for image in index['images']:
+                            for entry in image['files']:
+                                entry['path'] = entry['path'] \
+                                    .replace("/%s/" % old_name,
+                                             "/%s/" % new_name)
+                                entry['signature'] = entry['signature'] \
+                                    .replace("/%s/" % old_name,
+                                             "/%s/" % new_name)
+
+            channels.pop(old_name)
+
+        return True
+
+    def show_channel(self, channel_name):
+        """
+            Show a channel from the client's list.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            if "hidden" in channels[channel_name]:
+                channels[channel_name].pop("hidden")
+
+        return True
+
+    def set_device_keyring(self, channel_name, device_name, path):
+        """
+            Update the keyring entry for the given channel and device.
+            Passing None as the path will unset any existing value.
+        """
+
+        with channels_json(self.config, self.indexpath, True) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            if device_name not in channels[channel_name]['devices']:
+                raise KeyError("Couldn't find device: %s" % device_name)
+
+            abspath, relpath = tools.expand_path(path, self.path)
+
+            if not os.path.exists(abspath):
+                raise Exception("Specified GPG keyring doesn't exists: %s" %
+                                abspath)
+
+            if not os.path.exists("%s.asc" % abspath):
+                raise Exception("The GPG keyring signature doesn't exists: "
+                                "%s.asc" % abspath)
+
+            keyring = {}
+            keyring['path'] = "/%s" % "/".join(relpath.split(os.sep))
+            keyring['signature'] = "/%s.asc" % "/".join(relpath.split(os.sep))
+
+            channels[channel_name]['devices'][device_name]['keyring'] = keyring
+
+        return True
+
+    def sync_alias(self, channel_name):
+        """
+            Update a channel with data from its parent.
+        """
+
+        with channels_json(self.config, self.indexpath) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+            if "alias" not in channels[channel_name] or \
+                    channels[channel_name]['alias'] == channel_name:
+                raise TypeError("Not a channel alias")
+
+            target_name = channels[channel_name]['alias']
+
+            if target_name not in channels:
+                raise KeyError("Couldn't find target channel: %s" %
+                               target_name)
+
+            # Start by looking for added/removed devices
+            devices = set(channels[channel_name]['devices'].keys())
+            target_devices = set(channels[target_name]['devices'].keys())
+
+            # # Remove any removed device
+            for device in devices - target_devices:
+                self.remove_device(channel_name, device)
+
+            # # Add any missing device
+            for device in target_devices - devices:
+                self.create_device(channel_name, device)
+
+            # Iterate through all the devices to import builds
+            for device_name in target_devices:
+                device = self.get_device(channel_name, device_name)
+                target_device = self.get_device(target_name, device_name)
+
+                # Extract all the current builds
+                device_images = {(image['version'], image.get('base', None),
+                                  image['type'])
+                                 for image in device.list_images()}
+
+                target_images = {(image['version'], image.get('base', None),
+                                  image['type'])
+                                 for image in target_device.list_images()}
+
+                # Remove any removed image
+                for image in device_images - target_images:
+                    device.remove_image(image[2], image[0], base=image[1])
+
+                # Create the path if it doesn't exist
+                if not os.path.exists(device.path):
+                    os.makedirs(device.path)
+
+                # Add any missing image
+                with index_json(self.config, device.indexpath, True) as index:
+                    for image in sorted(target_images - device_images):
+                        orig = [entry for entry in target_device.list_images()
+                                if entry['type'] == image[2] and
+                                entry['version'] == image[0] and
+                                entry.get('base', None) == image[1]]
+
+                        entry = copy.deepcopy(orig[0])
+
+                        # Remove the current version tarball
+                        version_detail = None
+                        version_index = len(entry['files'])
+                        for fentry in entry['files']:
+                            if fentry['path'].endswith("version-%s.tar.xz" %
+                                                       entry['version']):
+
+                                version_path = "%s/%s" % (
+                                    self.config.publish_path, fentry['path'])
+
+                                if os.path.exists(
+                                        version_path.replace(".tar.xz",
+                                                             ".json")):
+                                    with open(
+                                            version_path.replace(
+                                                ".tar.xz", ".json")) as fd:
+                                        metadata = json.loads(fd.read())
+                                        if "channel.ini" in metadata:
+                                            version_detail = \
+                                                metadata['channel.ini'].get(
+                                                    "version_detail", None)
+
+                                version_index = fentry['order']
+                                entry['files'].remove(fentry)
+                                break
+
+                        # Generate a new one
+                        path = os.path.join(device.path,
+                                            "version-%s.tar.xz" %
+                                            entry['version'])
+                        abspath, relpath = tools.expand_path(path,
+                                                             device.pub_path)
+                        if not os.path.exists(abspath):
+                            tools.generate_version_tarball(
+                                self.config, channel_name, device_name,
+                                str(entry['version']),
+                                abspath.replace(".xz", ""),
+                                version_detail=version_detail,
+                                channel_target=target_name)
+                            tools.xz_compress(abspath.replace(".xz", ""))
+                            os.remove(abspath.replace(".xz", ""))
+                            gpg.sign_file(self.config, "image-signing",
+                                          abspath)
+
+                        with open(abspath, "rb") as fd:
+                            checksum = sha256(fd.read()).hexdigest()
+
+                        # Generate the new file entry
+                        version = {}
+                        version['order'] = version_index
+                        version['path'] = "/%s" % "/".join(
+                            relpath.split(os.sep))
+                        version['signature'] = "/%s.asc" % "/".join(
+                            relpath.split(os.sep))
+                        version['checksum'] = checksum
+                        version['size'] = int(os.stat(abspath).st_size)
+
+                        # And add it
+                        entry['files'].append(version)
+                        index['images'].append(entry)
+
+                # Sync phased-percentage
+                versions = sorted({entry[0] for entry in target_images})
+                if versions:
+                    device.set_phased_percentage(
+                        versions[-1],
+                        target_device.get_phased_percentage(versions[-1]))
+
+        return True
+
+    def sync_aliases(self, channel_name):
+        """
+            Update any channel that's an alias of the current one.
+        """
+
+        with channels_json(self.config, self.indexpath) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+        alias_channels = [name
+                          for name, channel
+                          in self.list_channels().items()
+                          if channel.get("alias", None) == channel_name
+                          and name != channel_name]
+
+        for alias_name in alias_channels:
+            self.sync_alias(alias_name)
+
+        return True
+
+    def sync_redirects(self, channel_name):
+        """
+            Update any channel that's a direct of the current one.
+        """
+
+        with channels_json(self.config, self.indexpath) as channels:
+            if channel_name not in channels:
+                raise KeyError("Couldn't find channel: %s" % channel_name)
+
+        redirect_channels = [name
+                             for name, channel
+                             in self.list_channels().items()
+                             if channel.get("redirect", None) == channel_name]
+
+        for redirect_name in redirect_channels:
+            self.remove_channel(redirect_name)
+            self.create_channel_redirect(redirect_name, channel_name)
+
+        return True
+
+
+class Device:
+    def __init__(self, config, path):
+        self.config = config
+        self.pub_path = self.config.publish_path
+        self.path = path
+        self.indexpath = os.path.join(path, "index.json")
+
+    def create_image(self, entry_type, version, description, paths,
+                     base=None, bootme=False, minversion=None):
+        """
+            Add a new image to the index.
+        """
+
+        if len(paths) == 0:
+            raise Exception("No file passed for this image.")
+
+        files = []
+        count = 0
+
+        with index_json(self.config, self.indexpath, True) as index:
+            for path in paths:
+                abspath, relpath = tools.expand_path(path, self.pub_path)
+
+                if not os.path.exists(abspath):
+                    raise Exception("Specified file doesn't exists: %s"
+                                    % abspath)
+
+                if not os.path.exists("%s.asc" % abspath):
+                    raise Exception("The GPG file signature doesn't exists: "
+                                    "%s.asc" % abspath)
+
+                with open(abspath, "rb") as fd:
+                    checksum = sha256(fd.read()).hexdigest()
+
+                files.append({'order': count,
+                              'path': "/%s" % "/".join(relpath.split(os.sep)),
+                              'checksum': checksum,
+                              'signature': "/%s.asc" % "/".join(
+                                  relpath.split(os.sep)),
+                              'size': int(os.stat(abspath).st_size)})
+
+                count += 1
+
+            image = {}
+
+            if entry_type == "delta":
+                if not base:
+                    raise KeyError("Missing base version for delta image.")
+                image['base'] = int(base)
+            elif base:
+                raise KeyError("Base version set for full image.")
+
+            if bootme:
+                image['bootme'] = bootme
+
+            if minversion:
+                if entry_type == "delta":
+                    raise KeyError("Minimum version set for delta image.")
+                image['minversion'] = minversion
+
+            image['description'] = description
+            image['files'] = files
+            image['type'] = entry_type
+            image['version'] = version
+            index['images'].append(image)
+
+        return True
+
+    def expire_images(self, max_images):
+        """
+            Expire images keeping the last <max_images> full images and
+            their deltas. Also remove any delta that has an expired image
+            as its base.
+        """
+
+        full_images = sorted([image for image in self.list_images()
+                              if image['type'] == "full"],
+                             key=lambda image: image['version'])
+
+        to_remove = len(full_images) - max_images
+        if to_remove <= 0:
+            return True
+
+        full_remove = full_images[:to_remove]
+        remove_version = [image['version'] for image in full_remove]
+
+        for image in self.list_images():
+            if image['type'] == "full":
+                if image['version'] in remove_version:
+                    self.remove_image(image['type'], image['version'])
+            else:
+                if (image['version'] in remove_version
+                        or image['base'] in remove_version):
+                    self.remove_image(image['type'], image['version'],
+                                      image['base'])
+
+        return True
+
+    def get_image(self, entry_type, version, base=None):
+        """
+            Look for an image and return a dict representation of it.
+        """
+
+        if entry_type not in ("full", "delta"):
+            raise ValueError("Invalid image type: %s" % entry_type)
+
+        if entry_type == "delta" and not base:
+            raise ValueError("Missing base version for delta image.")
+
+        with index_json(self.config, self.indexpath) as index:
+            match = []
+            for image in index['images']:
+                if (image['type'] == entry_type and image['version'] == version
+                        and (image['type'] == "full" or
+                             image['base'] == base)):
+                    match.append(image)
+
+            if len(match) != 1:
+                raise IndexError("Couldn't find a match.")
+
+            return match[0]
+
+    def get_phased_percentage(self, version):
+        """
+            Returns the phasing percentage for a given version.
+        """
+
+        for entry in self.list_images():
+            if entry['version'] == version:
+                if "phased-percentage" in entry:
+                    return entry['phased-percentage']
+                else:
+                    return 100
+        else:
+            raise IndexError("Invalid version number: %s" % version)
+
+    def list_images(self):
+        """
+            Returns a list of all existing images, each image is a dict.
+            This is simply a decoded version of the image array in index.json
+        """
+
+        with index_json(self.config, self.indexpath) as index:
+            return index['images']
+
+    def remove_image(self, entry_type, version, base=None):
+        """
+            Remove an image.
+        """
+
+        image = self.get_image(entry_type, version, base)
+        with index_json(self.config, self.indexpath, True) as index:
+            index['images'].remove(image)
+
+        return True
+
+    def set_description(self, entry_type, version, description,
+                        translations={}, base=None):
+        """
+            Set or update an image description.
+        """
+
+        if translations and not isinstance(translations, dict):
+            raise TypeError("translations must be a dict.")
+
+        image = self.get_image(entry_type, version, base)
+
+        with index_json(self.config, self.indexpath, True) as index:
+            for entry in index['images']:
+                if entry != image:
+                    continue
+
+                entry['description'] = description
+                for langid, value in translations.items():
+                    entry['description_%s' % langid] = value
+
+                break
+
+        return True
+
+    def set_phased_percentage(self, version, percentage):
+        """
+            Set the phasing percentage on an image version.
+        """
+
+        if not isinstance(percentage, int):
+            raise TypeError("percentage must be an integer.")
+
+        if percentage < 0 or percentage > 100:
+            raise ValueError("percentage must be >= 0 and <= 100.")
+
+        with index_json(self.config, self.indexpath, True) as index:
+            versions = sorted({entry['version'] for entry in index['images']})
+
+            last_version = None
+            if versions:
+                last_version = versions[-1]
+
+            if version not in versions:
+                raise IndexError("Version doesn't exist: %s" % version)
+
+            if version != last_version:
+                raise Exception("Phased percentage can only be set on the "
+                                "latest image")
+
+            for entry in index['images']:
+                if entry['version'] == version:
+                    if percentage == 100 and "phased-percentage" in entry:
+                        entry.pop("phased-percentage")
+                    elif percentage != 100:
+                        entry['phased-percentage'] = percentage
+
+        return True

=== added directory 'secret'
=== added directory 'secret/gpg'
=== added directory 'secret/gpg/keyrings'
=== added directory 'secret/gpg/keys'
=== added directory 'secret/ssh'
=== added directory 'state'
=== added directory 'tests'
=== added file 'tests/generate-keys'
--- tests/generate-keys	1970-01-01 00:00:00 +0000
+++ tests/generate-keys	2014-10-10 11:11:17 +0000
@@ -0,0 +1,52 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+
+import sys
+sys.path.insert(0, 'lib')
+
+from systemimage import gpg
+
+target_dir = "tests/keys/"
+if not os.path.exists(target_dir):
+    raise Exception("Missing tests/keys directory")
+
+keys = (("archive-master", "[TESTING] Ubuntu Archive Master Signing Key",
+         "[e-mail]", 0),
+        ("image-master", "[TESTING] Ubuntu System Image Master Signing Key",
+         "[e-mail]", 0),
+        ("image-signing", "[TESTING] Ubuntu System Image Signing Key (YYYY)",
+         "[e-mail]", "2y"),
+        ("device-signing", "[TESTING] Random OEM Signing Key (YYYY)",
+         "[e-mail]", "2y"))
+
+for key_name, key_description, key_email, key_expiry in keys:
+    key_dir = "%s/%s/" % (target_dir, key_name)
+    if os.path.exists(key_dir):
+        shutil.rmtree(key_dir)
+    os.makedirs(key_dir)
+
+    uid = gpg.generate_signing_key(key_dir, key_description, key_email,
+                                   key_expiry)
+
+    print("%s <%s>" % (uid.name, uid.email))
+
+# All done, let's mark it as done
+open("tests/keys/generated", "w+").close()

=== added directory 'tests/keys'
=== added file 'tests/run'
--- tests/run	1970-01-01 00:00:00 +0000
+++ tests/run	2014-10-10 11:11:17 +0000
@@ -0,0 +1,60 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Dependencies:
+#  - python2 (>= 2.7): python-gpgme, python-coverage
+#  - python3 (>= 3.2): python3-gpgme
+
+import glob
+import os
+import re
+import shutil
+import sys
+import unittest
+
+coverage = True
+try:
+    from coverage import coverage
+    cov = coverage()
+    cov.start()
+except ImportError:
+    print("No coverage report, make sure python-coverage is installed")
+    coverage = False
+
+sys.path.insert(0, 'lib')
+
+if len(sys.argv) > 1:
+    test_filter = sys.argv[1]
+else:
+    test_filter = ''
+
+tests = [t[:-3] for t in os.listdir('tests')
+         if t.startswith('test_') and t.endswith('.py') and
+         re.search(test_filter, t)]
+tests.sort()
+suite = unittest.TestLoader().loadTestsFromNames(tests)
+res = unittest.TextTestRunner(verbosity=2).run(suite)
+
+if coverage:
+    if os.path.exists('tests/coverage'):
+        shutil.rmtree('tests/coverage')
+    cov.stop()
+    cov.html_report(include=glob.glob("lib/systemimage/*.py"),
+                    directory='tests/coverage')
+    print("")
+    cov.report(include=glob.glob("lib/systemimage/*.py"))

=== added file 'tests/test_config.py'
--- tests/test_config.py	1970-01-01 00:00:00 +0000
+++ tests/test_config.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,281 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from systemimage import config
+from systemimage import tools
+
+try:
+    from unittest import mock
+except ImportError:
+    import mock
+
+
+class ConfigTests(unittest.TestCase):
+    def setUp(self):
+        temp_directory = tempfile.mkdtemp()
+        self.temp_directory = temp_directory
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_directory)
+
+    @mock.patch("subprocess.call")
+    def test_config(self, mock_call):
+        # Good complete config
+        config_path = os.path.join(self.temp_directory, "config")
+        key_path = os.path.join(self.temp_directory, "key")
+
+        with open(config_path, "w+") as fd:
+            fd.write("""[global]
+base_path = %s
+mirrors = a, b
+
+[mirror_default]
+ssh_user = user
+ssh_key = key
+ssh_port = 22
+ssh_command = command
+
+[mirror_a]
+ssh_host = hosta
+
+[mirror_b]
+ssh_host = hostb
+""" % self.temp_directory)
+
+        conf = config.Config(config_path)
+
+        # Test ssh sync
+        tools.sync_mirrors(conf)
+        expected_calls = [((['ssh', '-i', key_path, '-l', 'user',
+                             '-p', '22', 'hosta', 'command'],), {}),
+                          ((['ssh', '-i', key_path, '-l', 'user',
+                             '-p', '22', 'hostb', 'command'],), {})]
+        self.assertEquals(mock_call.call_args_list, expected_calls)
+
+        # Invalid config
+        invalid_config_path = os.path.join(self.temp_directory,
+                                           "invalid_config")
+        with open(invalid_config_path, "w+") as fd:
+            fd.write("""invalid""")
+
+        self.assertEquals(config.parse_config(invalid_config_path), {})
+
+        self.assertRaises(
+            Exception, config.Config, os.path.join(self.temp_directory,
+                                                   "invalid"))
+
+        # Test loading config from default location
+        config_file = os.path.join(os.path.dirname(config.__file__),
+                                   "../../etc/config")
+
+        old_pwd = os.getcwd()
+        os.chdir(self.temp_directory)
+        if not os.path.exists(config_file):
+            self.assertRaises(Exception, config.Config)
+        else:
+            self.assertTrue(config.Config())
+        os.chdir(old_pwd)
+
+        # Empty config
+        empty_config_path = os.path.join(self.temp_directory,
+                                         "empty_config")
+        with open(empty_config_path, "w+") as fd:
+            fd.write("")
+
+        conf = config.Config(empty_config_path)
+        self.assertEquals(conf.base_path, os.getcwd())
+
+        # Single mirror config
+        single_mirror_config_path = os.path.join(self.temp_directory,
+                                                 "single_mirror_config")
+        with open(single_mirror_config_path, "w+") as fd:
+            fd.write("""[global]
+mirrors = a
+
+[mirror_default]
+ssh_user = user
+ssh_key = key
+ssh_port = 22
+ssh_command = command
+
+[mirror_a]
+ssh_host = host
+""")
+
+        conf = config.Config(single_mirror_config_path)
+        self.assertEquals(conf.mirrors['a'].ssh_command, "command")
+
+        # Missing mirror_default
+        missing_default_config_path = os.path.join(self.temp_directory,
+                                                   "missing_default_config")
+        with open(missing_default_config_path, "w+") as fd:
+            fd.write("""[global]
+mirrors = a
+
+[mirror_a]
+ssh_host = host
+""")
+
+        self.assertRaises(KeyError, config.Config, missing_default_config_path)
+
+        # Missing mirror key
+        missing_key_config_path = os.path.join(self.temp_directory,
+                                               "missing_key_config")
+        with open(missing_key_config_path, "w+") as fd:
+            fd.write("""[global]
+mirrors = a
+
+[mirror_default]
+ssh_user = user
+ssh_port = 22
+ssh_command = command
+
+[mirror_a]
+ssh_host = host
+""")
+
+        self.assertRaises(KeyError, config.Config, missing_key_config_path)
+
+        # Missing mirror
+        missing_mirror_config_path = os.path.join(self.temp_directory,
+                                                  "missing_mirror_config")
+        with open(missing_mirror_config_path, "w+") as fd:
+            fd.write("""[global]
+mirrors = a
+
+[mirror_default]
+ssh_user = user
+ssh_port = 22
+ssh_command = command
+ssh_key = key
+""")
+
+        self.assertRaises(KeyError, config.Config, missing_mirror_config_path)
+
+        # Missing ssh_host
+        missing_host_config_path = os.path.join(self.temp_directory,
+                                                "missing_host_config")
+        with open(missing_host_config_path, "w+") as fd:
+            fd.write("""[global]
+mirrors = a
+
+[mirror_default]
+ssh_user = user
+ssh_port = 22
+ssh_command = command
+ssh_key = key
+
+[mirror_a]
+ssh_user = other-user
+""")
+
+        self.assertRaises(KeyError, config.Config, missing_host_config_path)
+
+        # Test with env path
+        test_path = os.path.join(self.temp_directory, "a", "b")
+        os.makedirs(os.path.join(test_path, "etc"))
+        with open(os.path.join(test_path, "etc", "config"), "w+") as fd:
+            fd.write("[global]\nbase_path = a/b/c")
+        os.environ['SYSTEM_IMAGE_ROOT'] = test_path
+        test_config = config.Config()
+        self.assertEquals(test_config.base_path, "a/b/c")
+
+        # Test the channels config
+        # # Multiple channels
+        channel_config_path = os.path.join(self.temp_directory,
+                                           "channel_config")
+        with open(channel_config_path, "w+") as fd:
+            fd.write("""[global]
+channels = a, b
+
+[channel_a]
+type = manual
+fullcount = 10
+
+[channel_b]
+type = auto
+versionbase = 5
+deltabase = a, b
+files = a, b
+file_a = test;arg1;arg2
+file_b = test;arg3;arg4
+""")
+
+        conf = config.Config(channel_config_path)
+        self.assertEquals(
+            conf.channels['b'].files,
+            [{'name': 'a', 'generator': 'test',
+              'arguments': ['arg1', 'arg2']},
+             {'name': 'b', 'generator': 'test',
+              'arguments': ['arg3', 'arg4']}])
+
+        self.assertEquals(conf.channels['a'].fullcount, 10)
+        self.assertEquals(conf.channels['a'].versionbase, 1)
+        self.assertEquals(conf.channels['a'].deltabase, ['a'])
+
+        self.assertEquals(conf.channels['b'].fullcount, 0)
+        self.assertEquals(conf.channels['b'].versionbase, 5)
+        self.assertEquals(conf.channels['b'].deltabase, ["a", "b"])
+
+        # # Single channel
+        single_channel_config_path = os.path.join(self.temp_directory,
+                                                  "single_channel_config")
+        with open(single_channel_config_path, "w+") as fd:
+            fd.write("""[global]
+channels = a
+
+[channel_a]
+deltabase = a
+versionbase = 1
+files = a
+file_a = test;arg1;arg2
+""")
+
+        conf = config.Config(single_channel_config_path)
+        self.assertEquals(
+            conf.channels['a'].files,
+            [{'name': 'a', 'generator': 'test',
+              'arguments': ['arg1', 'arg2']}])
+
+        # # Invalid channel
+        invalid_channel_config_path = os.path.join(self.temp_directory,
+                                                   "invalid_channel_config")
+        with open(invalid_channel_config_path, "w+") as fd:
+            fd.write("""[global]
+channels = a
+""")
+
+        self.assertRaises(KeyError, config.Config, invalid_channel_config_path)
+
+        # # Invalid file
+        invalid_file_channel_config_path = os.path.join(
+            self.temp_directory, "invalid_file_channel_config")
+        with open(invalid_file_channel_config_path, "w+") as fd:
+            fd.write("""[global]
+channels = a
+
+[channel_a]
+files = a
+""")
+
+        self.assertRaises(KeyError, config.Config,
+                          invalid_file_channel_config_path)

=== added file 'tests/test_diff.py'
--- tests/test_diff.py	1970-01-01 00:00:00 +0000
+++ tests/test_diff.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,265 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import shutil
+import sys
+import tarfile
+import tempfile
+import unittest
+
+from io import BytesIO, StringIO
+from systemimage.diff import ImageDiff, compare_files
+
+
+class DiffTests(unittest.TestCase):
+    def setUp(self):
+        temp_directory = tempfile.mkdtemp()
+
+        source_tarball_path = "%s/source.tar" % temp_directory
+        target_tarball_path = "%s/target.tar" % temp_directory
+
+        source_tarball = tarfile.open(source_tarball_path, "w")
+        target_tarball = tarfile.open(target_tarball_path, "w")
+
+        # Standard file
+        a = tarfile.TarInfo()
+        a.name = "a"
+        a.size = 4
+
+        # Standard file
+        b = tarfile.TarInfo()
+        b.name = "b"
+        b.size = 4
+
+        # Standard directory
+        c_dir = tarfile.TarInfo()
+        c_dir.name = "c"
+        c_dir.type = tarfile.DIRTYPE
+        c_dir.mode = 0o755
+
+        # Standard file
+        c = tarfile.TarInfo()
+        c.name = "c/c"
+        c.size = 4
+
+        # Standard file
+        d_source = tarfile.TarInfo()
+        d_source.name = "c/d"
+        d_source.size = 8
+        d_source.mtime = 1000
+
+        # Standard file
+        d_target = tarfile.TarInfo()
+        d_target.name = "c/d"
+        d_target.size = 8
+        d_target.mtime = 1234
+
+        # Symlink
+        e = tarfile.TarInfo()
+        e.name = "e"
+        e.type = tarfile.SYMTYPE
+        e.linkname = "a"
+
+        # Hard link
+        f = tarfile.TarInfo()
+        f.name = "f"
+        f.type = tarfile.LNKTYPE
+        f.linkname = "a"
+
+        # Standard file
+        g_source = tarfile.TarInfo()
+        g_source.name = "c/g"
+        g_source.size = 4
+        g_source.mtime = 1000
+
+        # Standard file
+        g_target = tarfile.TarInfo()
+        g_target.name = "c/g"
+        g_target.size = 4
+        g_target.mtime = 1001
+
+        # Hard link
+        h_source = tarfile.TarInfo()
+        h_source.name = "c/h"
+        h_source.type = tarfile.LNKTYPE
+        h_source.linkname = "d"
+        h_source.mtime = 1000
+
+        # Hard link
+        h_target = tarfile.TarInfo()
+        h_target.name = "c/h"
+        h_target.type = tarfile.LNKTYPE
+        h_target.linkname = "d"
+        h_target.mtime = 1001
+
+        # Hard link
+        i = tarfile.TarInfo()
+        i.name = "c/a_i"
+        i.type = tarfile.LNKTYPE
+        i.linkname = "c"
+
+        # Dangling symlink
+        j = tarfile.TarInfo()
+        j.name = "c/j"
+        j.type = tarfile.SYMTYPE
+        j.linkname = "j_non-existent"
+
+        # Standard directory
+        k_dir = tarfile.TarInfo()
+        k_dir.name = "dir"
+        k_dir.type = tarfile.DIRTYPE
+        k_dir.mode = 0o755
+
+        # Dangling symlink
+        l = tarfile.TarInfo()
+        l.name = "dir"
+        l.type = tarfile.SYMTYPE
+        l.linkname = "l_non-existent"
+
+        # Standard file
+        m_source = tarfile.TarInfo()
+        m_source.name = "m"
+        m_source.size = 4
+
+        # Hard link
+        m_target = tarfile.TarInfo()
+        m_target.name = "m"
+        m_target.type = tarfile.LNKTYPE
+        m_target.linkname = "n"
+
+        # Hard link
+        n_source = tarfile.TarInfo()
+        n_source.name = "n"
+        n_source.type = tarfile.LNKTYPE
+        n_source.linkname = "m"
+
+        # Standard file
+        n_target = tarfile.TarInfo()
+        n_target.name = "n"
+        n_target.size = 4
+
+        # Hard link
+        o_source = tarfile.TarInfo()
+        o_source.name = "system/o.1"
+        o_source.type = tarfile.LNKTYPE
+        o_source.linkname = "system/o"
+
+        # Standard file
+        o_target = tarfile.TarInfo()
+        o_target.name = "system/o"
+        o_target.size = 4
+
+        source_tarball.addfile(a, BytesIO(b"test"))
+        source_tarball.addfile(a, BytesIO(b"test"))
+        source_tarball.addfile(a, BytesIO(b"test"))
+        source_tarball.addfile(b, BytesIO(b"test"))
+        source_tarball.addfile(c_dir)
+        source_tarball.addfile(d_source, BytesIO(b"test-abc"))
+        source_tarball.addfile(g_source, BytesIO(b"test"))
+        source_tarball.addfile(h_source, BytesIO(b"test"))
+        source_tarball.addfile(k_dir)
+        source_tarball.addfile(m_source, BytesIO(b"test"))
+        source_tarball.addfile(n_source)
+
+        target_tarball.addfile(a, BytesIO(b"test"))
+        target_tarball.addfile(c_dir)
+        target_tarball.addfile(c, BytesIO(b"test"))
+        target_tarball.addfile(d_target, BytesIO(b"test-def"))
+        target_tarball.addfile(e)
+        target_tarball.addfile(f)
+        target_tarball.addfile(g_target, BytesIO(b"test"))
+        target_tarball.addfile(h_target, BytesIO(b"test"))
+        target_tarball.addfile(i)
+        target_tarball.addfile(j)
+        target_tarball.addfile(l)
+        target_tarball.addfile(n_target, BytesIO(b"test"))
+        target_tarball.addfile(m_target)
+        target_tarball.addfile(o_source)
+        target_tarball.addfile(o_target)
+
+        source_tarball.close()
+        target_tarball.close()
+
+        self.imagediff = ImageDiff(source_tarball_path, target_tarball_path)
+        self.source_tarball_path = source_tarball_path
+        self.target_tarball_path = target_tarball_path
+        self.temp_directory = temp_directory
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_directory)
+
+    def test_content(self):
+        content_set, content_dict = self.imagediff.scan_content("source")
+        self.assertEquals(sorted(content_dict.keys()),
+                          ['a', 'b', 'c', 'c/d', 'c/g', 'c/h', 'dir', 'm',
+                           'n'])
+
+        content_set, content_dict = self.imagediff.scan_content("target")
+        self.assertEquals(sorted(content_dict.keys()),
+                          ['a', 'c', 'c/a_i', 'c/c', 'c/d', 'c/g', 'c/h',
+                           'c/j', 'dir', 'e', 'f', 'm', 'n', 'system/o',
+                           'system/o.1'])
+
+    def test_content_invalid_image(self):
+        self.assertRaises(KeyError, self.imagediff.scan_content, "invalid")
+
+    def test_compare_files(self):
+        self.assertEquals(compare_files(None, None), True)
+        self.assertEquals(compare_files(None, BytesIO(b"abc")), False)
+
+    def test_compare_image(self):
+        diff_set = self.imagediff.compare_images()
+        self.assertTrue(("c/a_i", "add") in diff_set)
+
+    def test_print_changes(self):
+        # Redirect stdout
+        old_stdout = sys.stdout
+
+        # FIXME: Would be best to have something that works with both version
+        if sys.version[0] == "3":
+            sys.stdout = StringIO()
+        else:
+            sys.stdout = BytesIO()
+
+        self.imagediff.print_changes()
+
+        # Unredirect stdout
+        output = sys.stdout.getvalue()
+        sys.stdout = old_stdout
+
+        self.assertEquals(output, """ - b (del)
+ - c/a_i (add)
+ - c/c (add)
+ - c/d (mod)
+ - c/j (add)
+ - dir (mod)
+ - e (add)
+ - f (add)
+ - system/o (add)
+ - system/o.1 (add)
+""")
+
+    def test_generate_tarball(self):
+        output_tarball = "%s/output.tar" % self.temp_directory
+
+        self.imagediff.generate_diff_tarball(output_tarball)
+        tarball = tarfile.open(output_tarball, "r")
+
+        files_list = [entry.name for entry in tarball]
+        self.assertEquals(files_list, ['removed', 'c/c', 'c/a_i', 'c/d', 'c/j',
+                                       'dir', 'e', 'f', 'system/o',
+                                       'system/o.1'])

=== added file 'tests/test_generators.py'
--- tests/test_generators.py	1970-01-01 00:00:00 +0000
+++ tests/test_generators.py	2014-10-10 11:11:17 +0000
@@ -0,0 +1,1039 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2013 Canonical Ltd.
+# Author: Stéphane Graber <[e-mail]>