Merge lp:~maas-maintainers/maas/new-import-script-integration into lp:~maas-committers/maas/trunk

[Raphaël Badin]

Merge Oleg Strikov's new version of maas-import-pxe-files into trunk. Previously, m-i-p-f was a bash script; now it's a much more functional python script.

[Graham Binns (gmb)]

Approving on behalf of all who sailed the merry seas of this here branch.

review: Approve

[Jeroen T. Vermeulen (jtv)]

Re-landing this with the latest changes to the feature branch. After that, let's mark this branch as Merged so it doesn't stick around as an open development branch!

review: Needs Fixing

[Jeroen T. Vermeulen (jtv)]

review: Approve

=== modified file 'contrib/maas-cluster-http.conf'
--- contrib/maas-cluster-http.conf	2013-10-07 19:30:44 +0000
+++ contrib/maas-cluster-http.conf	2014-03-25 17:45:57 +0000
@@ -1,7 +1,7 @@
 # Server static files for tftp images as FPI
 # installer needs them
-Alias /MAAS/static/images/ /var/lib/maas/tftp/
-<Directory /var/lib/maas/tftp/>
+Alias /MAAS/static/images/ /var/lib/maas/boot-resources/current/
+<Directory /var/lib/maas/boot-resources/current/>
     <IfVersion >= 2.4>
         Require all granted
     </IfVersion>

=== added file 'etc/maas/bootresources.yaml'
--- etc/maas/bootresources.yaml	1970-01-01 00:00:00 +0000
+++ etc/maas/bootresources.yaml	2014-03-25 17:45:57 +0000
@@ -0,0 +1,62 @@
+##
+## Boot image download configuration.
+##
+
+boot:
+  ## Marker: this configuration has not been initialised yet.
+  #
+  # If this is set to True during installation or a later upgrade, it means
+  # that this configuration file has not been edited by a human or by MAAS
+  # itself.  When you edit this configuration to suit your needs, be sure to
+  # remove this setting, or change it to False, so that a future upgrade will
+  # know not to overwrite the configuration file.
+  #
+  # If the setting is True, the upgrade procedure will look for downloaded
+  # boot images that predate the current, Simplestreams-based import
+  # mechanism, and rewrite the configuration based on what it finds.  The new
+  # configuration will import the new-style equivalents of the images that
+  # were imported in the old setup.
+  configure_me: True
+
+  ## Storage location for boot images.
+  #
+  # These images can get quite large: some files are hundreds of megabytes,
+  # and you may need multiple copies for different architectures, OS releases,
+  # etc.
+  storage: "/var/lib/maas/boot-resources/"
+
+  ## Sources of downloadable boot images.
+  #
+  # Each source downloads from one simplestreams URL, though it is possible to
+  # have multiple sources using the same URL.  For example, if you want to
+  # download "release" images for one OS release and just the "beta-2" images
+  # for another, specify two separate selections.  Otherwise, the importer
+  # would have to download both kinds of images for both releases.
+  sources:
+    - path: "http://maas.ubuntu.com/images/ephemeral-v2/daily/"
+      keyring: "/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg"
+
+      ## Filters for this source's imports.
+      #
+      # Be careful: these items will multiply.  For example if you specify
+      # architectures "i386" and "amd64", and subarchitectures "generic",
+      # "hwe-s", and "hwe-t", the import script will need to import 6 images,
+      # for each possible combination.  If you only want some of those
+      # combinations, keep them as separate sources and/or selections.
+      selections:
+        - release: "trusty"
+          arches: ["i386", "amd64"]
+          subarches: ["generic"]
+        - release: "precise"
+          arches: ["i386", "amd64"]
+          subarches: ["generic"]
+      #selections:
+      #  - release: "*"
+      #    arches: ["*"]
+      #    subarches: ["*"]
+## other source example:
+#    - path: "http://hyperscale.ubuntu.com/images/"
+#      selections:
+#        - release: "*"
+#          arches: ["*"]
+#          subarches: ['*']

=== modified file 'etc/maas/import_ephemerals'
--- etc/maas/import_ephemerals	2014-03-18 00:29:45 +0000
+++ etc/maas/import_ephemerals	2014-03-25 17:45:57 +0000
@@ -1,13 +1,13 @@
 # Legacy configuration file for the maas-import-ephemerals script.
 #
 # This file is obsolete, but the script will still read it for compatibility.
-# Configure /etc/maas/pserv.yaml by preference.
+# Configure /etc/maas/bootresources.yaml by preference.
 
 ## Include settings from import_pxe_files.
 [ ! -f /etc/maas/import_pxe_files ] || . /etc/maas/import_pxe_files
 
 # These options can be defined here, although for future compatibility they
-# should be set in pserv.yaml instead:
+# should be set in bootresources.yaml instead:
 
 #DATA_DIR="/var/lib/maas/ephemeral"
 #ARCHES="amd64/generic i386/generic"

=== modified file 'etc/maas/pserv.yaml'
--- etc/maas/pserv.yaml	2014-02-10 22:32:17 +0000
+++ etc/maas/pserv.yaml	2014-03-25 17:45:57 +0000
@@ -38,31 +38,3 @@
   # generator: http://localhost/MAAS/api/1.0/pxeconfig/
   generator: http://localhost:5243/api/1.0/pxeconfig/
 
-## Boot configuration.
-boot:
-  ## CPU architectures for which boot images should be downloaded from the
-  ## server, e.g. ['i386/generic', 'amd64/generic'].  MAAS needs these images
-  ## in order to boot nodes up.
-  ##
-  ## Leave this option out to download images for all architectures.
-  #
-  # architectures:
-
-  ## Settings for ephemeral boot images.  These images are used when
-  ## commissioning nodes, and during fast-path installation.
-  #
-  ephemeral:
-
-    ## Directory where ephemeral boot images and related state should be
-    ## stored.
-    #
-    #   images_directory: /var/lib/maas/ephemeral
-    images_directory: /var/lib/maas/ephemeral
-
-    ## Releases for which ephemeral images should be downloaded.
-    ## These images are quite large (about a quarter GB each), so you may want
-    ## to restrict these separately even if you do want the regular install
-    ## images for all releases.  Leave this out to download all currently
-    ## supported releases.
-    #
-    # releases:

=== removed symlink 'etc/maas/templates/pxe/config.commissioning.armhf.template'
=== target was u'config.install.armhf.template'
=== modified file 'etc/maas/templates/pxe/config.install.armhf.template'
--- etc/maas/templates/pxe/config.install.armhf.template	2013-10-24 02:46:03 +0000
+++ etc/maas/templates/pxe/config.install.armhf.template	2014-03-25 17:45:57 +0000
@@ -2,11 +2,6 @@
 
 LABEL execute
   {{# SAY is not implemented in U-Boot }}
-  {{if kernel_params.release not in ("precise", "quantal")}}
-    {{# Return a copy of kernel_params with an overridden subarch.
-        See https://bugs.launchpad.net/maas/+bug/1166994 }}
-    {{py: kernel_params=kernel_params(subarch="generic")}}
-  {{endif}}
   KERNEL {{kernel_params | kernel_path }}
   INITRD {{kernel_params | initrd_path }}
   APPEND {{kernel_params | kernel_command}}

=== modified file 'scripts/maas-import-pxe-files'
--- scripts/maas-import-pxe-files	2014-03-25 09:25:40 +0000
+++ scripts/maas-import-pxe-files	2014-03-25 17:45:57 +0000
@@ -1,447 +1,40 @@
-#!/usr/bin/env bash
-# Copyright 2012-2014 Canonical Ltd.  This software is licensed under the
+#!/usr/bin/env python2.7
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
-#
-# Download static files needed for net-booting nodes through TFTP:
-# pre-boot loader, kernels, and initrd images.
-#
-# This script downloads the required files into the TFTP home directory
-# (by default, /var/lib/maas/tftp).  Run it with the necessarily privileges
-# to write them there.
-
-# Exit immediately if a command exits with a non-zero status.
-set -o errexit
-# Treat unset variables as an error when substituting.
-set -o nounset
-
-# Load settings if available.
-settings="/etc/maas/import_pxe_files"
-[ -r $settings ] && . $settings
-
-# Location of the GPG keyring for the Ubuntu archive.
-GPG_KEYRING="${GPG_KEYRING:-/usr/share/keyrings/ubuntu-archive-keyring.gpg}"
-
-# Whether to skip checking GPG keys (necessary for testing of this script).
-# Set this to a nonempty string to skip checking.  This is needed for test
-# programs, because they can't possibly sign with the right key.
-IGNORE_GPG="${IGNORE_GPG:-}"
-
-# Download locations for Ubuntu releases.  When the cluster controller runs
-# the import scripts, it provides settings from the server side.
-MAIN_ARCHIVE=${MAIN_ARCHIVE:-http://archive.ubuntu.com/ubuntu/}
-PORTS_ARCHIVE=${PORTS_ARCHIVE:-http://ports.ubuntu.com/ubuntu-ports/}
-
-# Ubuntu releases that are to be downloaded.
-SUPPORTED_RELEASES=$(distro-info --supported)
-RELEASES=${RELEASES:-$SUPPORTED_RELEASES}
-
-# The current Ubuntu release.
-STABLE_RELEASE=${STABLE_RELEASE:-$(distro-info --stable)}
-
-# Supported architectures.
-ARCHES=${ARCHES:-amd64/generic i386/generic armhf/highbank armhf/generic}
-
-# Command line to download a resource at a given URL into the current
-# directory.  A wget command line will work here, but curl will do as well.
-DOWNLOAD=${DOWNLOAD:-wget --no-verbose}
-
-# Whether to download ephemeral images as well: "1" for yes, "0" for no.
-# Default is yes.
-IMPORT_EPHEMERALS=${IMPORT_EPHEMERALS:-1}
-
-# Whether to check for broken grub2 efinet: "1" for yes, "0" for no.
-# Default is yes.
-CHECK_BROKEN_EFINET=${CHECK_BROKEN_EFINET:-1}
-
-# Whether to skip getting shim-signed from amd64-binary archive: "1" for yes, "0" for no.
-# This is used for testing, Default is no.
-SKIP_SHIM_SIGNED=${SKIP_SHIM_SIGNED:-0}
-
-fail() {
-    local msg=$1
-
-    echo $msg >&2
-    exit 1
-}
-
-# Show script usage/summary.
-show_usage() {
-    echo "Usage: ${0##*/}"
-    echo
-    echo "This helper script downloads the relevant boot images from an "
-    echo "Ubuntu archive and uses 'maas' to provision them for PXE booting "
-    echo "from TFTP."
-    echo
-    echo "This script takes no arguments, but you can adjust some parameters "
-    echo -e "by editing the config file found at \033[1m$settings\033[0m."
-    echo
-    echo "MAAS homepage:<http://maas.ubuntu.com>"
-    echo
-}
-
-# Return a URL that points to the images directory for the relevant
-# release/arch.
-compose_installer_base_url() {
-    local arch=$1 release=$2
-
-    case $arch in
-        amd64/*|i386/*)
-            local installer_url="$MAIN_ARCHIVE/dists/$release/main/installer-${arch%%/*}"
-            echo "$installer_url/current/images/"
-            ;;
-        armhf/*)
-            # No ARM server installers were available in precise, so always go for -updates for now
-            # A better general fix is LP: #1052397
-            if [ "$release" = "precise" ]; then
-                updates=-updates
-            else
-                updates=
-            fi
-            local installer_url="$PORTS_ARCHIVE/dists/${release}${updates}/main/installer-${arch%%/*}"
-            echo "$installer_url/current/images/"
-            ;;
-        *)
-            echo "Unknown architecture: $arch" >&2
-            exit 1
-            ;;
-    esac
-}
-
-# Return the URL part that is appended to the base url that gives the location
-# of the images.
-compose_installer_download_url_postfix() {
-    local arch=$1
-
-    case $arch in
-        amd64/*|i386/*)
-            echo "netboot/ubuntu-installer/${arch%%/*}/"
-            ;;
-        armhf/*)
-            echo "${arch#*/}/netboot/"
-            ;;
-        *)
-            echo "Unknown architecture: $arch" >&2
-            exit 1
-            ;;
-    esac
-}
-
-# Put together a full URL for where the installer files for architecture $1
-# and release $2 can be downloaded.
-compose_installer_download_url() {
-    local arch=$1 release=$2
-
-    base_url=$(compose_installer_base_url $arch $release)
-    postfix=$(compose_installer_download_url_postfix $arch)
-
-    echo "$base_url/$postfix"
-}
-
-# Fetch MD5SUMS file.  This returns false (i.e. nonzero) in the case of
-# survivable failure, so that the caller can skip this image and move on.
-fetch_server_md5sums() {
-    local base_url=$1
-
-    if ! $DOWNLOAD "$base_url/MD5SUMS"
-    then
-        echo "Unable to download $base_url/MD5SUMS" >&2
-        return 1
-    fi
-
-    if [ "x$IGNORE_GPG" == "x" ]
-    then
-        if ! $DOWNLOAD "$base_url/MD5SUMS.gpg"
-        then
-            echo "Unable to download $base_url/MD5SUMS.gpg" >&2
-            return 1
-        fi
-
-        if ! gpg --keyring=$GPG_KEYRING --verify MD5SUMS.gpg MD5SUMS >/dev/null 2>/dev/null
-        then
-            echo "Failed to verify MD5SUMS via $GPG_KEYRING ($base_url/MD5SUMS)" >&2
-            return 1
-        fi
-    fi
-    return 0
-}
-
-get_md5sum_for_file() {
-    local filename=$1
-
-    # The filename supplied in $1 must be the full path as seen in the
-    # MD5SUMS file.  The files are rooted from a single place so the grepped
-    # string will only match once.
-    server_md5sum=$(grep $filename MD5SUMS|awk '{print $1}') ||
-        fail "failed to find checksum for $filename"
-    echo $server_md5sum
-}
-
-check_checksum() {
-    local server_md5sum=$1 file_on_disk=$2
-    local md5sum
-
-    md5sum=$(md5sum $file_on_disk|awk '{print $1}')
-
-    if [ "$md5sum" != "$server_md5sum" ]; then
-        fail "md5 checksum mismatch for $file_on_disk: expected $server_md5sum, got $md5sum"
-    fi
-}
-
-# Return a list of files for architecture $1 and release $2 that need to be
-# downloaded
-compose_installer_download_files() {
-    local arch=$1 release=$2
-
-    case $arch in
-        amd64/*|i386/*)
-            echo "linux initrd.gz"
-            ;;
-        armhf/*)
-            echo "vmlinuz initrd.gz"
-            ;;
-        *)
-            echo "Unknown architecture: $arch" >&2
-            exit 1
-            ;;
-    esac
-}
-
-
-# Rename downloaded files for architecture $1 and release $2 into the form that
-# MAAS expects them
-rename_installer_download_files() {
-    local arch=$1 release=$2
-
-    case $arch in
-        amd64/*|i386/*)
-            # do nothing
-            ;;
-        armhf/*)
-            mv vmlinuz linux
-            ;;
-        *)
-            echo "Unknown architecture: $arch" >&2
-            exit 1
-            ;;
-    esac
-}
-
-
-# Copy the pre-boot loader pxelinux.0, and modules we need, from the
-# installed syslinux version.  Install it into the TFTP tree for
-# netbooting.
-update_pre_boot_loader() {
-    for loader_file in pxelinux.0 chain.c32 ifcpu64.c32
-    do
-        maas-provision install-pxe-bootloader \
-            --loader="/usr/lib/syslinux/$loader_file"
-    done
-}
-
-# Download load the archive package list, and check that it contains
-# the needed package.
-download_archive_package_list() {
-    local archive_url="$MAIN_ARCHIVE/dists/$1/main/binary-amd64/Packages.gz"
-    if $DOWNLOAD $archive_url
-    then
-        gunzip "Packages.gz"
-        if grep -Fxq "Package: $2" "Packages"
-        then
-            return 0
-        fi
-    fi
-    return 1
-}
-
-# Copy the signed efi and grubnet loader. Install it into the TFTP tree
-# for UEFI netbooting.
-update_uefi_boot_loader() {
-    local download_dir=$(mktemp -d)
-    pushd "$download_dir" >/dev/null
-
-    # Resolves bug=1295644 - Download the package manually from the
-    # archive giving support for other architectures.
-    if test "$SKIP_SHIM_SIGNED" != "1"
-    then
-        if ! download_archive_package_list "$STABLE_RELEASE-updates" "shim-signed"
-        then
-            if ! download_archive_package_list "$STABLE_RELEASE" "shim-signed"
-            then
-                echo "Failed to obtain shim-signed." >&2
-                popd >/dev/null
-                rm -rf -- "$download_dir"
-                return 1
-            fi
-        fi
-
-        # Extract the shim-signed package information.
-        local package="$(awk "/^Package: shim-signed$/{f=1}f;/MD5sum: .+$/{f=0}" Packages)"
-        local filepath="$(echo "${package}" | sed -n "s/^Filename: \(.*\)$/\1/p")"
-        local filename="${filepath##*/}"
-        local md5sum="$(echo "${package}" | sed -n "s/^MD5sum: \(.*\)$/\1/p")"
-        local url="${MAIN_ARCHIVE}/${filepath}"
-        
-        # Download the package from the pool and check that md5sum match.
-        trap 'popd >/dev/null; rm -rf -- "$download_dir"' RETURN
-        if ! $DOWNLOAD $url
-        then
-            echo "Failed to download ${filename}." >&2
-            return 1
-        fi
-        check_checksum "$md5sum" "$filename"
-
-        # Extract the shim.efi.signed file from the package, and install it for maas to use.
-        dpkg -x "$filename" .
-        mv "usr/lib/shim/shim.efi.signed" "bootx64.efi"
-        maas-provision install-pxe-bootloader --loader="bootx64.efi"
-    fi
-
-    local grub_version=$STABLE_RELEASE
-    if test "$CHECK_BROKEN_EFINET" != "0"
-    then
-        # grubnetx64 will not work below version trusty, as efinet is broken
-        # when loading kernel, warn and force trusty
-        if [ $grub_version == "saucy" -o $grub_version == "quantal" -o $grub_version == "precise" ];
-        then
-            echo "Warning: ${grub_version} and ${grub_version}-updates doesn't support UEFI, switching to trusty."
-            grub_version="trusty"
-        fi
-    fi
-
-    local grubnet_url="$MAIN_ARCHIVE/dists/$grub_version-updates/main/uefi/grub2-amd64/current/grubnetx64.efi.signed"
-    if ! $DOWNLOAD $grubnet_url
-    then
-        echo "$grub_version-updates not available, falling back to $grub_version"
-        grubnet_url="$MAIN_ARCHIVE/dists/$grub_version/main/uefi/grub2-amd64/current/grubnetx64.efi.signed"
-        if ! $DOWNLOAD $grubnet_url
-        then
-            echo "Failed to download $grubnet_url" >&2
-            return 1
-        fi
-    fi
-    mv "grubnetx64.efi.signed" "grubx64.efi"
-    maas-provision install-pxe-bootloader --loader="grubx64.efi"
-    maas-provision install-uefi-config
-    
-    trap - RETURN
-    popd >/dev/null
-    rm -rf -- "$download_dir"
-}
-
-
-# Download kernel/initrd for installing Ubuntu release $2 for
-# architecture $1.
-download_install_files() {
-    local arch=$1 release=$2
-    local files file url file_prefix filename_in_md5sums_file md5sum
-
-    files=$(compose_installer_download_files $arch $release)
-    url=$(compose_installer_download_url $arch $release)
-
-    mkdir "install"
-    pushd "install" >/dev/null
-    if ! fetch_server_md5sums $(compose_installer_base_url $arch $release)
-    then
-        echo "Failed to download MD5 sums for $arch $release." >&2
-        popd >/dev/null
-        rm -rf "install"
-        return 1
-    fi
-    echo "MD5SUMS GPG signature OK for $arch $release"
-    for file in $files
-    do
-        if ! $DOWNLOAD $url/$file
-        then
-            # Download failed.  Log error, and skip this image.
-            echo "Failed to download $url/$file" >&2
-            popd >/dev/null
-            rm -rf "install"
-            return 1
-        fi
-        file_prefix=$(compose_installer_download_url_postfix $arch)
-        filename_in_md5sums_file=./$file_prefix$file
-        md5sum=$(get_md5sum_for_file $filename_in_md5sums_file)
-        check_checksum $md5sum $file
-        echo "'$file' md5sum OK"
-    done
-    rename_installer_download_files $arch $release
-    popd >/dev/null
-    return 0
-}
-
-# Download kernel/initrd for installing Ubuntu release $2 for
-# architecture $1, and install them into the TFTP directory hierarchy.
-update_install_files() {
-    local arch=$1 release=$2
-
-    # Try the -updates pocket first, and fall back to release if it failed.
-    if ! download_install_files $arch ${release}-updates
-    then
-        echo "$release-updates not available, falling back to release"
-        if ! download_install_files $arch $release
-        then
-            return
-        fi
-    fi
-
-    maas-provision install-pxe-image \
-        --arch="${arch%%/*}" --subarch="${arch#*/}" \
-        --release=$release --purpose="install" \
-        --image="install"
-}
-
-
-# Download and install the "install" images.
-import_install_images() {
-    local arch release DOWNLOAD_DIR
-
-    DOWNLOAD_DIR=$(mktemp -d)
-    echo "Downloading to temporary location $DOWNLOAD_DIR."
-    pushd -- $DOWNLOAD_DIR
-
-    for arch in $ARCHES
-    do
-        for release in $RELEASES
-        do
-            update_install_files $arch $release
-        done
-    done
-
-    popd
-    rm -rf -- $DOWNLOAD_DIR
-}
-
-
-# Download and install the ephemeral images.
-import_ephemeral_images() {
-    if test "$IMPORT_EPHEMERALS" != "0"
-    then
-        maas-import-ephemerals
-    fi
-}
-
-
-main() {
-    # All files we create here are public.  The TFTP user will need to be
-    # able to read them.
-    umask a+r
-
-    update_pre_boot_loader
-    update_uefi_boot_loader
-    import_install_images
-    import_ephemeral_images
-}
-
-# check for commandline arguments
-if [ $# -gt 0 ]
-  then
-   case $1 in
-    "-h"|"--help") show_usage ; exit ;;
-    esac
-fi
-
-if [ ! -f "$GPG_KEYRING" ]; then
-    fail "gpg keyring $GPG_KEYRING is not a file"
-fi
-
-main
+
+"""Import boot resources into MAAS cluster controller."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+
+import sys
+from provisioningserver.import_images.boot_resources import (
+    main,
+    make_arg_parser,
+    NoConfigFile,
+    logger,
+    )
+
+if __name__ == "__main__":
+    parser = make_arg_parser(__doc__)
+    args = parser.parse_args()
+    try:
+        main(args)
+    except NoConfigFile:
+        logger.error("Config file %s not found." % args.config_file)
+        sys.exit(1)
+    except Exception:
+        # logger.exception() will log the error message, followed by the
+        # exception's stack trace. Using it here rather than allowing
+        # the exception to kill the process unhandled means we can be
+        # sure about where the exception ends up, rather than it
+        # potentially vanishing.
+        logger.exception("Unhandled exception; unable to continue.")
+        sys.exit(1)

=== modified file 'setup.py'
--- setup.py	2014-02-19 21:28:38 +0000
+++ setup.py	2014-03-25 17:45:57 +0000
@@ -65,6 +65,7 @@
     data_files=[
         ('/etc/maas',
             ['etc/maas/pserv.yaml',
+             'etc/maas/bootresources.yaml',
              'etc/maas_cluster.conf',
              'etc/txlongpoll.yaml',
              'contrib/maas_local_celeryconfig.py',

=== modified file 'src/maasserver/models/bootimage.py'
--- src/maasserver/models/bootimage.py	2014-03-18 10:33:34 +0000
+++ src/maasserver/models/bootimage.py	2014-03-25 17:45:57 +0000
@@ -154,7 +154,7 @@
     # Boot purpose (e.g. "commissioning" or "install") that the image is for.
     purpose = CharField(max_length=255, blank=False, editable=False)
 
-    # "Label" as in simplestreams parlance. (e.g. "release", "beta1")
+    # "Label" as in simplestreams parlance. (e.g. "release", "beta-1")
     label = CharField(
         max_length=255, blank=False, editable=False, default="release")
 

=== modified file 'src/maasserver/preseed.py'
--- src/maasserver/preseed.py	2014-01-28 02:37:29 +0000
+++ src/maasserver/preseed.py	2014-03-25 17:45:57 +0000
@@ -39,7 +39,9 @@
     PRESEED_TYPE,
     USERDATA_TYPE,
     )
+from maasserver.exceptions import MAASAPIException
 from maasserver.models import (
+    BootImage,
     Config,
     DHCPLease,
     )
@@ -94,10 +96,29 @@
     cluster_host = pick_cluster_controller_address(node)
     # XXX rvb(?): The path shouldn't be hardcoded like this, but rather synced
     # somehow with the content of contrib/maas-cluster-http.conf.
+    arch, subarch = node.architecture.split('/')
+    purpose = 'xinstall'
+    image = BootImage.objects.get_latest_image(
+        node.nodegroup, arch, subarch, series, purpose)
+    if image is None:
+        raise MAASAPIException(
+            "Error generating the URL of curtin's root-tgz file.  "
+            "No image could be found for the given selection: "
+            "arch=%s, subarch=%s, series=%s, purpose=%s." % (
+                arch,
+                subarch,
+                series,
+                purpose
+            ))
+    dyn_uri = '/'.join([
+        arch,
+        subarch,
+        series,
+        image.label,
+        'root-tgz'
+        ])
     return (
-        "http://" + cluster_host + "/MAAS/static/images/" +
-        node.architecture + "/" + series +
-        "/xinstall/root.tar.gz")
+        "http://" + cluster_host + "/MAAS/static/images/" + dyn_uri)
 
 
 def get_curtin_config(node):

=== modified file 'src/maasserver/tests/test_preseed.py'
--- src/maasserver/tests/test_preseed.py	2014-03-06 05:57:48 +0000
+++ src/maasserver/tests/test_preseed.py	2014-03-25 17:45:57 +0000
@@ -27,6 +27,7 @@
     NODEGROUPINTERFACE_MANAGEMENT,
     PRESEED_TYPE,
     )
+from maasserver.exceptions import MAASAPIException
 from maasserver.models import Config
 from maasserver.preseed import (
     compose_enlistment_preseed_url,
@@ -574,6 +575,11 @@
 
     def test_get_curtin_userdata(self):
         node = factory.make_node()
+        arch, subarch = node.architecture.split('/')
+        factory.make_boot_image(
+            architecture=arch, subarchitecture=subarch,
+            release=node.get_distro_series(), purpose='xinstall',
+            nodegroup=node.nodegroup)
         node.use_fastpath_installer()
         user_data = get_curtin_userdata(node)
         # Just check that the user data looks good.
@@ -603,20 +609,51 @@
         self.assertItemsEqual(['curtin_preseed'], context)
         self.assertIn('cloud-init', context['curtin_preseed'])
 
-    def test_get_curtin_installer_url(self):
+    def test_get_curtin_installer_url_returns_url(self):
         # Exclude DISTRO_SERIES.default. It's a special value that defers
         # to a run-time setting which we don't provide in this test.
         series = factory.getRandomEnum(
             DISTRO_SERIES, but_not=DISTRO_SERIES.default)
-        arch = make_usable_architecture(self)
-        node = factory.make_node(architecture=arch, distro_series=series)
+        architecture = make_usable_architecture(self)
+        node = factory.make_node(
+            architecture=architecture, distro_series=series)
+        arch, subarch = architecture.split('/')
+        boot_image = factory.make_boot_image(
+            architecture=arch, subarchitecture=subarch, release=series,
+            purpose='xinstall', nodegroup=node.nodegroup)
+
         installer_url = get_curtin_installer_url(node)
+
         [interface] = node.nodegroup.get_managed_interfaces()
         self.assertEqual(
-            'http://%s/MAAS/static/images/%s/%s/xinstall/root.tar.gz' % (
-                interface.ip, arch, series),
+            'http://%s/MAAS/static/images/%s/%s/%s/%s/root-tgz' % (
+                interface.ip, arch, subarch, series, boot_image.label),
             installer_url)
 
+    def test_get_curtin_installer_url_fails_if_no_boot_image(self):
+        series = factory.getRandomEnum(
+            DISTRO_SERIES, but_not=DISTRO_SERIES.default)
+        architecture = make_usable_architecture(self)
+        node = factory.make_node(
+            architecture=architecture, distro_series=series)
+        # Generate a boot image with a different arch/subarch.
+        factory.make_boot_image(
+            architecture=factory.make_name('arch'),
+            subarchitecture=factory.make_name('subarch'), release=series,
+            purpose='xinstall', nodegroup=node.nodegroup)
+
+        error = self.assertRaises(
+            MAASAPIException, get_curtin_installer_url, node)
+        arch, subarch = architecture.split('/')
+        msg = (
+            "No image could be found for the given selection: "
+            "arch=%s, subarch=%s, series=%s, purpose=xinstall." % (
+                arch,
+                subarch,
+                node.get_distro_series(),
+            ))
+        self.assertIn(msg, "%s" % error)
+
     def test_get_preseed_type_for(self):
         normal = factory.make_node()
         normal.use_traditional_installer()

=== modified file 'src/metadataserver/tests/test_api.py'
--- src/metadataserver/tests/test_api.py	2014-03-05 02:08:23 +0000
+++ src/metadataserver/tests/test_api.py	2014-03-25 17:45:57 +0000
@@ -39,9 +39,9 @@
     Tag,
     )
 from maasserver.testing import reload_object
-from maasserver.testing.testcase import MAASServerTestCase
 from maasserver.testing.factory import factory
 from maasserver.testing.oauthclient import OAuthAuthenticatedClient
+from maasserver.testing.testcase import MAASServerTestCase
 from maastesting.djangotestcase import DjangoTestCase
 from maastesting.matchers import MockCalledOnceWith
 from maastesting.utils import sample_binary_data
@@ -392,6 +392,11 @@
 
     def test_curtin_user_data_view_returns_curtin_data(self):
         node = factory.make_node()
+        arch, subarch = node.architecture.split('/')
+        factory.make_boot_image(
+            architecture=arch, subarchitecture=subarch,
+            release=node.get_distro_series(), purpose='xinstall',
+            nodegroup=node.nodegroup)
         client = make_node_client(node)
         response = client.get(
             reverse('curtin-metadata-user-data', args=['latest']))

=== modified file 'src/provisioningserver/boot_images.py'
--- src/provisioningserver/boot_images.py	2014-03-19 13:54:20 +0000
+++ src/provisioningserver/boot_images.py	2014-03-25 17:45:57 +0000
@@ -1,11 +1,7 @@
-# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2014 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-"""Dealing with boot images.
-
-Most of the lower-level logic is in the `tftppath` module, because it must
-correspond closely to the structure of the TFTP filesystem hierarchy.
-"""
+"""Dealing with boot images."""
 
 from __future__ import (
     absolute_import,
@@ -69,6 +65,6 @@
         return
 
     images = tftppath.list_boot_images(
-        Config.load_from_cache()['tftp']['root'])
+        Config.load_from_cache()['boot']['storage'] + '/current/')
 
     submit(maas_url, api_credentials, images)

=== modified file 'src/provisioningserver/config.py'
--- src/provisioningserver/config.py	2014-03-21 08:19:20 +0000
+++ src/provisioningserver/config.py	2014-03-25 17:45:57 +0000
@@ -70,6 +70,7 @@
     )
 from formencode.declarative import DeclarativeMeta
 from formencode.validators import (
+    Bool,
     Int,
     RequireIfPresent,
     Set,
@@ -109,7 +110,7 @@
 
     if_key_missing = None
 
-    root = String(if_missing="/var/lib/maas/tftp")
+    root = String(if_missing="/var/lib/maas/boot-resources/current/")
     port = Int(min=1, max=65535, if_missing=69)
     generator = String(if_missing=b"http://localhost/MAAS/api/1.0/pxeconfig/")
 
@@ -131,20 +132,16 @@
     releases = Set(if_missing=None)
 
 
-# XXX jtv 2014-03-21, bug=1295479: Unused until we start using the new import
-# script.
 class ConfigBootSourceSelection(Schema):
     """Configuration validator for boot source election onfiguration."""
 
     if_key_missing = None
 
     release = String(if_missing="*")
-    arch = String(if_missing="*")
+    arches = Set(if_missing=["*"])
     subarches = Set(if_missing=['*'])
 
 
-# XXX jtv 2014-03-21, bug=1295479: Unused until we start using the new import
-# script.
 class ConfigBootSource(Schema):
     """Configuration validator for boot source configuration."""
 
@@ -152,6 +149,8 @@
 
     path = String(
         if_missing="http://maas.ubuntu.com/images/ephemeral/releases/")
+    keyring = String(
+        if_missing="/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg")
     selections = ForEach(
         ConfigBootSourceSelection,
         if_missing=[ConfigBootSourceSelection.to_python({})])
@@ -167,12 +166,15 @@
     ephemeral = ConfigBootEphemeral
     architectures = Set(if_missing=None)
 
-    # XXX jtv 2014-03-21, bug=1295479: Unused until we start using the new
-    # import script.
     storage = String(if_missing="/var/lib/maas/boot-resources/")
     sources = ForEach(
         ConfigBootSource, if_missing=[ConfigBootSource.to_python({})])
 
+    # Marker in the bootresources.yaml file: if True, the file has not been
+    # edited yet and needs to be either configured with initial choices, or
+    # rewritten based on previously downloaded boot images.
+    configure_me = Bool(if_missing=False)
+
 
 class ConfigMeta(DeclarativeMeta):
     """Metaclass for the root configuration schema."""

=== modified file 'src/provisioningserver/driver/__init__.py'
--- src/provisioningserver/driver/__init__.py	2014-03-24 08:06:24 +0000
+++ src/provisioningserver/driver/__init__.py	2014-03-25 17:45:57 +0000
@@ -71,7 +71,7 @@
         :param at_location: URL to a Simplestreams index or a local path
             to a directory containing boot resources.
         :param filter: A simplestreams filter.
-            e.g. "release=trusty label=beta2 arch=amd64"
+            e.g. "release=trusty label=beta-2 arch=amd64"
             This is ignored if the location is a local path, all resources
             at the location will be imported.
         TBD: How to provide progress information.
@@ -90,7 +90,7 @@
             {
                 "release": "trusty",
                 "arch": "amd64",
-                "label": "beta2",
+                "label": "beta-2",
                 "size": 12344556,
             }
             ,

=== modified file 'src/provisioningserver/import_images/boot_resources.py'
--- src/provisioningserver/import_images/boot_resources.py	2014-03-17 18:42:45 +0000
+++ src/provisioningserver/import_images/boot_resources.py	2014-03-25 17:45:57 +0000
@@ -1,4 +1,4 @@
-# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# Copyright 2013-2014 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 from __future__ import (
@@ -13,20 +13,30 @@
 __all__ = [
     'main',
     'available_boot_resources',
+    'make_arg_parser',
     ]
 
+from argparse import ArgumentParser
 from collections import defaultdict
 from datetime import datetime
-import errno
+import functools
 import glob
 from gzip import GzipFile
-from json import dumps as jsondumps
+import json
+import logging
+from logging import getLogger
 import os
 from textwrap import dedent
 
 from provisioningserver.config import Config
 from provisioningserver.pxe.install_bootloader import install_bootloader
-from provisioningserver.utils import call_and_check
+from provisioningserver.pxe.tftppath import list_boot_images
+from provisioningserver.utils import (
+    atomic_write,
+    call_and_check,
+    locate_config,
+    read_text_file,
+    )
 from simplestreams.contentsource import FdContentSource
 from simplestreams.mirrors import (
     BasicMirrorWriter,
@@ -35,49 +45,158 @@
 from simplestreams.objectstores import FileStore
 from simplestreams.util import (
     item_checksums,
+    path_from_mirror_url,
+    policy_read_signed,
     products_exdata,
     )
 
 
+def init_logger():
+    logger = getLogger(__name__)
+    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+    return logger
+import errno
+
+
+logger = init_logger()
+
+
+class NoConfigFile(Exception):
+    """Raised when the config file for the script doesn't exist."""
+
+
 def create_empty_hierarchy():
+    """Create hierarchy of dicts which supports h[key1]...[keyN] accesses.
+
+    Generated object automatically creates nonexistent levels of hierarchy
+    when accessed the following way: h[arch][subarch][release]=something.
+
+    :return Generated hierarchy of dicts.
+    """
     return defaultdict(create_empty_hierarchy)
 
 
 def boot_walk(boot, func):
+    """Walk over multi-level depth dict and call callback func for every leaf.
+
+    Function walks over three level depth dictionary organized in a form of
+    d[arch][subarch][release]=value and passes control to a callback function
+    for each arch/subarch/release triplet available. Stored value is passed
+    to a callback function as an additional parameter.
+
+    :param boot: Hierarchy of dicts with a depth equals to three.
+    :param func: Callback function f(arch, subarch, release, value).
+    """
     for arch in boot:
         for subarch in boot[arch]:
             for release in boot[arch][subarch]:
-                func(arch, subarch, release, boot[arch][subarch][release])
+                for label in boot[arch][subarch][release]:
+                    func(
+                        arch, subarch, release, label,
+                        boot[arch][subarch][release][label])
+
+
+def value_passes_filter_list(filter_list, property_value):
+    """Does the given property of a boot image pass the given filter list?
+
+    The value passes if either it matches one of the entries in the list of
+    filter values, or one of the filter values is an asterisk (`*`).
+    """
+    return '*' in filter_list or property_value in filter_list
+
+
+def value_passes_filter(filter_value, property_value):
+    """Does the given property of a boot image pass the given filter?
+
+    The value passes the filter if either the filter value is an asterisk
+    (`*`) or the value is equal to the filter value.
+    """
+    return filter_value in ('*', property_value)
+
+
+def image_passes_filter(filters, arch, subarch, release):
+    """Filter a boot image against configured import filters.
+
+    :param filters: A list of dicts describing the filters, as in `boot_merge`.
+        If the list is empty, or `None`, any image matches.  Any entry in a
+        filter may be a string containing just an asterisk (`*`) to denote that
+        the entry will match any value.
+    :param arch: The given boot image's architecture.
+    :param subarch: The given boot image's subarchitecture.
+    :param release: The given boot image's OS release.
+    :return: Whether the image matches any of the dicts in `filters`.
+    """
+    # XXX jtv 2014-03-24: add label parameter?
+    if filters is None or len(filters) == 0:
+        return True
+    for filter_dict in filters:
+        item_matches = (
+            value_passes_filter(filter_dict['release'], release) and
+            value_passes_filter_list(filter_dict['arches'], arch) and
+            value_passes_filter_list(filter_dict['subarches'], subarch)
+        )
+        if item_matches:
+            return True
+    return False
 
 
 def boot_merge(boot1, boot2, filters=None):
-
-    def filter_func(arch, subarch, release):
-        for filter in filters:
-            item_matches = (
-                filter['release'] in ('*', release) and
-                filter['arch'] in ('*', arch) and
-                (
-                    '*' in filter['subarches'] or
-                    subarch in filter['subarches']
-                )
-            )
-            if item_matches:
-                return True
-        return False
-
-    def merge_func(arch, subarch, release, boot_resource):
-        if filters and not filter_func(arch, subarch, release):
-            return
-        boot1[arch][subarch][release] = boot_resource
+    """Add entries from the second multi-level dict to the first one.
+
+    Function copies d[arch][subarch][release]=value chains from the second
+    dictionary to the first one if they don't exist there and pass optional
+    check done by filters.
+
+    :param boot1: first dict which will be extended in-place.
+    :param boot2: second dict which will be used as a source of new entries.
+    :param filters: list of dicts each of which contains 'arch', 'subarch',
+        'release' keys; function takes d[arch][subarch][release] chain to the
+        first dict only if filters contain at least one dict with
+        arch in d['arches'], subarch in d['subarch'], d['release'] == release;
+        dict may have '*' as a value for 'arch' and 'release' keys and as a
+        member of 'subarch' list -- in that case key-specific check always
+        passes.
+    """
+    def merge_func(arch, subarch, release, label, boot_resource):
+        """Merge a boot resource into `boot1`, if it passes filters."""
+        if image_passes_filter(filters, arch, subarch, release):
+            logger.debug(
+                "Merging boot resource for %s/%s/%s/%s.",
+                arch, subarch, release, label)
+            boot1[arch][subarch][release][label] = boot_resource
 
     boot_walk(boot2, merge_func)
 
 
 def boot_reverse(boot):
+    """Determine a set of subarches which should be deployed by boot resource.
+
+    Function reverses h[arch][subarch][release]=boot_resource hierarchy to form
+    boot resource to subarch relation. Many subarches may be deployed by a
+    single boot resource (in which case boot_resource=[subarch1, subarch2]
+    relation will be created). We note only subarchitectures and ignore
+    architectures because boot resource is tightly coupled with architecture
+    it can deploy according to metadata format. We can figure out for which
+    architecture we need to use a specific boot resource by looking at its
+    description in metadata. We can't do the same with subarch because we may
+    want to use boot resource only for a specific subset of subarches it can be
+    used for. To represent boot resource to subarch relation we generate the
+    following multi-level dictionary: d[content_id][product_name]=[subarches]
+    where 'content_id' and 'product_name' values come from metadata information
+    and allow us to uniquely identify a specific boot resource.
+
+    :param boot: Hierarchy of dicts d[arch][subarch][release]=boot_resource
+    :return Hierarchy of dictionaries d[content_id][product_name]=[subarches]
+        which describes boot resource to subarches relation for all available
+        boot resources (products).
+    """
     reverse = create_empty_hierarchy()
 
-    def reverse_func(arch, subarch, release, boot_resource):
+    def reverse_func(arch, subarch, release, label, boot_resource):
         content_id = boot_resource['content_id']
         product_name = boot_resource['product_name']
         existent = list(reverse[content_id][product_name])
@@ -87,9 +206,29 @@
     return reverse
 
 
-def tgt_entry(arch, subarch, release, image):
+def tgt_entry(arch, subarch, release, label, image):
+    """Generate tgt target used to commission arch/subarch with release
+
+    Tgt target used to commission arch/subarch machine with a specific Ubuntu
+    release should have the following name: ephemeral-arch-subarch-release.
+    This function creates target description in a format used by tgt-admin.
+    It uses arch, subarch and release to generate target name and image as
+    a path to image file which should be shared. Tgt target is marked as
+    read-only. Tgt target has 'allow-in-use' option enabled because this
+    script actively uses hardlinks to do image management and root images
+    in different folders may point to the same inode. Tgt doesn't allow us to
+    use the same inode for different tgt targets (even read-only targets which
+    looks like a bug to me) without this option enabled.
+
+    :param arch: Architecture name we generate tgt target for
+    :param subarch: Subarchitecture name we generate tgt target for
+    :param release: Ubuntu release we generate tgt target for
+    :param label: The images' label
+    :param image: Path to the image which should be shared via tgt/iscsi
+    :return Tgt entry which can be written to tgt-admin configuration file
+    """
     prefix = 'iqn.2004-05.com.ubuntu:maas'
-    target_name = 'ephemeral-%s-%s-%s' % (arch, subarch, release)
+    target_name = 'ephemeral-%s-%s-%s-%s' % (arch, subarch, release, label)
     entry = dedent("""\
     <target {prefix}:{target_name}>
         readonly 1
@@ -101,15 +240,28 @@
     return entry
 
 
+def mirror_info_for_path(path, unsigned_policy=None, keyring=None):
+    if unsigned_policy is None:
+        unsigned_policy = lambda content, path, keyring: content
+    (mirror, rpath) = path_from_mirror_url(path, None)
+    policy = policy_read_signed
+    if rpath.endswith(".json"):
+        policy = unsigned_policy
+    if keyring:
+        policy = functools.partial(policy, keyring=keyring)
+    return(mirror, rpath, policy)
+
+
 class RepoDumper(BasicMirrorWriter):
 
     def __init__(self):
         super(RepoDumper, self).__init__({'max_items': 1})
 
-    def dump(self, path):
+    def dump(self, path, keyring=None):
         self._boot = create_empty_hierarchy()
-        reader = UrlMirrorReader(path)
-        super(RepoDumper, self).sync(reader, 'streams/v1/index.sjson')
+        (mirror, rpath, policy) = mirror_info_for_path(path, keyring=keyring)
+        reader = UrlMirrorReader(mirror, policy=policy)
+        super(RepoDumper, self).sync(reader, rpath)
         return self._boot
 
     def load_products(self, path=None, content_id=None):
@@ -124,9 +276,10 @@
         item = products_exdata(src, pedigree)
         arch, subarches = item['arch'], item['subarches']
         release = item['release']
+        label = item['label']
         compact_item = self.item_cleanup(item)
         for subarch in subarches.split(','):
-            self._boot[arch][subarch][release] = compact_item
+            self._boot[arch][subarch][release][label] = compact_item
 
 
 class RepoWriter(BasicMirrorWriter):
@@ -137,9 +290,10 @@
         self._cache = FileStore(os.path.abspath(cache_path))
         super(RepoWriter, self).__init__({'max_items': 1})
 
-    def write(self, path):
-        reader = UrlMirrorReader(path)
-        super(RepoWriter, self).sync(reader, 'streams/v1/index.sjson')
+    def write(self, path, keyring=None):
+        (mirror, rpath, policy) = mirror_info_for_path(path, keyring=keyring)
+        reader = UrlMirrorReader(mirror, policy=policy)
+        super(RepoWriter, self).sync(reader, rpath)
 
     def load_products(self, path=None, content_id=None):
         return
@@ -152,46 +306,58 @@
             product_name in self._info[content_id]
         )
 
-    def insert_uncompressed(self, tag, checksums, size, contentsource):
+    def insert_file(self, name, tag, checksums, size, contentsource):
+        logger.info("Inserting file %s (tag=%s, size=%s).", name, tag, size)
         self._cache.insert(
             tag, contentsource, checksums, mutable=False, size=size)
-        return self._cache._fullpath(tag)
+        return [(self._cache._fullpath(tag), name)]
 
-    def insert_compressed(self, tag, checksums, size, contentsource):
-        # TODO: Bake root.tar.gz required by fast-path installer (uec2roottar)
-        uncompressed_tag = 'uncompressed-%s' % tag
-        compressed_path = self._cache._fullpath(tag)
-        uncompressed_path = self._cache._fullpath(uncompressed_tag)
-        if not os.path.isfile(uncompressed_path):
-            self._cache.insert(tag, contentsource, checksums,
-                               mutable=False, size=size)
-            compressed_source = FdContentSource(GzipFile(compressed_path))
-            self._cache.insert(uncompressed_tag, compressed_source,
-                               mutable=False)
+    def insert_root_image(self, tag, checksums, size, contentsource):
+        root_image_tag = 'root-image-%s' % tag
+        root_image_path = self._cache._fullpath(root_image_tag)
+        root_tgz_tag = 'root-tgz-%s' % tag
+        root_tgz_path = self._cache._fullpath(root_tgz_tag)
+        if not os.path.isfile(root_image_path):
+            logger.info("New root image: %s.", root_image_path)
+            self._cache.insert(
+                tag, contentsource, checksums, mutable=False, size=size)
+            uncompressed = FdContentSource(
+                GzipFile(self._cache._fullpath(tag)))
+            self._cache.insert(root_image_tag, uncompressed, mutable=False)
             self._cache.remove(tag)
-        return uncompressed_path
+        if not os.path.isfile(root_tgz_path):
+            logger.info("Converting root tarball: %s.", root_tgz_path)
+            call_uec2roottar(root_image_path, root_tgz_path)
+        return [(root_image_path, 'root-image'), (root_tgz_path, 'root-tgz')]
 
     def insert_item(self, data, src, target, pedigree, contentsource):
         item = products_exdata(src, pedigree)
         checksums = item_checksums(data)
-        tag = checksums['md5']
+        tag = checksums['sha256']
         size = data['size']
-        if data['path'].endswith('.gz'):
-            src = self.insert_compressed(tag, checksums, size, contentsource)
+        ftype = item['ftype']
+        if ftype == 'root-image.gz':
+            links = self.insert_root_image(tag, checksums, size, contentsource)
         else:
-            src = self.insert_uncompressed(tag, checksums, size, contentsource)
+            links = self.insert_file(
+                ftype, tag, checksums, size, contentsource)
         for subarch in self._info[item['content_id']][item['product_name']]:
             dst_folder = os.path.join(
-                self._root_path, item['arch'], subarch, item['release'])
+                self._root_path, item['arch'], subarch, item['release'],
+                item['label'])
             if not os.path.exists(dst_folder):
                 os.makedirs(dst_folder)
-            os.link(src, os.path.join(dst_folder, item['ftype']))
+            for src, link_name in links:
+                link_path = os.path.join(dst_folder, link_name)
+                if os.path.isfile(link_path):
+                    os.remove(link_path)
+                os.link(src, link_path)
 
 
 def available_boot_resources(root):
-    for resource_path in glob.glob(os.path.join(root, '*/*/*')):
-        arch, subarch, release = resource_path.split('/')[-3:]
-        yield (arch, subarch, release)
+    for resource_path in glob.glob(os.path.join(root, '*/*/*/*')):
+        arch, subarch, release, label = resource_path.split('/')[-4:]
+        yield (arch, subarch, release, label)
 
 
 BOOTLOADERS = ['pxelinux.0', 'chain.c32', 'ifcpu64.c32']
@@ -212,14 +378,103 @@
         install_bootloader(bootloader_src, bootloader_dst)
 
 
-def main():
-
+def call_uec2roottar(*args):
+    """Invoke `uec2roottar` with the given arguments.
+
+    Here only so tests can stub it out.
+    """
+    call_and_check(["uec2roottar"] + list(args))
+
+
+def make_arg_parser(doc):
+    """Create an `argparse.ArgumentParser` for this script."""
+
+    parser = ArgumentParser(description=doc)
+    default_config = locate_config("bootresources.yaml")
+    parser.add_argument(
+        '--config-file', action="store", default=default_config,
+        help="Path to config file "
+             "(defaults to %s)" % default_config)
+    return parser
+
+
+def compose_targets_conf(snapshot_path):
+    """Produce the contents of a snapshot's tgt conf file.
+
+    :param snasphot_path: Filesystem path to a snapshot of boot images.
+    :return: Contents for a `targets.conf` file.
+    :rtype: bytes
+    """
+    # Use a set to make sure we don't register duplicate entries in tgt.
+    entries = set()
+    for item in list_boot_images(snapshot_path):
+        arch = item['architecture']
+        subarch = item['subarchitecture']
+        release = item['release']
+        label = item['label']
+        entries.add((arch, subarch, release, label))
+    tgt_entries = []
+    for arch, subarch, release, label in sorted(entries):
+        root_image = os.path.join(
+            snapshot_path, arch, subarch, release, label, 'root-image')
+        if os.path.isfile(root_image):
+            entry = tgt_entry(arch, subarch, release, label, root_image)
+            tgt_entries.append(entry)
+    text = ''.join(tgt_entries)
+    return text.encode('utf-8')
+
+
+def meta_contains(storage, content):
+    """Does the `maas.meta` file match `content`?
+
+    If the file's contents match the latest data, there is no need to update.
+    """
+    current_meta = os.path.join(storage, 'current', 'maas.meta')
+    return (
+        os.path.isfile(current_meta) and
+        content == read_text_file(current_meta)
+        )
+
+
+def compose_snapshot_path(storage):
+    """Put together a path for a new snapshot.
+
+    A snapshot is a directory in `storage` containing images.  The name
+    contains the date in a sortable format.
+    """
+    snapshot_name = 'snapshot-%s' % datetime.now().strftime('%Y%m%d-%H%M%S')
+    return os.path.join(storage, snapshot_name)
+
+
+def update_current_symlink(storage, latest_snapshot):
+    """Symlink `latest_snapshot` as the "current" snapshot."""
+    symlink_path = os.path.join(storage, 'current')
+    if os.path.lexists(symlink_path):
+        os.unlink(symlink_path)
+    os.symlink(latest_snapshot, symlink_path)
+
+
+def write_snapshot_metadata(snapshot, meta_file_content, targets_conf,
+                            targets_conf_content):
+    """Write "meta" file and tgt config for `snapshot`."""
+    meta_file = os.path.join(snapshot, 'maas.meta')
+    atomic_write(meta_file_content, meta_file, mode=0644)
+    atomic_write(targets_conf_content, targets_conf, mode=0644)
+
+
+def main(args):
+    logger.info("Importing boot resources.")
+    # The config file is required.  We do not fall back to defaults if it's
+    # not there.
     try:
-        config = Config.load_from_cache()
-    except IOError as e:
-        if e.errno != errno.ENOENT:
+        config = Config.load_from_cache(filename=args.config_file)
+    except IOError as ex:
+        if ex.errno == errno.ENOENT:
+            # No config file. We have helpful error output for this.
+            raise NoConfigFile(ex)
+        else:
+            # Unexpected error.
             raise
-        config = Config.get_defaults()
 
     storage = config['boot']['storage']
 
@@ -227,34 +482,34 @@
     dumper = RepoDumper()
 
     for source in reversed(config['boot']['sources']):
-        repo_boot = dumper.dump(source['path'])
+        repo_boot = dumper.dump(source['path'], keyring=source['keyring'])
         boot_merge(boot, repo_boot, source['selections'])
 
-    meta = jsondumps(boot)
-    current_meta = storage + '/current/maas.meta'
-    if os.path.isfile(current_meta) and meta == open(current_meta).read():
+    meta_file_content = json.dumps(boot, sort_keys=True)
+    if meta_contains(storage, meta_file_content):
+        # The current maas.meta already contains the new config.  No need to
+        # rewrite anything.
         return
 
-    snapshot_name = '/snapshot-%s/' % datetime.now().strftime('%d%m%Y-%H%M%S')
-    snapshot_path = storage + snapshot_name
     reverse_boot = boot_reverse(boot)
-    writer = RepoWriter(snapshot_path, storage + '/cache/', reverse_boot)
+    snapshot_path = compose_snapshot_path(storage)
+    cache_path = os.path.join(storage, 'cache')
+    targets_conf = os.path.join(snapshot_path, 'maas.tgt')
+    writer = RepoWriter(snapshot_path, cache_path, reverse_boot)
 
     for source in config['boot']['sources']:
-        writer.write(source['path'])
-
-    open(snapshot_path + '/maas.meta', 'w').write(meta)
-    symlink_path = storage + '/current'
-    if os.path.lexists(symlink_path):
-        os.unlink(symlink_path)
-    os.symlink(snapshot_path, symlink_path)
-
-    with open(snapshot_path + '/maas.tgt', 'w') as output:
-        for arch, subarch, release in available_boot_resources(snapshot_path):
-            disk = os.path.join(snapshot_path, arch, subarch, release, 'disk')
-            if os.path.isfile(disk):
-                output.write(tgt_entry(arch, subarch, release, disk))
-
-    call_and_check(['tgt-admin', '--update', 'ALL'])
-
+        writer.write(source['path'], source['keyring'])
+
+    targets_conf_content = compose_targets_conf(snapshot_path)
+
+    logger.info("Writing metadata and updating iSCSI targets.")
+    write_snapshot_metadata(
+        snapshot_path, meta_file_content, targets_conf, targets_conf_content)
+    call_and_check(['tgt-admin', '--conf', targets_conf, '--update', 'ALL'])
+
+    logger.info("Installing boot images snapshot %s.", snapshot_path)
     install_boot_loaders(snapshot_path)
+
+    # If we got here, all went well.  This is now truly the "current" snapshot.
+    update_current_symlink(storage, snapshot_path)
+    logger.info("Import done.")

=== added file 'src/provisioningserver/import_images/tests/test_boot_resources.py'
--- src/provisioningserver/import_images/tests/test_boot_resources.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/import_images/tests/test_boot_resources.py	2014-03-25 17:45:57 +0000
@@ -0,0 +1,54 @@
+# Copyright 2014 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for the boot_resources module."""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = []
+
+import errno
+import os
+from random import randint
+
+from maastesting.factory import factory
+from maastesting.testcase import MAASTestCase
+from mock import MagicMock
+from provisioningserver.config import Config
+from provisioningserver.import_images import boot_resources
+from provisioningserver.import_images.boot_resources import (
+    main,
+    NoConfigFile,
+    )
+
+
+class TestMain(MAASTestCase):
+
+    def test_raises_ioerror_when_no_config_file_found(self):
+        # Suppress log output.
+        self.logger = self.patch(boot_resources, 'logger')
+        filename = "/tmp/%s" % factory.make_name("config")
+        self.assertFalse(os.path.exists(filename))
+        args = MagicMock()
+        args.config_file = filename
+        self.assertRaises(NoConfigFile, main, args)
+
+    def test_raises_non_ENOENT_IOErrors(self):
+        # main() will raise a NoConfigFile error when it encounters an
+        # ENOENT IOError, but will otherwise just re-raise the original
+        # IOError.
+        args = MagicMock()
+        mock_load_from_cache = self.patch(Config, 'load_from_cache')
+        other_error = IOError(randint(errno.ENOENT + 1, 1000))
+        mock_load_from_cache.side_effect = other_error
+        # Suppress log output.
+        self.logger = self.patch(boot_resources, 'logger')
+        raised_error = self.assertRaises(IOError, main, args)
+        self.assertEqual(other_error, raised_error)

=== modified file 'src/provisioningserver/import_images/tests/test_ephemerals_script.py'
--- src/provisioningserver/import_images/tests/test_ephemerals_script.py	2014-03-13 05:05:21 +0000
+++ src/provisioningserver/import_images/tests/test_ephemerals_script.py	2014-03-25 17:45:57 +0000
@@ -14,22 +14,15 @@
 __metaclass__ = type
 __all__ = []
 
-from argparse import ArgumentParser
-from copy import deepcopy
 from os import (
     listdir,
     readlink,
     )
 import os.path
-from pipes import quote
 import subprocess
-from textwrap import dedent
 
-from fixtures import EnvironmentVariableFixture
 from maastesting.factory import factory
-from provisioningserver.config import Config
 from provisioningserver.import_images import (
-    config as config_module,
     ephemerals_script,
     )
 from provisioningserver.import_images.ephemerals_script import (
@@ -37,7 +30,6 @@
     create_symlinked_image_dir,
     extract_image_tarball,
     install_image_from_simplestreams,
-    make_arg_parser,
     move_file_by_glob,
     )
 from provisioningserver.pxe.tftppath import (
@@ -369,112 +361,3 @@
             temp_location=temp_location)
 
         self.assertItemsEqual([], listdir(temp_location))
-
-
-def make_legacy_config(data_dir=None, arches=None, releases=None):
-    """Create contents for a legacy, shell-script config file."""
-    if data_dir is None:
-        data_dir = factory.make_name('datadir')
-    if arches is None:
-        arches = [factory.make_name('arch') for counter in range(2)]
-    if releases is None:
-        releases = [factory.make_name('release') for counter in range(2)]
-    return dedent("""\
-    DATA_DIR=%s
-    ARCHES=%s
-    RELEASES=%s
-    """) % (
-        quote(data_dir),
-        quote(' '.join(arches)),
-        quote(' '.join(releases)),
-    )
-
-
-def install_legacy_config(testcase, contents):
-    """Set up a legacy config file with the given contents.
-
-    Returns the config file's path.
-    """
-    legacy_file = testcase.make_file(contents=contents)
-    testcase.patch(config_module, 'EPHEMERALS_LEGACY_CONFIG', legacy_file)
-    return legacy_file
-
-
-class TestMakeArgParser(PservTestCase):
-
-    def test_creates_parser(self):
-        self.useFixture(ConfigFixture({'boot': {'ephemeral': {}}}))
-        documentation = factory.getRandomString()
-
-        parser = make_arg_parser(documentation)
-
-        self.assertIsInstance(parser, ArgumentParser)
-        self.assertEqual(documentation, parser.description)
-
-    def test_defaults_to_config(self):
-        images_directory = self.make_dir()
-        arches = [factory.make_name('arch1'), factory.make_name('arch2')]
-        releases = [factory.make_name('rel1'), factory.make_name('rel2')]
-        self.useFixture(ConfigFixture({
-            'boot': {
-                'architectures': arches,
-                'ephemeral': {
-                    'images_directory': images_directory,
-                    'releases': releases,
-                },
-            },
-        }))
-
-        parser = make_arg_parser(factory.getRandomString())
-
-        args = parser.parse_args('')
-        self.assertEqual(images_directory, args.output)
-        self.assertItemsEqual(
-            [
-                compose_filter('arch', arches),
-                compose_filter('release', releases),
-            ],
-            args.filters)
-
-    def test_does_not_require_config(self):
-        defaults = Config.get_defaults()
-        no_file = os.path.join(self.make_dir(), factory.make_name() + '.yaml')
-        self.useFixture(
-            EnvironmentVariableFixture('MAAS_PROVISIONING_SETTINGS', no_file))
-
-        parser = make_arg_parser(factory.getRandomString())
-
-        args = parser.parse_args('')
-        self.assertEqual(
-            defaults['boot']['ephemeral']['images_directory'],
-            args.output)
-        self.assertItemsEqual([], args.filters)
-
-    def test_does_not_modify_config(self):
-        self.useFixture(ConfigFixture({
-            'boot': {
-                'architectures': [factory.make_name('arch')],
-                'ephemeral': {
-                    'images_directory': self.make_dir(),
-                    'releases': [factory.make_name('release')],
-                },
-            },
-        }))
-        original_boot_config = deepcopy(Config.load_from_cache()['boot'])
-        install_legacy_config(self, make_legacy_config())
-
-        make_arg_parser(factory.getRandomString())
-
-        self.assertEqual(
-            original_boot_config,
-            Config.load_from_cache()['boot'])
-
-    def test_uses_legacy_config(self):
-        data_dir = self.make_dir()
-        self.useFixture(ConfigFixture({}))
-        install_legacy_config(self, make_legacy_config(data_dir=data_dir))
-
-        parser = make_arg_parser(factory.getRandomString())
-
-        args = parser.parse_args('')
-        self.assertEqual(data_dir, args.output)

=== modified file 'src/provisioningserver/kernel_opts.py'
--- src/provisioningserver/kernel_opts.py	2014-03-18 03:23:55 +0000
+++ src/provisioningserver/kernel_opts.py	2014-03-25 17:45:57 +0000
@@ -21,9 +21,7 @@
 from collections import namedtuple
 import os
 
-from provisioningserver.config import Config
 from provisioningserver.driver import ArchitectureRegistry
-from provisioningserver.utils import parse_key_value_file
 
 
 class EphemeralImagesDirectoryNotFound(Exception):
@@ -92,26 +90,9 @@
 ISCSI_TARGET_NAME_PREFIX = "iqn.2004-05.com.ubuntu:maas"
 
 
-def get_ephemeral_name(release, arch):
-    """Return the name of the most recent ephemeral image.
-
-    That information is read from the config file named 'info' in the
-    ephemeral directory e.g:
-    /var/lib/maas/ephemeral/precise/ephemeral/i386/20120424/info
-    """
-    config = Config.load_from_cache()
-    root = os.path.join(
-        config["boot"]["ephemeral"]["images_directory"],
-        release, 'ephemeral', arch)
-    try:
-        filename = os.path.join(get_last_directory(root), 'info')
-    except OSError:
-        raise EphemeralImagesDirectoryNotFound(
-            "The directory containing the ephemeral images/info is missing "
-            "(%r).  Make sure to run the script "
-            "'maas-import-pxe-files'." % root)
-    name = parse_key_value_file(filename, separator="=")['name']
-    return name
+def get_ephemeral_name(arch, subarch, release, label):
+    """Return the name of the most recent ephemeral image."""
+    return "ephemeral-%s-%s-%s-%s" % (arch, subarch, release, label)
 
 
 def compose_hostname_opts(params):
@@ -137,7 +118,8 @@
     if params.purpose == "commissioning" or params.purpose == "xinstall":
         # These are kernel parameters read by the ephemeral environment.
         tname = prefix_target_name(
-            get_ephemeral_name(params.release, params.arch))
+            get_ephemeral_name(
+                params.arch, params.subarch, params.release, params.label))
         kernel_params = [
             # Read by the open-iscsi initramfs code.
             "iscsi_target_name=%s" % tname,

=== modified file 'src/provisioningserver/pxe/config.py'
--- src/provisioningserver/pxe/config.py	2014-03-13 05:05:21 +0000
+++ src/provisioningserver/pxe/config.py	2014-03-25 17:45:57 +0000
@@ -95,19 +95,22 @@
         kernel_params.purpose, kernel_params.arch,
         kernel_params.subarch)
 
-    # The locations of the kernel image and the initrd are defined by
-    # update_install_files(), in scripts/maas-import-pxe-files.
-
     def image_dir(params):
         return compose_image_path(
             params.arch, params.subarch,
-            params.release, params.label, params.purpose)
+            params.release, params.label)
 
     def initrd_path(params):
-        return "%s/initrd.gz" % image_dir(params)
+        if params.purpose == "install":
+            return "%s/di-initrd" % image_dir(params)
+        else:
+            return "%s/boot-initrd" % image_dir(params)
 
     def kernel_path(params):
-        return "%s/linux" % image_dir(params)
+        if params.purpose == "install":
+            return "%s/di-kernel" % image_dir(params)
+        else:
+            return "%s/boot-kernel" % image_dir(params)
 
     def kernel_command(params):
         return compose_kernel_command_line(params)

=== modified file 'src/provisioningserver/pxe/tests/test_config.py'
--- src/provisioningserver/pxe/tests/test_config.py	2014-03-17 08:18:55 +0000
+++ src/provisioningserver/pxe/tests/test_config.py	2014-03-25 17:45:57 +0000
@@ -161,7 +161,7 @@
 class TestRenderPXEConfig(MAASTestCase):
     """Tests for `provisioningserver.pxe.config.render_pxe_config`."""
 
-    def test_render(self):
+    def test_render_install(self):
         # Given the right configuration options, the PXE configuration is
         # correctly rendered.
         params = make_kernel_parameters(self, purpose="install")
@@ -174,14 +174,14 @@
         # The PXE parameters are all set according to the options.
         image_dir = compose_image_path(
             arch=params.arch, subarch=params.subarch,
-            release=params.release, label=params.label, purpose=params.purpose)
+            release=params.release, label=params.label)
         self.assertThat(
             output, MatchesAll(
                 MatchesRegex(
-                    r'.*^\s+KERNEL %s/linux$' % re.escape(image_dir),
+                    r'.*^\s+KERNEL %s/di-kernel$' % re.escape(image_dir),
                     re.MULTILINE | re.DOTALL),
                 MatchesRegex(
-                    r'.*^\s+INITRD %s/initrd[.]gz$' % re.escape(image_dir),
+                    r'.*^\s+INITRD %s/di-initrd$' % re.escape(image_dir),
                     re.MULTILINE | re.DOTALL),
                 MatchesRegex(
                     r'.*^\s+APPEND .+?$',
@@ -235,45 +235,6 @@
         self.assertNotIn("LOCALBOOT", output)
 
 
-class TestRenderArmhfSubarchScenarios(MAASTestCase):
-    """See bug https://bugs.launchpad.net/maas/+bug/1166994"""
-
-    scenarios = [
-        ("install_precise", dict(
-            arch="armhf", purpose="install", release="precise",
-            expect_in_output="highbank")),
-        ("install_quantal", dict(
-            arch="armhf", purpose="install", release="quantal",
-            expect_in_output="highbank")),
-        ("install_saucy", dict(
-            arch="armhf", purpose="install", release="saucy",
-            expect_in_output="generic")),
-        ("commission_precise", dict(
-            arch="armhf", purpose="commissioning", release="precise",
-            expect_in_output="highbank")),
-        ("commission_quantal", dict(
-            arch="armhf", purpose="commissioning", release="quantal",
-            expect_in_output="highbank")),
-        ("commission_saucy", dict(
-            arch="armhf", purpose="commissioning", release="saucy",
-            expect_in_output="generic")),
-    ]
-
-    def test_highbank_scenarios(self):
-        # get_ephemeral_name depends on ephemeral images being present but
-        # doesn't affect the test outcome, so patch it out.
-        self.patch(
-            kernel_opts,
-            "get_ephemeral_name").return_value = "ephemeral_name"
-        options = {
-            "kernel_params": make_kernel_parameters(
-                arch=self.arch, purpose=self.purpose, subarch="highbank",
-                release=self.release),
-        }
-        output = render_pxe_config(**options)
-        self.assertIn(self.expect_in_output, output)
-
-
 class TestRenderPXEConfigScenarios(MAASTestCase):
     """Tests for `provisioningserver.pxe.config.render_pxe_config`."""
 

=== modified file 'src/provisioningserver/pxe/tests/test_tftppath.py'
--- src/provisioningserver/pxe/tests/test_tftppath.py	2014-03-18 03:23:55 +0000
+++ src/provisioningserver/pxe/tests/test_tftppath.py	2014-03-25 17:45:57 +0000
@@ -30,7 +30,9 @@
     list_subdirs,
     locate_tftp_path,
     )
-from provisioningserver.testing.boot_images import make_boot_image_params
+from provisioningserver.testing.boot_images import (
+    make_boot_image_storage_params,
+    )
 from provisioningserver.testing.config import ConfigFixture
 from testtools.matchers import (
     Not,
@@ -38,6 +40,16 @@
     )
 
 
+def make_image(params, purpose):
+    """Describe an image as a dict similar to what `list_boot_images` returns.
+
+    The `params` are as returned from `make_boot_image_storage_params`.
+    """
+    image = params.copy()
+    image['purpose'] = purpose
+    return image
+
+
 class TestTFTPPath(MAASTestCase):
 
     def setUp(self):
@@ -53,8 +65,7 @@
                 arch=image_params['architecture'],
                 subarch=image_params['subarchitecture'],
                 release=image_params['release'],
-                label=image_params['label'],
-                purpose=image_params['purpose']),
+                label=image_params['label']),
             tftproot)
         os.makedirs(image_dir)
         factory.make_file(image_dir, 'linux')
@@ -111,23 +122,32 @@
             self.tftproot, locate_tftp_path(None, tftproot=self.tftproot))
 
     def test_list_boot_images_copes_with_empty_directory(self):
-        self.assertItemsEqual([], list_boot_images(self.tftproot))
+        self.assertEqual([], list_boot_images(self.tftproot))
 
     def test_list_boot_images_copes_with_unexpected_files(self):
         os.makedirs(os.path.join(self.tftproot, factory.make_name('empty')))
         factory.make_file(self.tftproot)
-        self.assertItemsEqual([], list_boot_images(self.tftproot))
+        self.assertEqual([], list_boot_images(self.tftproot))
 
     def test_list_boot_images_finds_boot_image(self):
-        image = make_boot_image_params()
-        self.make_image_dir(image, self.tftproot)
-        self.assertItemsEqual([image], list_boot_images(self.tftproot))
+        params = make_boot_image_storage_params()
+        self.make_image_dir(params, self.tftproot)
+        purposes = ['install', 'commissioning', 'xinstall']
+        self.assertItemsEqual(
+            [make_image(params, purpose) for purpose in purposes],
+            list_boot_images(self.tftproot))
 
     def test_list_boot_images_enumerates_boot_images(self):
-        images = [make_boot_image_params() for counter in range(3)]
-        for image in images:
-            self.make_image_dir(image, self.tftproot)
-        self.assertItemsEqual(images, list_boot_images(self.tftproot))
+        params = [make_boot_image_storage_params() for counter in range(3)]
+        for param in params:
+            self.make_image_dir(param, self.tftproot)
+        self.assertItemsEqual(
+            [
+                make_image(param, purpose)
+                for param in params
+                for purpose in ['install', 'commissioning', 'xinstall']
+            ],
+            list_boot_images(self.tftproot))
 
     def test_is_visible_subdir_ignores_regular_files(self):
         plain_file = self.make_file()

=== modified file 'src/provisioningserver/pxe/tftppath.py'
--- src/provisioningserver/pxe/tftppath.py	2014-03-18 20:22:07 +0000
+++ src/provisioningserver/pxe/tftppath.py	2014-03-25 17:45:57 +0000
@@ -40,7 +40,7 @@
     return "pxelinux.0"
 
 
-# TODO: move this; it is now only used for testing.
+# XXX allenap: move this; it is now only used for testing.
 def compose_config_path(mac):
     """Compose the TFTP path for a PXE configuration file.
 
@@ -60,7 +60,11 @@
         htype=ARP_HTYPE.ETHERNET, mac=mac)
 
 
-def compose_image_path(arch, subarch, release, label, purpose):
+# XXX rvb 2014-03-21 bug=1235479: The 'purpose' is made optional for now so
+# that this method can cope with both the old layout (which had the 'purpose'
+# as part of the images' path) and the new one.  Once the old script is
+# removed, this parameter should be removed as well.
+def compose_image_path(arch, subarch, release, label, purpose=None):
     """Compose the TFTP path for a PXE kernel/initrd directory.
 
     The path returned is relative to the TFTP root, as it would be
@@ -69,14 +73,16 @@
     :param arch: Main machine architecture.
     :param subarch: Sub-architecture, or "generic" if there is none.
     :param release: Operating system release, e.g. "precise".
-    :param label: An image label, e.g. for a beta version, or "release" for
-        the default images.
+    :param label: Release label, e.g. "release" or "alpha-2".
     :param purpose: Purpose of the image, e.g. "install" or
         "commissioning".
     :return: Path for the corresponding image directory (containing a
         kernel and initrd) as exposed over TFTP.
     """
-    return '/'.join([arch, subarch, release, label, purpose])
+    elements = [arch, subarch, release, label]
+    if purpose is not None:
+        elements.append(purpose)
+    return '/'.join(elements)
 
 
 def locate_tftp_path(path, tftproot):
@@ -143,23 +149,32 @@
 
 
 def extract_image_params(path):
-    """Represent a list of TFTP path elements as a boot-image dict.
+    """Represent a list of TFTP path elements as a list of boot-image dicts.
 
-    The path must consist of a full [architecture, subarchitecture, release,
-    purpose] that identify a kind of boot that we may need an image for.
+    The path must consist of a full [architecture, subarchitecture, release]
+    that identify a kind of boot that we may need an image for.
     """
-    arch, subarch, release, label, purpose = path
-    return dict(
-        architecture=arch, subarchitecture=subarch, release=release,
-        label=label, purpose=purpose)
+    arch, subarch, release, label = path
+    # XXX: rvb 2014-03-24: The images import script currently imports all the
+    # images for the configured selections (where a selection is an
+    # arch/subarch/series/label combination).  When the import script grows the
+    # ability to import the images for a particular purpose, we need to change
+    # this code to report what is actually present.
+    purposes = ['commissioning', 'install', 'xinstall']
+    return [
+        dict(
+            architecture=arch, subarchitecture=subarch,
+            release=release, label=label, purpose=purpose)
+        for purpose in purposes
+        ]
 
 
 def list_boot_images(tftproot):
     """List the available boot images.
 
     :param tftproot: TFTP root directory.
-    :return: An iterable of dicts, describing boot images as consumed by
-        the report_boot_images API call.
+    :return: A list of dicts, describing boot images as consumed by the
+        `report_boot_images` API call.
     """
     # The sub-directories directly under tftproot, if they contain
     # images, represent architectures.
@@ -170,10 +185,12 @@
     paths = [[subdir] for subdir in potential_archs]
 
     # Extend paths deeper into the filesystem, through the levels that
-    # represent sub-architecture, release, and purpose.  Any directory
+    # represent sub-architecture, release, and label.  Any directory
     # that doesn't extend this deep isn't a boot image.
-    for level in ['subarch', 'release', 'label', 'purpose']:
+    for level in ['subarch', 'release', 'label']:
         paths = drill_down(tftproot, paths)
 
     # Each path we find this way should be a boot image.
-    return [extract_image_params(path) for path in paths]
+    # This gets serialised to JSON, so we really have to return a list, not
+    # just any iterable.
+    return sum([extract_image_params(path) for path in paths], [])

=== modified file 'src/provisioningserver/rpc/tests/test_clusterservice.py'
--- src/provisioningserver/rpc/tests/test_clusterservice.py	2014-03-20 22:36:32 +0000
+++ src/provisioningserver/rpc/tests/test_clusterservice.py	2014-03-25 17:45:57 +0000
@@ -175,11 +175,11 @@
         subarchs = "generic", "special"
         releases = "precise", "trusty"
         labels = "beta-1", "release"
-        purposes = "commission", "install"
+        purposes = "commissioning", "install", "xinstall"
 
         # Create a TFTP file tree with a variety of subdirectories.
         tftpdir = self.make_dir()
-        for options in product(archs, subarchs, releases, labels, purposes):
+        for options in product(archs, subarchs, releases, labels):
             os.makedirs(os.path.join(tftpdir, *options))
 
         # Ensure that list_boot_images() uses the above TFTP file tree.

=== modified file 'src/provisioningserver/testing/boot_images.py'
--- src/provisioningserver/testing/boot_images.py	2014-03-11 07:34:00 +0000
+++ src/provisioningserver/testing/boot_images.py	2014-03-25 17:45:57 +0000
@@ -1,4 +1,4 @@
-# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# Copyright 2012-2014 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Test helpers for boot-image parameters."""
@@ -24,13 +24,26 @@
 
     These are the parameters that together describe a kind of boot for
     which we may need a kernel and initrd: architecture,
-    sub-architecture, Ubuntu release, boot purpose and simplestreams
-    label.  See the `tftppath` module for how these fit together.
+    sub-architecture, Ubuntu release, boot purpose, and release label.
     """
     return dict(
         architecture=factory.make_name('architecture'),
         subarchitecture=factory.make_name('subarchitecture'),
         release=factory.make_name('release'),
+        label=factory.make_name('label'),
         purpose=factory.make_name('purpose'),
+        )
+
+
+def make_boot_image_storage_params():
+    """Create a dict of boot-image parameters as used to store the image.
+
+    These are the parameters that together describe a path to store a boot
+    image: architecture, sub-architecture, Ubuntu release, and release label.
+    """
+    return dict(
+        architecture=factory.make_name('architecture'),
+        subarchitecture=factory.make_name('subarchitecture'),
+        release=factory.make_name('release'),
         label=factory.make_name('label'),
         )

=== modified file 'src/provisioningserver/tests/test_config.py'
--- src/provisioningserver/tests/test_config.py	2014-03-21 08:19:20 +0000
+++ src/provisioningserver/tests/test_config.py	2014-03-25 17:45:57 +0000
@@ -1,4 +1,4 @@
-# Copyright 2005-2013 Canonical Ltd.  This software is licensed under the
+# Copyright 2005-2014 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tests for provisioning configuration."""
@@ -136,15 +136,15 @@
                 'images_directory': '/var/lib/maas/ephemeral',
                 'releases': None,
                 },
-            # XXX jtv 2014-03-21, bug=1295479: Unused until we start using
-            # the new import script.
             'sources': [
                 {
                     'path': (
                         'http://maas.ubuntu.com/images/ephemeral/releases/'),
+                    'keyring': (
+                        '/usr/share/keyrings/ubuntu-cloudimage-keyring.gpg'),
                     'selections': [
                         {
-                            'arch': '*',
+                            'arches': ['*'],
                             'release': '*',
                             'subarches': ['*'],
                         },
@@ -152,6 +152,7 @@
                 },
             ],
             'storage': '/var/lib/maas/boot-resources/',
+            'configure_me': False,
         },
         'broker': {
             'host': 'localhost',
@@ -169,7 +170,7 @@
         'tftp': {
             'generator': 'http://localhost/MAAS/api/1.0/pxeconfig/',
             'port': 69,
-            'root': "/var/lib/maas/tftp",
+            'root': "/var/lib/maas/boot-resources/current/",
             },
         }
 
@@ -266,10 +267,10 @@
         self.assertNotEqual(first_load['logfile'], second_load['logfile'])
         self.assertEqual(logfile, second_load['logfile'])
         self.assertIsNot(first_load['boot'], second_load['boot'])
-        first_load['boot']['architectures'] = [factory.make_name('otherarch')]
+        first_load['boot']['storage'] = [factory.make_name('otherstorage')]
         self.assertNotEqual(
-            first_load['boot']['architectures'],
-            second_load['boot']['architectures'])
+            first_load['boot']['storage'],
+            second_load['boot']['storage'])
 
     def test_oops_directory_without_reporter(self):
         # It is an error to omit the OOPS reporter if directory is specified.

=== modified file 'src/provisioningserver/tests/test_kernel_opts.py'
--- src/provisioningserver/tests/test_kernel_opts.py	2014-03-17 08:48:46 +0000
+++ src/provisioningserver/tests/test_kernel_opts.py	2014-03-25 17:45:57 +0000
@@ -29,13 +29,12 @@
     compose_arch_opts,
     compose_kernel_command_line,
     compose_preseed_opt,
-    EphemeralImagesDirectoryNotFound,
+    get_ephemeral_name,
     get_last_directory,
     ISCSI_TARGET_NAME_PREFIX,
     KernelParameters,
     prefix_target_name,
     )
-from provisioningserver.testing.config import ConfigFixture
 from testtools.matchers import (
     Contains,
     ContainsAll,
@@ -169,8 +168,6 @@
     def test_xinstall_compose_kernel_command_line_inc_purpose_opts(self):
         # The result of compose_kernel_command_line includes the purpose
         # options for a non "xinstall" node.
-        get_ephemeral_name = self.patch(kernel_opts, "get_ephemeral_name")
-        get_ephemeral_name.return_value = "RELEASE-ARCH"
         params = self.make_kernel_parameters(purpose="xinstall")
         cmdline = compose_kernel_command_line(params)
         self.assertThat(
@@ -184,8 +181,6 @@
     def test_commissioning_compose_kernel_command_line_inc_purpose_opts(self):
         # The result of compose_kernel_command_line includes the purpose
         # options for a non "commissioning" node.
-        get_ephemeral_name = self.patch(kernel_opts, "get_ephemeral_name")
-        get_ephemeral_name.return_value = "RELEASE-ARCH"
         params = self.make_kernel_parameters(purpose="commissioning")
         cmdline = compose_kernel_command_line(params)
         self.assertThat(
@@ -212,8 +207,6 @@
     def test_compose_kernel_command_line_inc_common_opts(self):
         # Test that some kernel arguments appear on commissioning, install
         # and xinstall command lines.
-        get_ephemeral_name = self.patch(kernel_opts, "get_ephemeral_name")
-        get_ephemeral_name.return_value = "RELEASE-ARCH"
         expected = ["nomodeset"]
 
         params = self.make_kernel_parameters(
@@ -231,32 +224,12 @@
         cmdline = compose_kernel_command_line(params)
         self.assertThat(cmdline, ContainsAll(expected))
 
-    def create_ephemeral_info(self, name, arch, release):
-        """Create a pseudo-real ephemeral info file."""
-        ephemeral_info = """
-            release=%s
-            stream=ephemeral
-            label=release
-            serial=20120424
-            arch=%s
-            name=%s
-            """ % (release, arch, name)
-        ephemeral_root = self.make_dir()
-        config = {"boot": {"ephemeral": {"images_directory": ephemeral_root}}}
-        self.useFixture(ConfigFixture(config))
-        ephemeral_dir = os.path.join(
-            ephemeral_root, release, 'ephemeral', arch, release)
-        os.makedirs(ephemeral_dir)
-        factory.make_file(
-            ephemeral_dir, name='info', contents=ephemeral_info)
-
     def test_compose_kernel_command_line_inc_purpose_opts_xinstall_node(self):
         # The result of compose_kernel_command_line includes the purpose
         # options for a "xinstall" node.
-        ephemeral_name = factory.make_name("ephemeral")
         params = self.make_kernel_parameters(purpose="xinstall")
-        self.create_ephemeral_info(
-            ephemeral_name, params.arch, params.release)
+        ephemeral_name = get_ephemeral_name(
+            params.arch, params.subarch, params.release, params.label)
         self.assertThat(
             compose_kernel_command_line(params),
             ContainsAll([
@@ -269,10 +242,9 @@
     def test_compose_kernel_command_line_inc_purpose_opts_comm_node(self):
         # The result of compose_kernel_command_line includes the purpose
         # options for a "commissioning" node.
-        ephemeral_name = factory.make_name("ephemeral")
         params = self.make_kernel_parameters(purpose="commissioning")
-        self.create_ephemeral_info(
-            ephemeral_name, params.arch, params.release)
+        ephemeral_name = get_ephemeral_name(
+            params.arch, params.subarch, params.release, params.label)
         self.assertThat(
             compose_kernel_command_line(params),
             ContainsAll([
@@ -282,15 +254,6 @@
                 "iscsi_target_ip=%s" % params.fs_host,
                 ]))
 
-    def test_compose_kernel_command_line_reports_error_about_missing_dir(self):
-        params = self.make_kernel_parameters(purpose="commissioning")
-        missing_dir = factory.make_name('missing-dir')
-        config = {"boot": {"ephemeral": {"images_directory": missing_dir}}}
-        self.useFixture(ConfigFixture(config))
-        self.assertRaises(
-            EphemeralImagesDirectoryNotFound,
-            compose_kernel_command_line, params)
-
     def test_compose_preseed_kernel_opt_returns_kernel_option(self):
         dummy_preseed_url = factory.make_name("url")
         self.assertEqual(

=== modified file 'src/provisioningserver/tests/test_maas_import_pxe_files.py'
--- src/provisioningserver/tests/test_maas_import_pxe_files.py	2014-03-22 17:21:55 +0000
+++ src/provisioningserver/tests/test_maas_import_pxe_files.py	2014-03-25 17:45:57 +0000
@@ -16,6 +16,7 @@
 
 import os
 from subprocess import check_call
+import unittest
 
 from maastesting import root
 from maastesting.factory import factory
@@ -168,6 +169,11 @@
 
     def setUp(self):
         super(TestImportPXEFiles, self).setUp()
+        raise unittest.SkipTest(
+            "XXX rvb 2014-03-21 bug=1295479: Disabled.  The "
+            "maas-import-pxe-files script has been replaced with a "
+            "new version to use simplestreams v2's data.  These tests need "
+            "to be completely refactored.")
         self.tftproot = self.make_dir()
         self.config = {"tftp": {"root": self.tftproot}}
         self.config_fixture = ConfigFixture(self.config)

=== modified file 'src/provisioningserver/tests/test_upgrade_cluster.py'
--- src/provisioningserver/tests/test_upgrade_cluster.py	2014-03-18 04:25:00 +0000
+++ src/provisioningserver/tests/test_upgrade_cluster.py	2014-03-25 17:45:57 +0000
@@ -25,10 +25,17 @@
 import os.path
 
 from maastesting.factory import factory
-from maastesting.matchers import MockCalledOnceWith
+from maastesting.matchers import (
+    MockCalledOnceWith,
+    MockNotCalled,
+    )
 from maastesting.testcase import MAASTestCase
-from mock import Mock
+from mock import (
+    ANY,
+    Mock,
+    )
 from provisioningserver import upgrade_cluster
+from provisioningserver.config import Config
 from provisioningserver.pxe.install_image import install_image
 from provisioningserver.testing.config import ConfigFixture
 from testtools.matchers import (
@@ -580,3 +587,50 @@
         entries_before = listdir(tftproot)
         upgrade_cluster.add_label_directory_level_to_boot_images()
         self.assertItemsEqual(entries_before, listdir(tftproot))
+
+
+class TestGenerateBootResourcesConfig(MAASTestCase):
+    """Tests for the `generate_boot_resources_config` upgrade."""
+
+    def patch_rewrite_boot_resources_config(self):
+        """Patch `rewrite_boot_resources_config` with a mock."""
+        return self.patch(upgrade_cluster, 'rewrite_boot_resources_config')
+
+    def patch_config(self, config):
+        """Patch the `bootresources.yaml` config with a given dict."""
+        original_load = Config.load_from_cache
+
+        @classmethod
+        def fake_config_load(cls, filename=None):
+            """Fake `Config.load_from_cache`.
+
+            Returns a susbtitute for `bootresources.yaml`, but defers to the
+            original implementation for other files.  This means we can still
+            patch the original, and it means we'll probably get a tell-tale
+            error if any code underneath the tests accidentally tries to
+            load pserv.yaml.
+            """
+            if os.path.basename(filename) == 'bootresources.yaml':
+                return config
+            else:
+                return original_load(Config, filename=filename)
+
+        self.patch(Config, 'load_from_cache', fake_config_load)
+
+    def test_does_nothing_if_configure_me_is_False(self):
+        self.patch_config({'boot': {'configure_me': False}})
+        rewrite_config = self.patch_rewrite_boot_resources_config()
+        upgrade_cluster.generate_boot_resources_config()
+        self.assertThat(rewrite_config, MockNotCalled())
+
+    def test_does_nothing_if_configure_me_is_missing(self):
+        self.patch_config({'boot': {}})
+        rewrite_config = self.patch_rewrite_boot_resources_config()
+        upgrade_cluster.generate_boot_resources_config()
+        self.assertThat(rewrite_config, MockNotCalled())
+
+    def test_rewrites_if_configure_me_is_True(self):
+        self.patch_config({'boot': {'configure_me': True}})
+        rewrite_config = self.patch_rewrite_boot_resources_config()
+        upgrade_cluster.generate_boot_resources_config()
+        self.assertThat(rewrite_config, MockCalledOnceWith(ANY))

=== modified file 'src/provisioningserver/tftp.py'
--- src/provisioningserver/tftp.py	2014-03-22 17:21:55 +0000
+++ src/provisioningserver/tftp.py	2014-03-25 17:45:57 +0000
@@ -329,6 +329,8 @@
         :param root: The root directory for this TFTP server.
         :param port: The port on which each server should be started.
         :param generator: The URL to be queried for PXE configuration.
+            This will normally point to the `pxeconfig` endpoint on the
+            region-controller API.
         """
         super(TFTPService, self).__init__()
         self.backend, self.port = TFTPBackend(root, generator), port

=== modified file 'src/provisioningserver/upgrade_cluster.py'
--- src/provisioningserver/upgrade_cluster.py	2014-03-18 04:10:37 +0000
+++ src/provisioningserver/upgrade_cluster.py	2014-03-25 17:45:57 +0000
@@ -44,7 +44,10 @@
 from shutil import rmtree
 
 from provisioningserver.config import Config
-from provisioningserver.utils import ensure_dir
+from provisioningserver.utils import (
+    ensure_dir,
+    locate_config,
+    )
 
 
 logger = getLogger(__name__)
@@ -246,6 +249,25 @@
             move_real_boot_image(tftproot, image)
 
 
+def rewrite_boot_resources_config(config_file):
+    """Rewrite the `bootresources.yaml` configuration."""
+
+
+def generate_boot_resources_config():
+    """Upgrade hook: rewrite `bootresources.yaml` based on boot images.
+
+    This finds boot images downloaded into the old, pre-Simplestreams tftp
+    root, and writes a boot-resources configuration to import a similar set of
+    images using Simplestreams.
+    """
+    config_file = locate_config('bootresources.yaml')
+    boot_resources = Config.load_from_cache(config_file)
+    if not boot_resources['boot'].get('configure_me', False):
+        # Already configured.
+        return
+    rewrite_boot_resources_config(config_file)
+
+
 # Upgrade hooks, from oldest to newest.  The hooks are callables, taking no
 # arguments.  They are called in order.
 #
@@ -253,6 +275,7 @@
 # no record of previous upgrades.
 UPGRADE_HOOKS = [
     add_label_directory_level_to_boot_images,
+    generate_boot_resources_config,
     ]