Merge lp:~ttx/glance/d3-merge into lp:~hudson-openstack/glance/milestone-proposed

[Thierry Carrez]

Merge diablo-3 development from trunk (rev160)

Merge diablo-3 development from trunk (rev160)

This really should be automated.
Check using 'bzr diff --old lp:glance --new lp:~ttx/glance/d3-merge'

[Thierry Carrez (ttx)]

Verified identical to lp:glance

review: Approve

=== modified file 'Authors'
--- Authors	2011-06-27 14:37:40 +0000
+++ Authors	2011-07-26 10:13:35 +0000
@@ -6,11 +6,14 @@
 Donal Lafferty <[e-mail]>
 Eldar Nugaev <[e-mail]>
 Ewan Mellor <[e-mail]>
+Isaku Yamahata <[e-mail]>
+Jason Koelker <[e-mail]>
 Jay Pipes <[e-mail]>
 Jinwoo 'Joseph' Suh <[e-mail]>
 Josh Kearney <[e-mail]>
 Justin Shepherd <[e-mail]>
 Ken Pepple <[e-mail]>
+Kevin L. Mitchell <[e-mail]>
 Matt Dietz <[e-mail]>
 Monty Taylor <[e-mail]>
 Rick Clark <[e-mail]>

=== modified file 'bin/glance'
--- bin/glance	2011-06-26 20:43:41 +0000
+++ bin/glance	2011-07-26 10:13:35 +0000
@@ -22,6 +22,7 @@
 stored in one or more Glance nodes.
 """
 
+import functools
 import optparse
 import os
 import re
@@ -36,15 +37,45 @@
 if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
     sys.path.insert(0, possible_topdir)
 
-from glance import client
+from glance import client as glance_client
 from glance import version
 from glance.common import exception
-from glance.common import utils
+from glance.common import utils as common_utils
+from glance import utils
 
 SUCCESS = 0
 FAILURE = 1
 
 
+#TODO(sirp): make more of the actions use this decorator
+def catch_error(action):
+    """Decorator to provide sensible default error handling for actions."""
+    def wrap(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                ret = func(*args, **kwargs)
+                return SUCCESS if ret is None else ret
+            except Exception, e:
+                print "Failed to %s. Got error:" % action
+                pieces = str(e).split('\n')
+                for piece in pieces:
+                    print piece
+                return FAILURE
+
+        return wrapper
+    return wrap
+
+
+def get_percent_done(image):
+    try:
+        pct_done = image['size'] * 100 / int(image['expected_size'])
+    except (ValueError, ZeroDivisionError):
+        # NOTE(sirp): Ignore if expected_size isn't a number, or if it's 0
+        pct_done = "UNKNOWN"
+    return pct_done
+
+
 def get_image_fields_from_args(args):
     """
     Validate the set of arguments passed as field name/value pairs
@@ -152,7 +183,7 @@
         return FAILURE
 
     image_meta = {'name': fields.pop('name'),
-                  'is_public': utils.bool_from_string(
+                  'is_public': common_utils.bool_from_string(
                       fields.pop('is_public', False)),
                   'disk_format': fields.pop('disk_format', 'raw'),
                   'container_format': fields.pop('container_format', 'ovf')}
@@ -276,7 +307,7 @@
 
     # Have to handle "boolean" values specially...
     if 'is_public' in fields:
-        image_meta['is_public'] = utils.bool_from_string(
+        image_meta['is_public'] = common_utils.bool_from_string(
             fields.pop('is_public'))
 
     # Add custom attributes, which are all the arguments remaining
@@ -364,40 +395,37 @@
         return FAILURE
 
 
+@catch_error('show index')
 def images_index(options, args):
     """
 %(prog)s index [options]
 
 Returns basic information for all public images
 a Glance server knows about"""
-    c = get_client(options)
-    try:
-        images = c.get_images()
-        if len(images) == 0:
-            print "No public images found."
-            return SUCCESS
-
-        print "Found %d public images..." % len(images)
-        print "%-16s %-30s %-20s %-20s %-14s" % (("ID"),
-                                           ("Name"),
-                                           ("Disk Format"),
-                                           ("Container Format"),
-                                           ("Size"))
-        print ('-' * 16) + " " + ('-' * 30) + " "\
-              + ('-' * 20) + " " + ('-' * 20) + " " + ('-' * 14)
-        for image in images:
-            print "%-16s %-30s %-20s %-20s %14d" % (image['id'],
-                                              image['name'],
-                                              image['disk_format'],
-                                              image['container_format'],
-                                              int(image['size']))
+    client = get_client(options)
+    images = client.get_images()
+    if not images:
+        print "No public images found."
         return SUCCESS
-    except Exception, e:
-        print "Failed to show index. Got error:"
-        pieces = str(e).split('\n')
-        for piece in pieces:
-            print piece
-        return FAILURE
+
+    print "Found %d public images..." % len(images)
+
+    pretty_table = utils.PrettyTable()
+    pretty_table.add_column(16, label="ID")
+    pretty_table.add_column(30, label="Name")
+    pretty_table.add_column(20, label="Disk Format")
+    pretty_table.add_column(20, label="Container Format")
+    pretty_table.add_column(14, label="Size", just="r")
+
+    print pretty_table.make_header()
+
+    for image in images:
+        print pretty_table.make_row(
+            image['id'],
+            image['name'],
+            image['disk_format'],
+            image['container_format'],
+            image['size'])
 
 
 def images_detailed(options, args):
@@ -458,13 +486,256 @@
     return SUCCESS
 
 
+@catch_error('show cached images')
+def cache_index(options, args):
+    """
+%(prog)s cache-index [options]
+
+List all images currently cached"""
+    client = get_client(options)
+    images = client.get_cached_images()
+    if not images:
+        print "No cached images."
+        return SUCCESS
+
+    print "Found %d cached images..." % len(images)
+
+    pretty_table = utils.PrettyTable()
+    pretty_table.add_column(16, label="ID")
+    pretty_table.add_column(30, label="Name")
+    pretty_table.add_column(19, label="Last Accessed (UTC)")
+    # 1 TB takes 13 characters to display: len(str(2**40)) == 13
+    pretty_table.add_column(14, label="Size", just="r")
+    pretty_table.add_column(10, label="Hits", just="r")
+
+    print pretty_table.make_header()
+
+    for image in images:
+        print pretty_table.make_row(
+            image['id'],
+            image['name'],
+            image['last_accessed'],
+            image['size'],
+            image['hits'])
+
+
+@catch_error('show invalid cache images')
+def cache_invalid(options, args):
+    """
+%(prog)s cache-invalid [options]
+
+List current invalid cache images"""
+    client = get_client(options)
+    images = client.get_invalid_cached_images()
+    if not images:
+        print "No invalid cached images."
+        return SUCCESS
+
+    print "Found %d invalid cached images..." % len(images)
+
+    pretty_table = utils.PrettyTable()
+    pretty_table.add_column(16, label="ID")
+    pretty_table.add_column(30, label="Name")
+    pretty_table.add_column(30, label="Error")
+    pretty_table.add_column(19, label="Last Modified (UTC)")
+    # 1 TB takes 13 characters to display: len(str(2**40)) == 13
+    pretty_table.add_column(14, label="Size", just="r")
+    pretty_table.add_column(14, label="Expected Size", just="r")
+    pretty_table.add_column(7, label="% Done", just="r")
+
+    print pretty_table.make_header()
+
+    for image in images:
+        print pretty_table.make_row(
+            image['id'],
+            image['name'],
+            image['error'],
+            image['last_accessed'],
+            image['size'],
+            image['expected_size'],
+            get_percent_done(image))
+
+
+@catch_error('show incomplete cache images')
+def cache_incomplete(options, args):
+    """
+%(prog)s cache-incomplete [options]
+
+List images currently being fetched"""
+    client = get_client(options)
+    images = client.get_incomplete_cached_images()
+    if not images:
+        print "No incomplete cached images."
+        return SUCCESS
+
+    print "Found %d incomplete cached images..." % len(images)
+
+    pretty_table = utils.PrettyTable()
+    pretty_table.add_column(16, label="ID")
+    pretty_table.add_column(30, label="Name")
+    pretty_table.add_column(19, label="Last Modified (UTC)")
+    # 1 TB takes 13 characters to display: len(str(2**40)) == 13
+    pretty_table.add_column(14, label="Size", just="r")
+    pretty_table.add_column(14, label="Expected Size", just="r")
+    pretty_table.add_column(7, label="% Done", just="r")
+
+    print pretty_table.make_header()
+
+    for image in images:
+        print pretty_table.make_row(
+            image['id'],
+            image['name'],
+            image['last_modified'],
+            image['size'],
+            image['expected_size'],
+            get_percent_done(image))
+
+
+@catch_error('purge the specified cached image')
+def cache_purge(options, args):
+    """
+%(prog)s cache-purge [options]
+
+Purges an image from the cache"""
+    try:
+        image_id = args.pop()
+    except IndexError:
+        print "Please specify the ID of the image you wish to purge "
+        print "from the cache as the first argument"
+        return FAILURE
+
+    if not options.force and \
+       not user_confirm("Purge cached image %s?" % (image_id,), default=False):
+        print 'Not purging cached image %s' % (image_id,)
+        return FAILURE
+
+    client = get_client(options)
+    client.purge_cached_image(image_id)
+
+    if options.verbose:
+        print "done"
+
+
+@catch_error('clear all cached images')
+def cache_clear(options, args):
+    """
+%(prog)s cache-clear [options]
+
+Removes all images from the cache"""
+    if not options.force and \
+       not user_confirm("Clear all cached images?", default=False):
+        print 'Not purging any cached images.'
+        return FAILURE
+
+    client = get_client(options)
+    num_purged = client.clear_cached_images()
+
+    if options.verbose:
+        print "Purged %(num_purged)s cached images" % locals()
+
+
+@catch_error('reap invalid images')
+def cache_reap_invalid(options, args):
+    """
+%(prog)s cache-reap-invalid [options]
+
+Reaps any invalid images that were left for
+debugging purposes"""
+    if not options.force and \
+       not user_confirm("Reap all invalid cached images?", default=False):
+        print 'Not reaping any invalid cached images.'
+        return FAILURE
+
+    client = get_client(options)
+    num_reaped = client.reap_invalid_cached_images()
+
+    if options.verbose:
+        print "Reaped %(num_reaped)s invalid cached images" % locals()
+
+
+@catch_error('reap stalled images')
+def cache_reap_stalled(options, args):
+    """
+%(prog)s cache-reap-stalled [options]
+
+Reaps any stalled incomplete images"""
+    if not options.force and \
+       not user_confirm("Reap all stalled cached images?", default=False):
+        print 'Not reaping any stalled cached images.'
+        return FAILURE
+
+    client = get_client(options)
+    num_reaped = client.reap_stalled_cached_images()
+
+    if options.verbose:
+        print "Reaped %(num_reaped)s stalled cached images" % locals()
+
+
+@catch_error('prefetch the specified cached image')
+def cache_prefetch(options, args):
+    """
+%(prog)s cache-prefetch [options]
+
+Pre-fetch an image or list of images into the cache"""
+    image_ids = args
+    if not image_ids:
+        print "Please specify the ID or a list of image IDs of the images "\
+              "you wish to "
+        print "prefetch from the cache as the first argument"
+        return FAILURE
+
+    client = get_client(options)
+    for image_id in image_ids:
+        if options.verbose:
+            print "Prefetching image '%s'" % image_id
+
+        try:
+            client.prefetch_cache_image(image_id)
+        except exception.NotFound:
+            print "No image with ID %s was found" % image_id
+            continue
+
+    if options.verbose:
+        print "done"
+
+
+@catch_error('show prefetching images')
+def cache_prefetching(options, args):
+    """
+%(prog)s cache-prefetching [options]
+
+List images that are being prefetched"""
+    client = get_client(options)
+    images = client.get_prefetching_cache_images()
+    if not images:
+        print "No images being prefetched."
+        return SUCCESS
+
+    print "Found %d images being prefetched..." % len(images)
+
+    pretty_table = utils.PrettyTable()
+    pretty_table.add_column(16, label="ID")
+    pretty_table.add_column(30, label="Name")
+    pretty_table.add_column(19, label="Last Accessed (UTC)")
+    pretty_table.add_column(10, label="Status", just="r")
+
+    print pretty_table.make_header()
+
+    for image in images:
+        print pretty_table.make_row(
+            image['id'],
+            image['name'],
+            image['last_accessed'],
+            image['status'])
+
+
 def get_client(options):
     """
     Returns a new client object to a Glance server
     specified by the --host and --port options
     supplied to the CLI
     """
-    return client.Client(host=options.host,
+    return glance_client.Client(host=options.host,
                          port=options.port)
 
 
@@ -500,28 +771,23 @@
 
     :param parser: The option parser
     """
-    COMMANDS = {'help': print_help,
-                'add': image_add,
-                'update': image_update,
-                'delete': image_delete,
-                'index': images_index,
-                'details': images_detailed,
-                'show': image_show,
-                'clear': images_clear}
-
     if not cli_args:
         cli_args.append('-h')  # Show options in usage output...
 
     (options, args) = parser.parse_args(cli_args)
 
+    # HACK(sirp): Make the parser available to the print_help method
+    # print_help is a command, so it only accepts (options, args); we could
+    # one-off have it take (parser, options, args), however, for now, I think
+    # this little hack will suffice
+    options.__parser = parser
+
     if not args:
         parser.print_usage()
         sys.exit(0)
-    else:
-        command_name = args.pop(0)
-        if command_name not in COMMANDS.keys():
-            sys.exit("Unknown command: %s" % command_name)
-    command = COMMANDS[command_name]
+
+    command_name = args.pop(0)
+    command = lookup_command(parser, command_name)
 
     return (options, command, args)
 
@@ -530,28 +796,55 @@
     """
     Print help specific to a command
     """
-    COMMANDS = {'add': image_add,
-                'update': image_update,
-                'delete': image_delete,
-                'index': images_index,
-                'details': images_detailed,
-                'show': image_show,
-                'clear': images_clear}
-
     if len(args) != 1:
         sys.exit("Please specify a command")
 
-    command = args.pop()
-    if command not in COMMANDS.keys():
+    parser = options.__parser
+    command_name = args.pop()
+    command = lookup_command(parser, command_name)
+
+    print command.__doc__ % {'prog': os.path.basename(sys.argv[0])}
+
+
+def lookup_command(parser, command_name):
+    BASE_COMMANDS = {'help': print_help}
+
+    IMAGE_COMMANDS = {
+                'add': image_add,
+                'update': image_update,
+                'delete': image_delete,
+                'index': images_index,
+                'details': images_detailed,
+                'show': image_show,
+                'clear': images_clear}
+
+    CACHE_COMMANDS = {
+                'cache-index': cache_index,
+                'cache-invalid': cache_invalid,
+                'cache-incomplete': cache_incomplete,
+                'cache-prefetching': cache_prefetching,
+                'cache-prefetch': cache_prefetch,
+                'cache-purge': cache_purge,
+                'cache-clear': cache_clear,
+                'cache-reap-invalid': cache_reap_invalid,
+                'cache-reap-stalled': cache_reap_stalled}
+
+    commands = {}
+    for command_set in (BASE_COMMANDS, IMAGE_COMMANDS, CACHE_COMMANDS):
+        commands.update(command_set)
+
+    try:
+        command = commands[command_name]
+    except KeyError:
         parser.print_usage()
-        if args:
-            sys.exit("Unknown command: %s" % command)
+        sys.exit("Unknown command: %s" % command_name)
 
-    print COMMANDS[command].__doc__ % {'prog': os.path.basename(sys.argv[0])}
+    return command
 
 
 def user_confirm(prompt, default=False):
-    """Yes/No question dialog with user.
+    """
+    Yes/No question dialog with user.
 
     :param prompt: question/statement to present to user (string)
     :param default: boolean value to return if empty string
@@ -595,6 +888,27 @@
 
     clear           Removes all images and metadata from Glance
 
+
+Cache Commands:
+
+    cache-index          List all images currently cached
+
+    cache-invalid        List current invalid cache images
+
+    cache-incomplete     List images currently being fetched
+
+    cache-prefetching    List images that are being prefetched
+
+    cache-prefetch       Pre-fetch an image or list of images into the cache
+
+    cache-purge          Purges an image from the cache
+
+    cache-clear          Removes all images from the cache
+
+    cache-reap-invalid   Reaps any invalid images that were left for
+                         debugging purposes
+
+    cache-reap-stalled   Reaps any stalled incomplete images
 """
 
     oparser = optparse.OptionParser(version='%%prog %s'

=== added file 'bin/glance-cache-prefetcher'
--- bin/glance-cache-prefetcher	1970-01-01 00:00:00 +0000
+++ bin/glance-cache-prefetcher	2011-07-26 10:13:35 +0000
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Glance Image Cache Pre-fetcher
+
+This is meant to be run as a periodic task from cron.
+"""
+
+import optparse
+import os
+import sys
+
+# If ../glance/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+from glance import version
+from glance.common import config
+from glance.common import wsgi
+
+
+def create_options(parser):
+    """
+    Sets up the CLI and config-file options that may be
+    parsed and program commands.
+
+    :param parser: The option parser
+    """
+    config.add_common_options(parser)
+    config.add_log_options(parser)
+
+
+if __name__ == '__main__':
+    oparser = optparse.OptionParser(version='%%prog %s'
+                                    % version.version_string())
+    create_options(oparser)
+    (options, args) = config.parse_options(oparser)
+
+    try:
+        conf, app = config.load_paste_app('glance-prefetcher', options, args)
+        app.run()
+    except RuntimeError, e:
+        sys.exit("ERROR: %s" % e)

=== added file 'bin/glance-cache-pruner'
--- bin/glance-cache-pruner	1970-01-01 00:00:00 +0000
+++ bin/glance-cache-pruner	2011-07-26 10:13:35 +0000
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Glance Image Cache Pruner
+
+This is meant to be run as a periodic task, perhaps every half-hour.
+"""
+
+import optparse
+import os
+import sys
+
+# If ../glance/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+from glance import version
+from glance.common import config
+from glance.common import wsgi
+
+
+def create_options(parser):
+    """
+    Sets up the CLI and config-file options that may be
+    parsed and program commands.
+
+    :param parser: The option parser
+    """
+    config.add_common_options(parser)
+    config.add_log_options(parser)
+
+
+if __name__ == '__main__':
+    oparser = optparse.OptionParser(version='%%prog %s'
+                                    % version.version_string())
+    create_options(oparser)
+    (options, args) = config.parse_options(oparser)
+
+    try:
+        conf, app = config.load_paste_app('glance-pruner', options, args)
+        app.run()
+    except RuntimeError, e:
+        sys.exit("ERROR: %s" % e)

=== added file 'bin/glance-cache-reaper'
--- bin/glance-cache-reaper	1970-01-01 00:00:00 +0000
+++ bin/glance-cache-reaper	2011-07-26 10:13:35 +0000
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Glance Image Cache Invalid Cache Entry and Stalled Image  Reaper
+
+This is meant to be run as a periodic task from cron.
+
+If something goes wrong while we're caching an image (for example the fetch
+times out, or an exception is raised), we create an 'invalid' entry. These
+entires are left around for debugging purposes. However, after some period of
+time, we want to cleans these up, aka reap them.
+
+Also, if an incomplete image hangs around past the image_cache_stall_timeout
+period, we automatically sweep it up.
+"""
+
+import optparse
+import os
+import sys
+
+# If ../glance/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+from glance import version
+from glance.common import config
+from glance.common import wsgi
+
+
+def create_options(parser):
+    """
+    Sets up the CLI and config-file options that may be
+    parsed and program commands.
+
+    :param parser: The option parser
+    """
+    config.add_common_options(parser)
+    config.add_log_options(parser)
+
+
+if __name__ == '__main__':
+    oparser = optparse.OptionParser(version='%%prog %s'
+                                    % version.version_string())
+    create_options(oparser)
+    (options, args) = config.parse_options(oparser)
+
+    try:
+        conf, app = config.load_paste_app('glance-reaper', options, args)
+        app.run()
+    except RuntimeError, e:
+        sys.exit("ERROR: %s" % e)

=== modified file 'bin/glance-control'
--- bin/glance-control	2011-05-09 18:55:46 +0000
+++ bin/glance-control	2011-07-26 10:13:35 +0000
@@ -44,15 +44,16 @@
 
 ALL_COMMANDS = ['start', 'stop', 'shutdown', 'restart',
                 'reload', 'force-reload']
-ALL_SERVERS = ['glance-api', 'glance-registry']
-GRACEFUL_SHUTDOWN_SERVERS = ['glance-api', 'glance-registry']
+ALL_SERVERS = ['glance-api', 'glance-registry', 'glance-scrubber']
+GRACEFUL_SHUTDOWN_SERVERS = ['glance-api', 'glance-registry',
+                             'glance-scrubber']
 MAX_DESCRIPTORS = 32768
 MAX_MEMORY = (1024 * 1024 * 1024) * 2  # 2 GB
 USAGE = """%prog [options] <SERVER> <COMMAND> [CONFPATH]
 
 Where <SERVER> is one of:
 
-    all, api, registry
+    all, api, registry, scrubber
 
 And command is one of:
 

=== added file 'bin/glance-scrubber'
--- bin/glance-scrubber	1970-01-01 00:00:00 +0000
+++ bin/glance-scrubber	2011-07-26 10:13:35 +0000
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Glance Scrub Service
+"""
+
+import optparse
+import os
+import sys
+
+# If ../glance/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+from glance import version
+from glance.common import config
+from glance.store import scrubber
+
+
+def create_options(parser):
+    """
+    Sets up the CLI and config-file options that may be
+    parsed and program commands.
+
+    :param parser: The option parser
+    """
+    config.add_common_options(parser)
+    config.add_log_options(parser)
+    parser.add_option("-D", "--daemon", default=False, dest="daemon",
+                      action="store_true",
+                      help="Run as a long-running process. When not "
+                           "specified (the default) run the scrub "
+                           "operation once and then exits. When specified "
+                           "do not exit and run scrub on wakeup_time "
+                           "interval as specified in the config file.")
+
+
+if __name__ == '__main__':
+    oparser = optparse.OptionParser(version='%%prog %s'
+                                    % version.version_string())
+    create_options(oparser)
+    (options, args) = config.parse_options(oparser)
+
+    try:
+        conf, app = config.load_paste_app('glance-scrubber', options, args)
+        daemon = options.get('daemon') or \
+                 config.get_option(conf, 'daemon', type='bool',
+                                   default=False)
+
+        if daemon:
+            wakeup_time = int(conf.get('wakeup_time', 300))
+            server = scrubber.Daemon(wakeup_time)
+            server.start(app)
+            server.wait()
+        else:
+            import eventlet
+            pool = eventlet.greenpool.GreenPool(1000)
+            scrubber = app.run(pool)
+    except RuntimeError, e:
+        sys.exit("ERROR: %s" % e)

=== modified file 'etc/glance-api.conf'
--- etc/glance-api.conf	2011-05-11 23:03:51 +0000
+++ etc/glance-api.conf	2011-07-26 10:13:35 +0000
@@ -51,8 +51,26 @@
 # Do we create the container if it does not exist?
 swift_store_create_container_on_put = False
 
+# ============ Image Cache Options ========================
+
+# Directory that the Image Cache writes data to
+# Make sure this is also set in glance-pruner.conf
+image_cache_datadir = /var/lib/glance/image-cache/
+
+# Number of seconds after which we should consider an incomplete image to be
+# stalled and eligible for reaping
+image_cache_stall_timeout = 86400
+
+# ============ Delayed Delete Options =============================
+
+# Turn on/off delayed delete
+delayed_delete = False
+
 [pipeline:glance-api]
-pipeline = versionnegotiation apiv1app
+pipeline = versionnegotiation context apiv1app
+
+# To enable Image Cache Management API replace pipeline with below:
+# pipeline = versionnegotiation imagecache apiv1app
 
 [pipeline:versions]
 pipeline = versionsapp
@@ -65,3 +83,9 @@
 
 [filter:versionnegotiation]
 paste.filter_factory = glance.api.middleware.version_negotiation:filter_factory
+
+[filter:imagecache]
+paste.filter_factory = glance.api.middleware.image_cache:filter_factory
+
+[filter:context]
+paste.filter_factory = glance.common.context:filter_factory

=== added file 'etc/glance-prefetcher.conf'
--- etc/glance-prefetcher.conf	1970-01-01 00:00:00 +0000
+++ etc/glance-prefetcher.conf	2011-07-26 10:13:35 +0000
@@ -0,0 +1,21 @@
+[DEFAULT]
+# Show more verbose log output (sets INFO log level output)
+verbose = True
+
+# Show debugging output in logs (sets DEBUG log level output)
+debug = False
+
+log_file = /var/log/glance/prefetcher.log
+
+# Directory that the Image Cache writes data to
+# Make sure this is also set in glance-api.conf
+image_cache_datadir = /var/lib/glance/image-cache/
+
+# Address to find the registry server
+registry_host = 0.0.0.0
+
+# Port the registry server is listening on
+registry_port = 9191
+
+[app:glance-prefetcher]
+paste.app_factory = glance.image_cache.prefetcher:app_factory

=== added file 'etc/glance-pruner.conf'
--- etc/glance-pruner.conf	1970-01-01 00:00:00 +0000
+++ etc/glance-pruner.conf	2011-07-26 10:13:35 +0000
@@ -0,0 +1,28 @@
+[DEFAULT]
+# Show more verbose log output (sets INFO log level output)
+verbose = True
+
+# Show debugging output in logs (sets DEBUG log level output)
+debug = False
+
+log_file = /var/log/glance/pruner.log
+
+image_cache_max_size_bytes = 1073741824
+
+# Percentage of the cache that should be freed (in addition to the overage)
+# when the cache is pruned
+#
+# A percentage of 0% means we prune only as many files as needed to remain
+# under the cache's max_size. This is space efficient but will lead to
+# constant pruning as the size bounces just-above and just-below the max_size.
+#
+# To mitigate this 'thrashing', you can specify an additional amount of the
+# cache that should be tossed out on each prune.
+image_cache_percent_extra_to_free = 0.20
+
+# Directory that the Image Cache writes data to
+# Make sure this is also set in glance-api.conf
+image_cache_datadir = /var/lib/glance/image-cache/
+
+[app:glance-pruner]
+paste.app_factory = glance.image_cache.pruner:app_factory

=== added file 'etc/glance-reaper.conf'
--- etc/glance-reaper.conf	1970-01-01 00:00:00 +0000
+++ etc/glance-reaper.conf	2011-07-26 10:13:35 +0000
@@ -0,0 +1,29 @@
+[DEFAULT]
+# Show more verbose log output (sets INFO log level output)
+verbose = True
+
+# Show debugging output in logs (sets DEBUG log level output)
+debug = False
+
+log_file = /var/log/glance/reaper.log
+
+# Directory that the Image Cache writes data to
+# Make sure this is also set in glance-api.conf
+image_cache_datadir = /var/lib/glance/image-cache/
+
+# image_cache_invalid_entry_grace_period - seconds
+#
+# If an exception is raised as we're writing to the cache, the cache-entry is
+# deemed invalid and moved to <image_cache_datadir>/invalid so that it can be
+# inspected for debugging purposes. 
+#
+# This is number of seconds to leave these invalid images around before they
+# are elibible to be reaped.
+image_cache_invalid_entry_grace_period = 3600
+
+# Number of seconds after which we should consider an incomplete image to be
+# stalled and eligible for reaping
+image_cache_stall_timeout = 86400
+
+[app:glance-reaper]
+paste.app_factory = glance.image_cache.reaper:app_factory

=== modified file 'etc/glance-registry.conf'
--- etc/glance-registry.conf	2011-05-09 18:55:46 +0000
+++ etc/glance-registry.conf	2011-07-26 10:13:35 +0000
@@ -29,5 +29,11 @@
 # before MySQL can drop the connection.
 sql_idle_timeout = 3600
 
-[app:glance-registry]
+[pipeline:glance-registry]
+pipeline = context registryapp
+
+[app:registryapp]
 paste.app_factory = glance.registry.server:app_factory
+
+[filter:context]
+paste.filter_factory = glance.common.context:filter_factory

=== added file 'etc/glance-scrubber.conf'
--- etc/glance-scrubber.conf	1970-01-01 00:00:00 +0000
+++ etc/glance-scrubber.conf	2011-07-26 10:13:35 +0000
@@ -0,0 +1,36 @@
+[DEFAULT]
+# Show more verbose log output (sets INFO log level output)
+verbose = True
+
+# Show debugging output in logs (sets DEBUG log level output)
+debug = False
+
+# Log to this file. Make sure you do not set the same log
+# file for both the API and registry servers!
+log_file = /var/log/glance/scrubber.log
+
+# Delayed delete time in seconds
+scrub_time = 43200
+
+# Should we run our own loop or rely on cron/scheduler to run us
+daemon = False
+
+# Loop time between checking the db for new items to schedule for delete
+wakeup_time = 300
+
+# SQLAlchemy connection string for the reference implementation
+# registry server. Any valid SQLAlchemy connection string is fine.
+# See: http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html#sqlalchemy.create_engine
+sql_connection = sqlite:///glance.sqlite
+
+# Period in seconds after which SQLAlchemy should reestablish its connection
+# to the database.
+#
+# MySQL uses a default `wait_timeout` of 8 hours, after which it will drop
+# idle connections. This can result in 'MySQL Gone Away' exceptions. If you
+# notice this, you can lower this value to ensure that SQLAlchemy reconnects
+# before MySQL can drop the connection.
+sql_idle_timeout = 3600
+
+[app:glance-scrubber]
+paste.app_factory = glance.store.scrubber:app_factory

=== modified file 'glance/api/__init__.py'
--- glance/api/__init__.py	2011-05-05 23:12:21 +0000
+++ glance/api/__init__.py	2011-07-26 10:13:35 +0000
@@ -14,3 +14,50 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import logging
+import webob.exc
+
+from glance import registry
+from glance.common import exception
+
+
+logger = logging.getLogger('glance.api')
+
+
+class BaseController(object):
+    def get_image_meta_or_404(self, request, id):
+        """
+        Grabs the image metadata for an image with a supplied
+        identifier or raises an HTTPNotFound (404) response
+
+        :param request: The WSGI/Webob Request object
+        :param id: The opaque image identifier
+
+        :raises HTTPNotFound if image does not exist
+        """
+        context = request.context
+        try:
+            return registry.get_image_metadata(self.options, context, id)
+        except exception.NotFound:
+            msg = "Image with identifier %s not found" % id
+            logger.debug(msg)
+            raise webob.exc.HTTPNotFound(
+                    msg, request=request, content_type='text/plain')
+        except exception.NotAuthorized:
+            msg = "Unauthorized image access"
+            logger.debug(msg)
+            raise webob.exc.HTTPForbidden(msg, request=request,
+                                content_type='text/plain')
+
+    def get_active_image_meta_or_404(self, request, id):
+        """
+        Same as get_image_meta_or_404 except that it will raise a 404 if the
+        image isn't 'active'.
+        """
+        image = self.get_image_meta_or_404(request, id)
+        if image['status'] != 'active':
+            msg = "Image %s is not active" % id
+            logger.debug(msg)
+            raise webob.exc.HTTPNotFound(
+                    msg, request=request, content_type='text/plain')
+        return image

=== added file 'glance/api/cached_images.py'
--- glance/api/cached_images.py	1970-01-01 00:00:00 +0000
+++ glance/api/cached_images.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,105 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Controller for Image Cache Management API
+"""
+
+import httplib
+import json
+
+import webob.dec
+import webob.exc
+
+from glance.common import exception
+from glance.common import wsgi
+from glance import api
+from glance import image_cache
+from glance import registry
+
+
+class Controller(api.BaseController):
+    """
+    A controller that produces information on the Glance API versions.
+    """
+
+    def __init__(self, options):
+        self.options = options
+        self.cache = image_cache.ImageCache(self.options)
+
+    def index(self, req):
+        status = req.str_params.get('status')
+        if status == 'invalid':
+            entries = list(self.cache.invalid_entries())
+        elif status == 'incomplete':
+            entries = list(self.cache.incomplete_entries())
+        elif status == 'prefetching':
+            entries = list(self.cache.prefetch_entries())
+        else:
+            entries = list(self.cache.entries())
+
+        return dict(cached_images=entries)
+
+    def delete(self, req, id):
+        self.cache.purge(id)
+
+    def delete_collection(self, req):
+        """
+        DELETE /cached_images - Clear all active cached images
+        DELETE /cached_images?status=invalid - Reap invalid cached images
+        DELETE /cached_images?status=incomplete - Reap stalled cached images
+        """
+        status = req.str_params.get('status')
+        if status == 'invalid':
+            num_reaped = self.cache.reap_invalid()
+            return dict(num_reaped=num_reaped)
+        elif status == 'incomplete':
+            num_reaped = self.cache.reap_stalled()
+            return dict(num_reaped=num_reaped)
+        else:
+            num_purged = self.cache.clear()
+            return dict(num_purged=num_purged)
+
+    def update(self, req, id):
+        """PUT /cached_images/1 is used to prefetch an image into the cache"""
+        image_meta = self.get_active_image_meta_or_404(req, id)
+        try:
+            self.cache.queue_prefetch(image_meta)
+        except exception.Invalid, e:
+            raise webob.exc.HTTPBadRequest(explanation=str(e))
+
+
+class CachedImageDeserializer(wsgi.JSONRequestDeserializer):
+    pass
+
+
+class CachedImageSerializer(wsgi.JSONResponseSerializer):
+    pass
+
+
+def create_resource(options):
+    """Cached Images resource factory method"""
+    deserializer = CachedImageDeserializer()
+    serializer = CachedImageSerializer()
+    return wsgi.Resource(Controller(options), deserializer, serializer)
+
+
+def app_factory(global_conf, **local_conf):
+    """paste.deploy app factory for creating Cached Images apps"""
+    conf = global_conf.copy()
+    conf.update(local_conf)
+    return Controller(conf)

=== added file 'glance/api/middleware/image_cache.py'
--- glance/api/middleware/image_cache.py	1970-01-01 00:00:00 +0000
+++ glance/api/middleware/image_cache.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,57 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Image Cache Management API
+"""
+
+import logging
+
+from glance.api import cached_images
+from glance.common import wsgi
+
+logger = logging.getLogger('glance.api.middleware.image_cache')
+
+
+class ImageCacheFilter(wsgi.Middleware):
+    def __init__(self, app, options):
+        super(ImageCacheFilter, self).__init__(app)
+
+        map = app.map
+        resource = cached_images.create_resource(options)
+        map.resource("cached_image", "cached_images",
+                     controller=resource,
+                     collection={'reap_invalid': 'POST',
+                                 'reap_stalled': 'POST'})
+
+        map.connect("/cached_images",
+                    controller=resource,
+                    action="delete_collection",
+                    conditions=dict(method=["DELETE"]))
+
+
+def filter_factory(global_conf, **local_conf):
+    """
+    Factory method for paste.deploy
+    """
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def filter(app):
+        return ImageCacheFilter(app, conf)
+
+    return filter

=== modified file 'glance/api/middleware/version_negotiation.py'
--- glance/api/middleware/version_negotiation.py	2011-05-13 22:28:51 +0000
+++ glance/api/middleware/version_negotiation.py	2011-07-26 10:13:35 +0000
@@ -74,7 +74,7 @@
                              req.environ['api.minor_version'])
                 return self.versions_app
 
-        accept = req.headers['Accept']
+        accept = str(req.accept)
         if accept.startswith('application/vnd.openstack.images-'):
             token_loc = len('application/vnd.openstack.images-')
             accept_version = accept[token_loc:]

=== modified file 'glance/api/v1/__init__.py'
--- glance/api/v1/__init__.py	2011-06-27 18:05:29 +0000
+++ glance/api/v1/__init__.py	2011-07-26 10:13:35 +0000
@@ -34,7 +34,7 @@
         mapper = routes.Mapper()
         resource = images.create_resource(options)
         mapper.resource("image", "images", controller=resource,
-                       collection={'detail': 'GET'})
+                        collection={'detail': 'GET'})
         mapper.connect("/", controller=resource, action="index")
         mapper.connect("/images/{id}", controller=resource, action="meta",
                        conditions=dict(method=["HEAD"]))

=== modified file 'glance/api/v1/images.py'
--- glance/api/v1/images.py	2011-06-28 14:03:10 +0000
+++ glance/api/v1/images.py	2011-07-26 10:13:35 +0000
@@ -27,12 +27,15 @@
 import webob
 from webob.exc import (HTTPNotFound,
                        HTTPConflict,
-                       HTTPBadRequest)
+                       HTTPBadRequest,
+                       HTTPForbidden)
 
+from glance import api
+from glance import image_cache
 from glance.common import exception
 from glance.common import wsgi
 from glance.store import (get_from_backend,
-                          delete_from_backend,
+                          schedule_delete_from_backend,
                           get_store_from_location,
                           get_backend_class,
                           UnsupportedBackend)
@@ -43,13 +46,12 @@
 logger = logging.getLogger('glance.api.v1.images')
 
 SUPPORTED_FILTERS = ['name', 'status', 'container_format', 'disk_format',
-                     'size_min', 'size_max']
+                     'size_min', 'size_max', 'is_public']
 
 SUPPORTED_PARAMS = ('limit', 'marker', 'sort_key', 'sort_dir')
 
 
-class Controller(object):
-
+class Controller(api.BaseController):
     """
     WSGI controller for images resource in Glance v1 API
 
@@ -95,7 +97,12 @@
             ]}
         """
         params = self._get_query_params(req)
-        images = registry.get_images_list(self.options, **params)
+        try:
+            images = registry.get_images_list(self.options, req.context,
+                                              **params)
+        except exception.Invalid, e:
+            raise HTTPBadRequest(explanation=str(e))
+
         return dict(images=images)
 
     def detail(self, req):
@@ -121,7 +128,11 @@
             ]}
         """
         params = self._get_query_params(req)
-        images = registry.get_images_detail(self.options, **params)
+        try:
+            images = registry.get_images_detail(self.options, req.context,
+                                                **params)
+        except exception.Invalid, e:
+            raise HTTPBadRequest(explanation=str(e))
         return dict(images=images)
 
     def _get_query_params(self, req):
@@ -176,18 +187,63 @@
 
         :raises HTTPNotFound if image is not available to user
         """
-        image = self.get_image_meta_or_404(req, id)
-
-        def image_iterator():
-            chunks = get_from_backend(image['location'],
-                                      expected_size=image['size'],
-                                      options=self.options)
-
-            for chunk in chunks:
-                yield chunk
+        image = self.get_active_image_meta_or_404(req, id)
+
+        def get_from_store(image):
+            """Called if caching disabled"""
+            return get_from_backend(image['location'],
+                                    expected_size=image['size'],
+                                    options=self.options)
+
+        def get_from_cache(image, cache):
+            """Called if cache hit"""
+            with cache.open(image, "rb") as cache_file:
+                chunks = utils.chunkiter(cache_file)
+                for chunk in chunks:
+                    yield chunk
+
+        def get_from_store_tee_into_cache(image, cache):
+            """Called if cache miss"""
+            with cache.open(image, "wb") as cache_file:
+                chunks = get_from_store(image)
+                for chunk in chunks:
+                    cache_file.write(chunk)
+                    yield chunk
+
+        cache = image_cache.ImageCache(self.options)
+        if cache.enabled:
+            if cache.hit(id):
+                # hit
+                logger.debug("image cache HIT, retrieving image '%s'"
+                             " from cache", id)
+                image_iterator = get_from_cache(image, cache)
+            else:
+                # miss
+                logger.debug("image cache MISS, retrieving image '%s'"
+                             " from store and tee'ing into cache", id)
+
+                # We only want to tee-into the cache if we're not currently
+                # prefetching an image
+                image_id = image['id']
+                if cache.is_image_currently_prefetching(image_id):
+                    image_iterator = get_from_store(image)
+                else:
+                    # NOTE(sirp): If we're about to download and cache an
+                    # image which is currently in the prefetch queue, just
+                    # delete the queue items since we're caching it anyway
+                    if cache.is_image_queued_for_prefetch(image_id):
+                        cache.delete_queued_prefetch_image(image_id)
+
+                    image_iterator = get_from_store_tee_into_cache(
+                        image, cache)
+        else:
+            # disabled
+            logger.debug("image cache DISABLED, retrieving image '%s'"
+                         " from store", id)
+            image_iterator = get_from_store(image)
 
         return {
-            'image_iterator': image_iterator(),
+            'image_iterator': image_iterator,
             'image_meta': image,
         }
 
@@ -219,6 +275,7 @@
 
         try:
             image_meta = registry.add_image_metadata(self.options,
+                                                     req.context,
                                                      image_meta)
             return image_meta
         except exception.Duplicate:
@@ -231,6 +288,11 @@
             for line in msg.split('\n'):
                 logger.error(line)
             raise HTTPBadRequest(msg, request=req, content_type="text/plain")
+        except exception.NotAuthorized:
+            msg = "Not authorized to reserve image."
+            logger.error(msg)
+            raise HTTPForbidden(msg, request=req,
+                                content_type="text/plain")
 
     def _upload(self, req, image_meta):
         """
@@ -259,12 +321,12 @@
         store = self.get_store_or_400(req, store_name)
 
         image_id = image_meta['id']
-        logger.debug("Setting image %s to status 'saving'" % image_id)
-        registry.update_image_metadata(self.options, image_id,
+        logger.debug("Setting image %s to status 'saving'", image_id)
+        registry.update_image_metadata(self.options, req.context, image_id,
                                        {'status': 'saving'})
         try:
             logger.debug("Uploading image data for image %(image_id)s "
-                         "to %(store_name)s store" % locals())
+                         "to %(store_name)s store", locals())
             location, size, checksum = store.add(image_meta['id'],
                                                  req.body_file,
                                                  self.options)
@@ -286,8 +348,9 @@
             # from the backend store
             logger.debug("Updating image %(image_id)s data. "
                          "Checksum set to %(checksum)s, size set "
-                         "to %(size)d" % locals())
-            registry.update_image_metadata(self.options, image_id,
+                         "to %(size)d", locals())
+            registry.update_image_metadata(self.options, req.context,
+                                           image_id,
                                            {'checksum': checksum,
                                             'size': size})
 
@@ -299,6 +362,13 @@
             self._safe_kill(req, image_id)
             raise HTTPConflict(msg, request=req)
 
+        except exception.NotAuthorized, e:
+            msg = ("Unauthorized upload attempt: %s") % str(e)
+            logger.error(msg)
+            self._safe_kill(req, image_id)
+            raise HTTPForbidden(msg, request=req,
+                                content_type='text/plain')
+
         except Exception, e:
             msg = ("Error uploading image: %s") % str(e)
             logger.error(msg)
@@ -318,6 +388,7 @@
         image_meta['location'] = location
         image_meta['status'] = 'active'
         return registry.update_image_metadata(self.options,
+                                       req.context,
                                        image_id,
                                        image_meta)
 
@@ -329,6 +400,7 @@
         :param image_id: Opaque image identifier
         """
         registry.update_image_metadata(self.options,
+                                       req.context,
                                        image_id,
                                        {'status': 'killed'})
 
@@ -397,6 +469,12 @@
                 and the request body is not application/octet-stream
                 image data.
         """
+        if req.context.read_only:
+            msg = "Read-only access"
+            logger.debug(msg)
+            raise HTTPForbidden(msg, request=req,
+                                content_type="text/plain")
+
         image_meta = self._reserve(req, image_meta)
         image_id = image_meta['id']
 
@@ -418,6 +496,12 @@
 
         :retval Returns the updated image information as a mapping
         """
+        if req.context.read_only:
+            msg = "Read-only access"
+            logger.debug(msg)
+            raise HTTPForbidden(msg, request=req,
+                                content_type="text/plain")
+
         orig_image_meta = self.get_image_meta_or_404(req, id)
         orig_status = orig_image_meta['status']
 
@@ -425,8 +509,9 @@
             raise HTTPConflict("Cannot upload to an unqueued image")
 
         try:
-            image_meta = registry.update_image_metadata(self.options, id,
-                                                         image_meta, True)
+            image_meta = registry.update_image_metadata(self.options,
+                                                        req.context, id,
+                                                        image_meta, True)
             if image_data is not None:
                 image_meta = self._upload_and_activate(req, image_meta)
         except exception.Invalid, e:
@@ -450,6 +535,12 @@
         :raises HttpNotAuthorized if image or any chunk is not
                 deleteable by the requesting user
         """
+        if req.context.read_only:
+            msg = "Read-only access"
+            logger.debug(msg)
+            raise HTTPForbidden(msg, request=req,
+                                content_type="text/plain")
+
         image = self.get_image_meta_or_404(req, id)
 
         # The image's location field may be None in the case
@@ -457,32 +548,9 @@
         # to delete the image if the backend doesn't yet store it.
         # See https://bugs.launchpad.net/glance/+bug/747799
         if image['location']:
-            try:
-                delete_from_backend(image['location'])
-            except (UnsupportedBackend, exception.NotFound):
-                msg = "Failed to delete image from store (%s). " + \
-                      "Continuing with deletion from registry."
-                logger.error(msg % (image['location'],))
-
-        registry.delete_image_metadata(self.options, id)
-
-    def get_image_meta_or_404(self, request, id):
-        """
-        Grabs the image metadata for an image with a supplied
-        identifier or raises an HTTPNotFound (404) response
-
-        :param request: The WSGI/Webob Request object
-        :param id: The opaque image identifier
-
-        :raises HTTPNotFound if image does not exist
-        """
-        try:
-            return registry.get_image_metadata(self.options, id)
-        except exception.NotFound:
-            msg = "Image with identifier %s not found" % id
-            logger.debug(msg)
-            raise HTTPNotFound(msg, request=request,
-                               content_type='text/plain')
+            schedule_delete_from_backend(image['location'], self.options,
+                                         req.context, id)
+        registry.delete_image_metadata(self.options, req.context, id)
 
     def get_store_or_400(self, request, store_name):
         """

=== modified file 'glance/client.py'
--- glance/client.py	2011-06-28 23:20:51 +0000
+++ glance/client.py	2011-07-26 10:13:35 +0000
@@ -35,7 +35,8 @@
 
     DEFAULT_PORT = 9292
 
-    def __init__(self, host, port=None, use_ssl=False, doc_root="/v1"):
+    def __init__(self, host, port=None, use_ssl=False, doc_root="/v1",
+                 auth_tok=None):
         """
         Creates a new client to a Glance API service.
 
@@ -43,11 +44,11 @@
         :param port: The port where Glance resides (defaults to 9292)
         :param use_ssl: Should we use HTTPS? (defaults to False)
         :param doc_root: Prefix for all URLs we request from host
+        :param auth_tok: The auth token to pass to the server
         """
-
         port = port or self.DEFAULT_PORT
         self.doc_root = doc_root
-        super(Client, self).__init__(host, port, use_ssl)
+        super(Client, self).__init__(host, port, use_ssl, auth_tok)
 
     def do_request(self, method, action, body=None, headers=None, params=None):
         action = "%s/%s" % (self.doc_root, action.lstrip("/"))
@@ -81,7 +82,6 @@
         :param sort_key: results will be ordered by this image attribute
         :param sort_dir: direction in which to to order results (asc, desc)
         """
-
         params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS)
         res = self.do_request("GET", "/images/detail", params=params)
         data = json.loads(res.read())['images']
@@ -126,7 +126,6 @@
 
         :retval The newly-stored image's metadata.
         """
-
         headers = utils.image_meta_to_http_headers(image_meta or {})
 
         if image_data:
@@ -165,5 +164,120 @@
         self.do_request("DELETE", "/images/%s" % image_id)
         return True
 
+    def get_cached_images(self, **kwargs):
+        """
+        Returns a list of images stored in the image cache.
+
+        :param filters: dictionary of attributes by which the resulting
+                        collection of images should be filtered
+        :param marker: id after which to start the page of images
+        :param limit: maximum number of items to return
+        :param sort_key: results will be ordered by this image attribute
+        :param sort_dir: direction in which to to order results (asc, desc)
+        """
+        params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS)
+        res = self.do_request("GET", "/cached_images", params=params)
+        data = json.loads(res.read())['cached_images']
+        return data
+
+    def get_invalid_cached_images(self, **kwargs):
+        """
+        Returns a list of invalid images stored in the image cache.
+
+        :param filters: dictionary of attributes by which the resulting
+                        collection of images should be filtered
+        :param marker: id after which to start the page of images
+        :param limit: maximum number of items to return
+        :param sort_key: results will be ordered by this image attribute
+        :param sort_dir: direction in which to to order results (asc, desc)
+        """
+        params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS)
+        params['status'] = 'invalid'
+        res = self.do_request("GET", "/cached_images", params=params)
+        data = json.loads(res.read())['cached_images']
+        return data
+
+    def get_incomplete_cached_images(self, **kwargs):
+        """
+        Returns a list of incomplete images being fetched into cache
+
+        :param filters: dictionary of attributes by which the resulting
+                        collection of images should be filtered
+        :param marker: id after which to start the page of images
+        :param limit: maximum number of items to return
+        :param sort_key: results will be ordered by this image attribute
+        :param sort_dir: direction in which to to order results (asc, desc)
+        """
+        params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS)
+        params['status'] = 'incomplete'
+        res = self.do_request("GET", "/cached_images", params=params)
+        data = json.loads(res.read())['cached_images']
+        return data
+
+    def purge_cached_image(self, image_id):
+        """
+        Delete a specified image from the cache
+        """
+        self.do_request("DELETE", "/cached_images/%s" % image_id)
+        return True
+
+    def clear_cached_images(self):
+        """
+        Clear all cached images
+        """
+        res = self.do_request("DELETE", "/cached_images")
+        data = json.loads(res.read())
+        num_purged = data['num_purged']
+        return num_purged
+
+    def reap_invalid_cached_images(self, **kwargs):
+        """
+        Reaps any invalid cached images
+        """
+        params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS)
+        params['status'] = 'invalid'
+        res = self.do_request("DELETE", "/cached_images", params=params)
+        data = json.loads(res.read())
+        num_reaped = data['num_reaped']
+        return num_reaped
+
+    def reap_stalled_cached_images(self, **kwargs):
+        """
+        Reaps any stalled cached images
+        """
+        params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS)
+        params['status'] = 'incomplete'
+        res = self.do_request("DELETE", "/cached_images", params=params)
+        data = json.loads(res.read())
+        num_reaped = data['num_reaped']
+        return num_reaped
+
+    def prefetch_cache_image(self, image_id):
+        """
+        Pre-fetch a specified image from the cache
+        """
+        res = self.do_request("HEAD", "/images/%s" % image_id)
+        image = utils.get_image_meta_from_headers(res)
+        self.do_request("PUT", "/cached_images/%s" % image_id)
+        return True
+
+    def get_prefetching_cache_images(self, **kwargs):
+        """
+        Returns a list of images which are actively being prefetched or are
+        queued to be prefetched in the future.
+
+        :param filters: dictionary of attributes by which the resulting
+                        collection of images should be filtered
+        :param marker: id after which to start the page of images
+        :param limit: maximum number of items to return
+        :param sort_key: results will be ordered by this image attribute
+        :param sort_dir: direction in which to to order results (asc, desc)
+        """
+        params = self._extract_params(kwargs, v1_images.SUPPORTED_PARAMS)
+        params['status'] = 'prefetching'
+        res = self.do_request("GET", "/cached_images", params=params)
+        data = json.loads(res.read())['cached_images']
+        return data
+
 
 Client = V1Client

=== modified file 'glance/common/client.py'
--- glance/common/client.py	2011-06-29 13:07:16 +0000
+++ glance/common/client.py	2011-07-26 10:13:35 +0000
@@ -41,17 +41,19 @@
 
     CHUNKSIZE = 65536
 
-    def __init__(self, host, port, use_ssl):
+    def __init__(self, host, port, use_ssl, auth_tok):
         """
         Creates a new client to some service.
 
         :param host: The host where service resides
         :param port: The port where service resides
         :param use_ssl: Should we use HTTPS?
+        :param auth_tok: The auth token to pass to the server
         """
         self.host = host
         self.port = port
         self.use_ssl = use_ssl
+        self.auth_tok = auth_tok
         self.connection = None
 
     def get_connection_type(self):
@@ -99,6 +101,8 @@
         try:
             connection_type = self.get_connection_type()
             headers = headers or {}
+            if 'x-auth-token' not in headers and self.auth_tok:
+                headers['x-auth-token'] = self.auth_tok
             c = connection_type(self.host, self.port)
 
             # Do a simple request or a chunked request, depending

=== added file 'glance/common/context.py'
--- glance/common/context.py	1970-01-01 00:00:00 +0000
+++ glance/common/context.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,97 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from glance.common import utils
+from glance.common import wsgi
+
+
+class RequestContext(object):
+    """
+    Stores information about the security context under which the user
+    accesses the system, as well as additional request information.
+    """
+
+    def __init__(self, auth_tok=None, user=None, tenant=None, is_admin=False,
+                 read_only=False, show_deleted=False):
+        self.auth_tok = auth_tok
+        self.user = user
+        self.tenant = tenant
+        self.is_admin = is_admin
+        self.read_only = read_only
+        self.show_deleted = show_deleted
+
+    def is_image_visible(self, image):
+        """Return True if the image is visible in this context."""
+        # Is admin == image visible
+        if self.is_admin:
+            return True
+
+        # No owner == image visible
+        if image.owner is None:
+            return True
+
+        # Image is_public == image visible
+        if image.is_public:
+            return True
+
+        # Private image
+        return self.owner is not None and self.owner == image.owner
+
+    @property
+    def owner(self):
+        """Return the owner to correlate with an image."""
+        return self.tenant
+
+
+class ContextMiddleware(wsgi.Middleware):
+    def __init__(self, app, options):
+        self.options = options
+        super(ContextMiddleware, self).__init__(app)
+
+    def make_context(self, *args, **kwargs):
+        """
+        Create a context with the given arguments.
+        """
+
+        # Determine the context class to use
+        ctxcls = RequestContext
+        if 'context_class' in self.options:
+            ctxcls = utils.import_class(self.options['context_class'])
+
+        return ctxcls(*args, **kwargs)
+
+    def process_request(self, req):
+        """
+        Extract any authentication information in the request and
+        construct an appropriate context from it.
+        """
+        # Use the default empty context, with admin turned on for
+        # backwards compatibility
+        req.context = self.make_context(is_admin=True)
+
+
+def filter_factory(global_conf, **local_conf):
+    """
+    Factory method for paste.deploy
+    """
+    conf = global_conf.copy()
+    conf.update(local_conf)
+
+    def filter(app):
+        return ContextMiddleware(app, conf)
+
+    return filter

=== modified file 'glance/common/exception.py'
--- glance/common/exception.py	2011-06-26 20:43:41 +0000
+++ glance/common/exception.py	2011-07-26 10:13:35 +0000
@@ -54,6 +54,24 @@
     pass
 
 
+class UnknownScheme(Error):
+
+    msg = "Unknown scheme '%s' found in URI"
+
+    def __init__(self, scheme):
+        msg = self.__class__.msg % scheme
+        super(UnknownScheme, self).__init__(msg)
+
+
+class BadStoreUri(Error):
+
+    msg = "The Store URI %s was malformed. Reason: %s"
+
+    def __init__(self, uri, reason):
+        msg = self.__class__.msg % (uri, reason)
+        super(BadStoreUri, self).__init__(msg)
+
+
 class Duplicate(Error):
     pass
 

=== added directory 'glance/image_cache'
=== added file 'glance/image_cache/__init__.py'
--- glance/image_cache/__init__.py	1970-01-01 00:00:00 +0000
+++ glance/image_cache/__init__.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,451 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+LRU Cache for Image Data
+"""
+from contextlib import contextmanager
+import datetime
+import itertools
+import logging
+import os
+import sys
+import time
+
+from glance.common import config
+from glance.common import exception
+from glance import utils
+
+logger = logging.getLogger('glance.image_cache')
+
+
+class ImageCache(object):
+    """Provides an LRU cache for image data.
+
+    Data is cached on READ not on WRITE; meaning if the cache is enabled, we
+    attempt to read from the cache first, if we don't find the data, we begin
+    streaming the data from the 'store' while simultaneously tee'ing the data
+    into the cache. Subsequent reads will generate cache HITs for this image.
+
+    Assumptions
+    ===========
+
+        1. Cache data directory exists on a filesytem that updates atime on
+           reads ('noatime' should NOT be set)
+
+        2. Cache data directory exists on a filesystem that supports xattrs.
+           This is optional, but highly recommended since it allows us to
+           present ops with useful information pertaining to the cache, like
+           human readable filenames and statistics.
+
+        3. `glance-prune` is scheduled to run as a periodic job via cron. This
+            is needed to run the LRU prune strategy to keep the cache size
+            within the limits set by the config file.
+
+
+    Cache Directory Notes
+    =====================
+
+    The image cache data directory contains the main cache path, where the
+    active cache entries and subdirectories for handling partial downloads
+    and errored-out cache images.
+
+    The layout looks like:
+
+        image-cache/
+            entry1
+            entry2
+            ...
+            incomplete/
+            invalid/
+            prefetch/
+            prefetching/
+    """
+    def __init__(self, options):
+        self.options = options
+        self._make_cache_directory_if_needed()
+
+    def _make_cache_directory_if_needed(self):
+        """Creates main cache directory along with incomplete subdirectory"""
+        if not self.enabled:
+            return
+
+        # NOTE(sirp): making the incomplete_path will have the effect of
+        # creating the main cache path directory as well
+        paths = [self.incomplete_path, self.invalid_path, self.prefetch_path,
+                 self.prefetching_path]
+
+        for path in paths:
+            if os.path.exists(path):
+                continue
+            logger.info("image cache directory doesn't exist, creating '%s'",
+                        path)
+            os.makedirs(path)
+
+    @property
+    def enabled(self):
+        return self.options.get('image_cache_enabled', False)
+
+    @property
+    def path(self):
+        """This is the base path for the image cache"""
+        datadir = self.options['image_cache_datadir']
+        return datadir
+
+    @property
+    def incomplete_path(self):
+        """This provides a temporary place to write our cache entries so that
+        we we're not storing incomplete objects in the cache directly.
+
+        When the file is finished writing to, it is moved from the incomplete
+        path back out into the main cache directory.
+
+        The incomplete_path is a subdirectory of the main cache path to ensure
+        that they both reside on the same filesystem and thus making moves
+        cheap.
+        """
+        return os.path.join(self.path, 'incomplete')
+
+    @property
+    def invalid_path(self):
+        """Place to move corrupted images
+
+        If an exception is raised while we're writing an image to the
+        incomplete_path, we move the incomplete image to here.
+        """
+        return os.path.join(self.path, 'invalid')
+
+    @property
+    def prefetch_path(self):
+        """This contains a list of image ids that should be pre-fetched into
+        the cache
+        """
+        return os.path.join(self.path, 'prefetch')
+
+    @property
+    def prefetching_path(self):
+        """This contains image ids that currently being prefetched"""
+        return os.path.join(self.path, 'prefetching')
+
+    def path_for_image(self, image_id):
+        """This crafts an absolute path to a specific entry"""
+        return os.path.join(self.path, str(image_id))
+
+    def incomplete_path_for_image(self, image_id):
+        """This crafts an absolute path to a specific entry in the incomplete
+        directory
+        """
+        return os.path.join(self.incomplete_path, str(image_id))
+
+    def invalid_path_for_image(self, image_id):
+        """This crafts an absolute path to a specific entry in the invalid
+        directory
+        """
+        return os.path.join(self.invalid_path, str(image_id))
+
+    @contextmanager
+    def open(self, image_meta, mode="rb"):
+        """Open a cache image for reading or writing.
+
+        We have two possible scenarios:
+
+            1. READ: we should attempt to read the file from the cache's
+               main directory
+
+            2. WRITE: we should write to a file under the cache's incomplete
+               directory, and when it's finished, move it out the main cache
+               directory.
+        """
+        if mode == 'wb':
+            with self._open_write(image_meta, mode) as cache_file:
+                yield cache_file
+        elif mode == 'rb':
+            with self._open_read(image_meta, mode) as cache_file:
+                yield cache_file
+        else:
+            # NOTE(sirp): `rw` and `a' modes are not supported since image
+            # data is immutable, we `wb` it once, then `rb` multiple times.
+            raise Exception("mode '%s' not supported" % mode)
+
+    @contextmanager
+    def _open_write(self, image_meta, mode):
+        image_id = image_meta['id']
+        incomplete_path = self.incomplete_path_for_image(image_id)
+
+        def set_xattr(key, value):
+            utils.set_xattr(incomplete_path, key, value)
+
+        def commit():
+            set_xattr('image_name', image_meta['name'])
+            set_xattr('hits', 0)
+
+            final_path = self.path_for_image(image_id)
+            logger.debug("fetch finished, commiting by moving "
+                         "'%(incomplete_path)s' to '%(final_path)s'",
+                         dict(incomplete_path=incomplete_path,
+                              final_path=final_path))
+            os.rename(incomplete_path, final_path)
+
+        def rollback(e):
+            set_xattr('image_name', image_meta['name'])
+            set_xattr('error', str(e))
+
+            invalid_path = self.invalid_path_for_image(image_id)
+            logger.debug("fetch errored, rolling back by moving "
+                         "'%(incomplete_path)s' to '%(invalid_path)s'",
+                         dict(incomplete_path=incomplete_path,
+                              invalid_path=invalid_path))
+            os.rename(incomplete_path, invalid_path)
+
+        try:
+            with open(incomplete_path, mode) as cache_file:
+                set_xattr('expected_size', image_meta['size'])
+                yield cache_file
+        except Exception as e:
+            rollback(e)
+            raise
+        else:
+            commit()
+
+    @contextmanager
+    def _open_read(self, image_meta, mode):
+        image_id = image_meta['id']
+        path = self.path_for_image(image_id)
+        with open(path, mode) as cache_file:
+            yield cache_file
+
+        utils.inc_xattr(path, 'hits')  # bump the hit count
+
+    def hit(self, image_id):
+        return os.path.exists(self.path_for_image(image_id))
+
+    @staticmethod
+    def _delete_file(path):
+        if os.path.exists(path):
+            logger.debug("deleting image cache file '%s'", path)
+            os.unlink(path)
+        else:
+            logger.warn("image cache file '%s' doesn't exist, unable to"
+                        " delete", path)
+
+    def purge(self, image_id):
+        path = self.path_for_image(image_id)
+        self._delete_file(path)
+
+    def clear(self):
+        purged = 0
+        for path in self.get_all_regular_files(self.path):
+            self._delete_file(path)
+            purged += 1
+        return purged
+
+    def is_image_currently_being_written(self, image_id):
+        """Returns true if we're currently downloading an image"""
+        incomplete_path = self.incomplete_path_for_image(image_id)
+        return os.path.exists(incomplete_path)
+
+    def is_currently_prefetching_any_images(self):
+        """True if we are currently prefetching an image.
+
+        We only allow one prefetch to occur at a time.
+        """
+        return len(os.listdir(self.prefetching_path)) > 0
+
+    def is_image_queued_for_prefetch(self, image_id):
+        prefetch_path = os.path.join(self.prefetch_path, str(image_id))
+        return os.path.exists(prefetch_path)
+
+    def is_image_currently_prefetching(self, image_id):
+        prefetching_path = os.path.join(self.prefetching_path, str(image_id))
+        return os.path.exists(prefetching_path)
+
+    def queue_prefetch(self, image_meta):
+        """This adds a image to be prefetched to the queue directory.
+
+        If the image already exists in the queue directory or the
+        prefetching directory, we ignore it.
+        """
+        image_id = image_meta['id']
+
+        if self.hit(image_id):
+            msg = "Skipping prefetch, image '%s' already cached" % image_id
+            logger.warn(msg)
+            raise exception.Invalid(msg)
+
+        if self.is_image_currently_prefetching(image_id):
+            msg = "Skipping prefetch, already prefetching image '%s'"\
+                  % image_id
+            logger.warn(msg)
+            raise exception.Invalid(msg)
+
+        if self.is_image_queued_for_prefetch(image_id):
+            msg = "Skipping prefetch, image '%s' already queued for"\
+                  " prefetching" % image_id
+            logger.warn(msg)
+            raise exception.Invalid(msg)
+
+        prefetch_path = os.path.join(self.prefetch_path, str(image_id))
+
+        # Touch the file to add it to the queue
+        with open(prefetch_path, "w") as f:
+            pass
+
+        utils.set_xattr(prefetch_path, 'image_name', image_meta['name'])
+
+    def delete_queued_prefetch_image(self, image_id):
+        prefetch_path = os.path.join(self.prefetch_path, str(image_id))
+        self._delete_file(prefetch_path)
+
+    def delete_prefetching_image(self, image_id):
+        prefetching_path = os.path.join(self.prefetching_path, str(image_id))
+        self._delete_file(prefetching_path)
+
+    def pop_prefetch_item(self):
+        """This returns the next prefetch job.
+
+        The prefetch directory is treated like a FIFO; so we sort by modified
+        time and pick the oldest.
+        """
+        items = []
+        for path in self.get_all_regular_files(self.prefetch_path):
+            mtime = os.path.getmtime(path)
+            items.append((mtime, path))
+
+        if not items:
+            raise IndexError
+
+        # Sort oldest files to the end of the list
+        items.sort(reverse=True)
+
+        mtime, path = items.pop()
+        image_id = os.path.basename(path)
+        return image_id
+
+    def do_prefetch(self, image_id):
+        """This moves the file from the prefetch queue path to the in-progress
+        prefetching path (so we don't try to prefetch something twice).
+        """
+        prefetch_path = os.path.join(self.prefetch_path, str(image_id))
+        prefetching_path = os.path.join(self.prefetching_path, str(image_id))
+        os.rename(prefetch_path, prefetching_path)
+
+    @staticmethod
+    def get_all_regular_files(basepath):
+        for fname in os.listdir(basepath):
+            path = os.path.join(basepath, fname)
+            if os.path.isfile(path):
+                yield path
+
+    def _base_entries(self, basepath):
+        def iso8601_from_timestamp(timestamp):
+            return datetime.datetime.utcfromtimestamp(timestamp)\
+                                    .isoformat()
+
+        for path in self.get_all_regular_files(basepath):
+            filename = os.path.basename(path)
+            try:
+                image_id = int(filename)
+            except ValueError, TypeError:
+                continue
+
+            entry = {}
+            entry['id'] = image_id
+            entry['path'] = path
+            entry['name'] = utils.get_xattr(path, 'image_name',
+                                            default='UNKNOWN')
+
+            mtime = os.path.getmtime(path)
+            entry['last_modified'] = iso8601_from_timestamp(mtime)
+
+            atime = os.path.getatime(path)
+            entry['last_accessed'] = iso8601_from_timestamp(atime)
+
+            entry['size'] = os.path.getsize(path)
+
+            entry['expected_size'] = utils.get_xattr(
+                    path, 'expected_size', default='UNKNOWN')
+
+            yield entry
+
+    def invalid_entries(self):
+        """Cache info for invalid cached images"""
+        for entry in self._base_entries(self.invalid_path):
+            path = entry['path']
+            entry['error'] = utils.get_xattr(path, 'error', default='UNKNOWN')
+            yield entry
+
+    def incomplete_entries(self):
+        """Cache info for invalid cached images"""
+        for entry in self._base_entries(self.incomplete_path):
+            yield entry
+
+    def prefetch_entries(self):
+        """Cache info for both queued and in-progress prefetch jobs"""
+        both_entries = itertools.chain(
+                        self._base_entries(self.prefetch_path),
+                        self._base_entries(self.prefetching_path))
+
+        for entry in both_entries:
+            path = entry['path']
+            entry['status'] = 'in-progress' if 'prefetching' in path\
+                                            else 'queued'
+            yield entry
+
+    def entries(self):
+        """Cache info for currently cached images"""
+        for entry in self._base_entries(self.path):
+            path = entry['path']
+            entry['hits'] = utils.get_xattr(path, 'hits', default='UNKNOWN')
+            yield entry
+
+    def _reap_old_files(self, dirpath, entry_type, grace=None):
+        """
+        """
+        now = time.time()
+        reaped = 0
+        for path in self.get_all_regular_files(dirpath):
+            mtime = os.path.getmtime(path)
+            age = now - mtime
+            if not grace:
+                logger.debug("No grace period, reaping '%(path)s'"
+                             " immediately", locals())
+                self._delete_file(path)
+                reaped += 1
+            elif age > grace:
+                logger.debug("Cache entry '%(path)s' exceeds grace period, "
+                             "(%(age)i s > %(grace)i s)", locals())
+                self._delete_file(path)
+                reaped += 1
+
+        logger.info("Reaped %(reaped)s %(entry_type)s cache entries",
+                    locals())
+        return reaped
+
+    def reap_invalid(self, grace=None):
+        """Remove any invalid cache entries
+
+        :param grace: Number of seconds to keep an invalid entry around for
+                      debugging purposes. If None, then delete immediately.
+        """
+        return self._reap_old_files(self.invalid_path, 'invalid', grace=grace)
+
+    def reap_stalled(self):
+        """Remove any stalled cache entries"""
+        stall_timeout = int(self.options.get('image_cache_stall_timeout',
+                            86400))
+        return self._reap_old_files(self.incomplete_path, 'stalled',
+                                    grace=stall_timeout)

=== added file 'glance/image_cache/prefetcher.py'
--- glance/image_cache/prefetcher.py	1970-01-01 00:00:00 +0000
+++ glance/image_cache/prefetcher.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,90 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Prefetches images into the Image Cache
+"""
+import logging
+import os
+import stat
+import time
+
+from glance.common import config
+from glance.common import context
+from glance.image_cache import ImageCache
+from glance import registry
+from glance.store import get_from_backend
+
+
+logger = logging.getLogger('glance.image_cache.prefetcher')
+
+
+class Prefetcher(object):
+    def __init__(self, options):
+        self.options = options
+        self.cache = ImageCache(options)
+
+    def fetch_image_into_cache(self, image_id):
+        ctx = context.RequestContext(is_admin=True, show_deleted=True)
+        image_meta = registry.get_image_metadata(
+                    self.options, ctx, image_id)
+        with self.cache.open(image_meta, "wb") as cache_file:
+            chunks = get_from_backend(image_meta['location'],
+                                      expected_size=image_meta['size'],
+                                      options=self.options)
+            for chunk in chunks:
+                cache_file.write(chunk)
+
+    def run(self):
+        if self.cache.is_currently_prefetching_any_images():
+            logger.debug("Currently prefetching, going back to sleep...")
+            return
+
+        try:
+            image_id = self.cache.pop_prefetch_item()
+        except IndexError:
+            logger.debug("Nothing to prefetch, going back to sleep...")
+            return
+
+        if self.cache.hit(image_id):
+            logger.warn("Image %s is already in the cache, deleting "
+                        "prefetch job and going back to sleep...", image_id)
+            self.cache.delete_queued_prefetch_image(image_id)
+            return
+
+        # NOTE(sirp): if someone is already downloading an image that is in
+        # the prefetch queue, then go ahead and delete that item and try to
+        # prefetch another
+        if self.cache.is_image_currently_being_written(image_id):
+            logger.warn("Image %s is already being cached, deleting "
+                        "prefetch job and going back to sleep...", image_id)
+            self.cache.delete_queued_prefetch_image(image_id)
+            return
+
+        logger.debug("Prefetching '%s'", image_id)
+        self.cache.do_prefetch(image_id)
+
+        try:
+            self.fetch_image_into_cache(image_id)
+        finally:
+            self.cache.delete_prefetching_image(image_id)
+
+
+def app_factory(global_config, **local_conf):
+    conf = global_config.copy()
+    conf.update(local_conf)
+    return Prefetcher(conf)

=== added file 'glance/image_cache/pruner.py'
--- glance/image_cache/pruner.py	1970-01-01 00:00:00 +0000
+++ glance/image_cache/pruner.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,115 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Prunes the Image Cache
+"""
+import logging
+import os
+import stat
+import time
+
+from glance.common import config
+from glance.image_cache import ImageCache
+
+logger = logging.getLogger('glance.image_cache.pruner')
+
+
+class Pruner(object):
+    def __init__(self, options):
+        self.options = options
+        self.cache = ImageCache(options)
+
+    @property
+    def max_size(self):
+        default = 1 * 1024 * 1024 * 1024  # 1 GB
+        return config.get_option(
+            self.options, 'image_cache_max_size_bytes',
+            type='int', default=default)
+
+    @property
+    def percent_extra_to_free(self):
+        return config.get_option(
+            self.options, 'image_cache_percent_extra_to_free',
+            type='float', default=0.05)
+
+    def run(self):
+        self.prune_cache()
+
+    def prune_cache(self):
+        """Prune the cache using an LRU strategy"""
+
+        # NOTE(sirp): 'Recency' is determined via the filesystem, first using
+        # atime (access time) and falling back to mtime (modified time).
+        #
+        # It has become more common to disable access-time updates by setting
+        # the `noatime` option for the filesystem. `noatime` is NOT compatible
+        # with this method.
+        #
+        # If `noatime` needs to be supported, we will need to persist access
+        # times elsewhere (either as a separate file, in the DB, or as
+        # an xattr).
+        def get_stats():
+            stats = []
+            for path in self.cache.get_all_regular_files(self.cache.path):
+                file_info = os.stat(path)
+                stats.append((file_info[stat.ST_ATIME],  # access time
+                              file_info[stat.ST_MTIME],  # modification time
+                              file_info[stat.ST_SIZE],   # size in bytes
+                              path))                     # absolute path
+            return stats
+
+        def prune_lru(stats, to_free):
+            # Sort older access and modified times to the back
+            stats.sort(reverse=True)
+
+            freed = 0
+            while to_free > 0:
+                atime, mtime, size, path = stats.pop()
+                logger.debug("deleting '%(path)s' to free %(size)d B",
+                             locals())
+                os.unlink(path)
+                to_free -= size
+                freed += size
+
+            return freed
+
+        stats = get_stats()
+
+        # Check for overage
+        cur_size = sum(s[2] for s in stats)
+        max_size = self.max_size
+        logger.debug("cur_size=%(cur_size)d B max_size=%(max_size)d B",
+                     locals())
+        if cur_size <= max_size:
+            logger.debug("cache has free space, skipping prune...")
+            return
+
+        overage = cur_size - max_size
+        extra = max_size * self.percent_extra_to_free
+        to_free = overage + extra
+        logger.debug("overage=%(overage)d B extra=%(extra)d B"
+                     " total=%(to_free)d B", locals())
+
+        freed = prune_lru(stats, to_free)
+        logger.debug("finished pruning, freed %(freed)d bytes", locals())
+
+
+def app_factory(global_config, **local_conf):
+    conf = global_config.copy()
+    conf.update(local_conf)
+    return Pruner(conf)

=== added file 'glance/image_cache/reaper.py'
--- glance/image_cache/reaper.py	1970-01-01 00:00:00 +0000
+++ glance/image_cache/reaper.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,45 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Reaps any invalid cache entries that exceed the grace period
+"""
+import logging
+
+from glance.image_cache import ImageCache
+
+
+logger = logging.getLogger('glance.image_cache.reaper')
+
+
+class Reaper(object):
+    def __init__(self, options):
+        self.options = options
+        self.cache = ImageCache(options)
+
+    def run(self):
+        invalid_grace = int(self.options.get(
+                            'image_cache_invalid_entry_grace_period',
+                            3600))
+        self.cache.reap_invalid(grace=invalid_grace)
+        self.cache.reap_stalled()
+
+
+def app_factory(global_config, **local_conf):
+    conf = global_config.copy()
+    conf.update(local_conf)
+    return Reaper(conf)

=== modified file 'glance/registry/__init__.py'
--- glance/registry/__init__.py	2011-05-16 14:43:33 +0000
+++ glance/registry/__init__.py	2011-07-26 10:13:35 +0000
@@ -26,33 +26,33 @@
 logger = logging.getLogger('glance.registry')
 
 
-def get_registry_client(options):
+def get_registry_client(options, cxt):
     host = options['registry_host']
     port = int(options['registry_port'])
-    return client.RegistryClient(host, port)
-
-
-def get_images_list(options, **kwargs):
-    c = get_registry_client(options)
+    return client.RegistryClient(host, port, auth_tok=cxt.auth_tok)
+
+
+def get_images_list(options, context, **kwargs):
+    c = get_registry_client(options, context)
     return c.get_images(**kwargs)
 
 
-def get_images_detail(options, **kwargs):
-    c = get_registry_client(options)
+def get_images_detail(options, context, **kwargs):
+    c = get_registry_client(options, context)
     return c.get_images_detailed(**kwargs)
 
 
-def get_image_metadata(options, image_id):
-    c = get_registry_client(options)
+def get_image_metadata(options, context, image_id):
+    c = get_registry_client(options, context)
     return c.get_image(image_id)
 
 
-def add_image_metadata(options, image_meta):
+def add_image_metadata(options, context, image_meta):
     if options['debug']:
         logger.debug("Adding image metadata...")
         _debug_print_metadata(image_meta)
 
-    c = get_registry_client(options)
+    c = get_registry_client(options, context)
     new_image_meta = c.add_image(image_meta)
 
     if options['debug']:
@@ -63,12 +63,13 @@
     return new_image_meta
 
 
-def update_image_metadata(options, image_id, image_meta, purge_props=False):
+def update_image_metadata(options, context, image_id, image_meta,
+                          purge_props=False):
     if options['debug']:
         logger.debug("Updating image metadata for image %s...", image_id)
         _debug_print_metadata(image_meta)
 
-    c = get_registry_client(options)
+    c = get_registry_client(options, context)
     new_image_meta = c.update_image(image_id, image_meta, purge_props)
 
     if options['debug']:
@@ -79,9 +80,9 @@
     return new_image_meta
 
 
-def delete_image_metadata(options, image_id):
+def delete_image_metadata(options, context, image_id):
     logger.debug("Deleting image metadata for image %s...", image_id)
-    c = get_registry_client(options)
+    c = get_registry_client(options, context)
     return c.delete_image(image_id)
 
 

=== modified file 'glance/registry/client.py'
--- glance/registry/client.py	2011-06-26 20:43:41 +0000
+++ glance/registry/client.py	2011-07-26 10:13:35 +0000
@@ -33,17 +33,17 @@
 
     DEFAULT_PORT = 9191
 
-    def __init__(self, host, port=None, use_ssl=False):
+    def __init__(self, host, port=None, use_ssl=False, auth_tok=None):
         """
         Creates a new client to a Glance Registry service.
 
         :param host: The host where Glance resides
         :param port: The port where Glance resides (defaults to 9191)
         :param use_ssl: Should we use HTTPS? (defaults to False)
+        :param auth_tok: The auth token to pass to the server
         """
-
         port = port or self.DEFAULT_PORT
-        super(RegistryClient, self).__init__(host, port, use_ssl)
+        super(RegistryClient, self).__init__(host, port, use_ssl, auth_tok)
 
     def get_images(self, **kwargs):
         """

=== modified file 'glance/registry/db/api.py'
--- glance/registry/db/api.py	2011-06-26 20:56:17 +0000
+++ glance/registry/db/api.py	2011-07-26 10:13:35 +0000
@@ -24,6 +24,7 @@
 import logging
 
 from sqlalchemy import asc, create_engine, desc
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import exc
 from sqlalchemy.orm import joinedload
@@ -33,7 +34,6 @@
 from glance.common import config
 from glance.common import exception
 from glance.common import utils
-from glance.registry.db import migration
 from glance.registry.db import models
 
 _ENGINE = None
@@ -46,12 +46,14 @@
 
 IMAGE_ATTRS = BASE_MODEL_ATTRS | set(['name', 'status', 'size',
                                       'disk_format', 'container_format',
-                                      'is_public', 'location', 'checksum'])
+                                      'is_public', 'location', 'checksum',
+                                      'owner'])
 
 CONTAINER_FORMATS = ['ami', 'ari', 'aki', 'bare', 'ovf']
 DISK_FORMATS = ['ami', 'ari', 'aki', 'vhd', 'vmdk', 'raw', 'qcow2', 'vdi',
                'iso']
-STATUSES = ['active', 'saving', 'queued', 'killed']
+STATUSES = ['active', 'saving', 'queued', 'killed', 'pending_delete',
+            'deleted']
 
 
 def configure_db(options):
@@ -77,7 +79,7 @@
         elif verbose:
             logger.setLevel(logging.INFO)
 
-        migration.db_sync(options)
+        models.register_models(_ENGINE)
 
 
 def get_session(autocommit=True, expire_on_commit=False):
@@ -97,10 +99,10 @@
 
 
 def image_update(context, image_id, values, purge_props=False):
-    """Set the given properties on an image and update it.
-
-    Raises NotFound if image does not exist.
-
+    """
+    Set the given properties on an image and update it.
+
+    :raises NotFound if image does not exist.
     """
     return _image_update(context, values, image_id, purge_props)
 
@@ -120,7 +122,7 @@
     """Get an image or raise if it does not exist."""
     session = session or get_session()
     try:
-        return session.query(models.Image).\
+        image = session.query(models.Image).\
                        options(joinedload(models.Image.properties)).\
                        filter_by(deleted=_deleted(context)).\
                        filter_by(id=image_id).\
@@ -128,10 +130,40 @@
     except exc.NoResultFound:
         raise exception.NotFound("No image found with ID %s" % image_id)
 
-
-def image_get_all_public(context, filters=None, marker=None, limit=None,
-                         sort_key='created_at', sort_dir='desc'):
-    """Get all public images that match zero or more filters.
+    # Make sure they can look at it
+    if not context.is_image_visible(image):
+        raise exception.NotAuthorized("Image not visible to you")
+
+    return image
+
+
+def image_get_all_pending_delete(context, delete_time=None, limit=None):
+    """Get all images that are pending deletion
+
+    :param limit: maximum number of images to return
+    """
+    session = get_session()
+    query = session.query(models.Image).\
+                   options(joinedload(models.Image.properties)).\
+                   filter_by(deleted=True).\
+                   filter(models.Image.status == 'pending_delete')
+
+    if delete_time:
+        query = query.filter(models.Image.deleted_at <= delete_time)
+
+    query = query.order_by(desc(models.Image.deleted_at)).\
+                  order_by(desc(models.Image.id))
+
+    if limit:
+        query = query.limit(limit)
+
+    return query.all()
+
+
+def image_get_all(context, filters=None, marker=None, limit=None,
+                  sort_key='created_at', sort_dir='desc'):
+    """
+    Get all images that match zero or more filters.
 
     :param filters: dict of filter keys and values. If a 'properties'
                     key is present, it is treated as a dict of key/value
@@ -147,7 +179,6 @@
     query = session.query(models.Image).\
                    options(joinedload(models.Image.properties)).\
                    filter_by(deleted=_deleted(context)).\
-                   filter_by(is_public=True).\
                    filter(models.Image.status != 'killed')
 
     sort_dir_func = {
@@ -168,11 +199,19 @@
         query = query.filter(models.Image.size <= filters['size_max'])
         del filters['size_max']
 
+    if 'is_public' in filters and filters['is_public'] is not None:
+        the_filter = models.Image.is_public == filters['is_public']
+        if filters['is_public'] and context.owner is not None:
+            the_filter = or_(the_filter, models.Image.owner == context.owner)
+        query = query.filter(the_filter)
+        del filters['is_public']
+
     for (k, v) in filters.pop('properties', {}).items():
         query = query.filter(models.Image.properties.any(name=k, value=v))
 
     for (k, v) in filters.items():
-        query = query.filter(getattr(models.Image, k) == v)
+        if v is not None:
+            query = query.filter(getattr(models.Image, k) == v)
 
     if marker != None:
         # images returned should be created before the image defined by marker
@@ -189,7 +228,8 @@
 
 
 def _drop_protected_attrs(model_class, values):
-    """Removed protected attributes from values dictionary using the models
+    """
+    Removed protected attributes from values dictionary using the models
     __protected_attributes__ field.
     """
     for attr in model_class.__protected_attributes__:
@@ -204,7 +244,6 @@
 
     :param values: Mapping of image metadata to check
     """
-
     status = values.get('status')
     disk_format = values.get('disk_format')
     container_format = values.get('container_format')
@@ -237,13 +276,13 @@
 
 
 def _image_update(context, values, image_id, purge_props=False):
-    """Used internally by image_create and image_update
+    """
+    Used internally by image_create and image_update
 
     :param context: Request context
     :param values: A dict of attributes to set
     :param image_id: If None, create the image, otherwise, find and update it
     """
-
     session = get_session()
     with session.begin():
 
@@ -273,7 +312,11 @@
         # idiotic.
         validate_image(image_ref.to_dict())
 
-        image_ref.save(session=session)
+        try:
+            image_ref.save(session=session)
+        except IntegrityError, e:
+            raise exception.Duplicate("Image ID %s already exists!"
+                                      % values['id'])
 
         _set_properties_for_image(context, image_ref, properties, purge_props,
                                   session)
@@ -325,7 +368,8 @@
 
 
 def _image_property_update(context, prop_ref, values, session=None):
-    """Used internally by image_property_create and image_property_update
+    """
+    Used internally by image_property_create and image_property_update
     """
     _drop_protected_attrs(models.ImageProperty, values)
     values["deleted"] = False
@@ -335,7 +379,8 @@
 
 
 def image_property_delete(context, prop_ref, session=None):
-    """Used internally by image_property_create and image_property_update
+    """
+    Used internally by image_property_create and image_property_update
     """
     prop_ref.update(dict(deleted=True))
     prop_ref.save(session=session)
@@ -344,10 +389,12 @@
 
 # pylint: disable-msg=C0111
 def _deleted(context):
-    """Calculates whether to include deleted objects based on context.
-
+    """
+    Calculates whether to include deleted objects based on context.
     Currently just looks for a flag called deleted in the context dict.
     """
+    if hasattr(context, 'show_deleted'):
+        return context.show_deleted
     if not hasattr(context, 'get'):
         return False
     return context.get('deleted', False)

=== modified file 'glance/registry/db/migrate_repo/schema.py'
--- glance/registry/db/migrate_repo/schema.py	2011-03-31 14:43:28 +0000
+++ glance/registry/db/migrate_repo/schema.py	2011-07-26 10:13:35 +0000
@@ -51,7 +51,8 @@
 
 
 def from_migration_import(module_name, fromlist):
-    """Import a migration file and return the module
+    """
+    Import a migration file and return the module
 
     :param module_name: name of migration module to import from
         (ex: 001_add_images_table)
@@ -84,7 +85,6 @@
         images = define_images_table(meta)
 
         # Refer to images table
-
     """
     module_path = 'glance.registry.db.migrate_repo.versions.%s' % module_name
     module = __import__(module_path, globals(), locals(), fromlist, -1)

=== added file 'glance/registry/db/migrate_repo/versions/007_add_owner.py'
--- glance/registry/db/migrate_repo/versions/007_add_owner.py	1970-01-01 00:00:00 +0000
+++ glance/registry/db/migrate_repo/versions/007_add_owner.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,82 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from migrate.changeset import *
+from sqlalchemy import *
+from sqlalchemy.sql import and_, not_
+
+from glance.registry.db.migrate_repo.schema import (
+    Boolean, DateTime, BigInteger, Integer, String,
+    Text, from_migration_import)
+
+
+def get_images_table(meta):
+    """
+    Returns the Table object for the images table that corresponds to
+    the images table definition of this version.
+    """
+    images = Table('images', meta,
+        Column('id', Integer(), primary_key=True, nullable=False),
+        Column('name', String(255)),
+        Column('disk_format', String(20)),
+        Column('container_format', String(20)),
+        Column('size', BigInteger()),
+        Column('status', String(30), nullable=False),
+        Column('is_public', Boolean(), nullable=False, default=False,
+               index=True),
+        Column('location', Text()),
+        Column('created_at', DateTime(), nullable=False),
+        Column('updated_at', DateTime()),
+        Column('deleted_at', DateTime()),
+        Column('deleted', Boolean(), nullable=False, default=False,
+               index=True),
+        Column('checksum', String(32)),
+        Column('owner', String(255)),
+        mysql_engine='InnoDB',
+        useexisting=True)
+
+    return images
+
+
+def get_image_properties_table(meta):
+    """
+    No changes to the image properties table from 006...
+    """
+    (get_image_properties_table,) = from_migration_import(
+        '006_key_to_name', ['get_image_properties_table'])
+
+    image_properties = get_image_properties_table(meta)
+    return image_properties
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    images = get_images_table(meta)
+
+    owner = Column('owner', String(255))
+    owner.create(images)
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    images = get_images_table(meta)
+
+    images.columns['owner'].drop()

=== modified file 'glance/registry/db/migration.py'
--- glance/registry/db/migration.py	2011-03-23 13:45:11 +0000
+++ glance/registry/db/migration.py	2011-07-26 10:13:35 +0000
@@ -33,7 +33,8 @@
 
 
 def db_version(options):
-    """Return the database's current migration number
+    """
+    Return the database's current migration number
 
     :param options: options dict
     :retval version number
@@ -49,7 +50,8 @@
 
 
 def upgrade(options, version=None):
-    """Upgrade the database's current migration level
+    """
+    Upgrade the database's current migration level
 
     :param options: options dict
     :param version: version to upgrade (defaults to latest)
@@ -65,7 +67,8 @@
 
 
 def downgrade(options, version):
-    """Downgrade the database's current migration level
+    """
+    Downgrade the database's current migration level
 
     :param options: options dict
     :param version: version to downgrade to
@@ -80,7 +83,8 @@
 
 
 def version_control(options):
-    """Place a database under migration control
+    """
+    Place a database under migration control
 
     :param options: options dict
     """
@@ -94,7 +98,8 @@
 
 
 def _version_control(options):
-    """Place a database under migration control
+    """
+    Place a database under migration control
 
     :param options: options dict
     """
@@ -104,7 +109,8 @@
 
 
 def db_sync(options, version=None):
-    """Place a database under migration control and perform an upgrade
+    """
+    Place a database under migration control and perform an upgrade
 
     :param options: options dict
     :retval version number

=== modified file 'glance/registry/db/models.py'
--- glance/registry/db/models.py	2011-04-07 19:07:36 +0000
+++ glance/registry/db/models.py	2011-07-26 10:13:35 +0000
@@ -105,6 +105,7 @@
     is_public = Column(Boolean, nullable=False, default=False)
     location = Column(Text)
     checksum = Column(String(32))
+    owner = Column(String(255))
 
 
 class ImageProperty(BASE, ModelBase):
@@ -118,3 +119,12 @@
 
     name = Column(String(255), index=True, nullable=False)
     value = Column(Text)
+
+
+def register_models(engine):
+    """
+    Creates database tables for all models with the given engine
+    """
+    models = (Image, ImageProperty)
+    for model in models:
+        model.metadata.create_all(engine)

=== modified file 'glance/registry/server.py'
--- glance/registry/server.py	2011-06-27 18:05:29 +0000
+++ glance/registry/server.py	2011-07-26 10:13:35 +0000
@@ -56,8 +56,19 @@
         self.options = options
         db_api.configure_db(options)
 
+    def _get_images(self, context, **params):
+        """
+        Get images, wrapping in exception if necessary.
+        """
+        try:
+            return db_api.image_get_all(context, **params)
+        except exception.NotFound, e:
+            msg = "Invalid marker. Image could not be found."
+            raise exc.HTTPBadRequest(explanation=msg)
+
     def index(self, req):
-        """Return a basic filtered list of public, non-deleted images
+        """
+        Return a basic filtered list of public, non-deleted images
 
         :param req: the Request object coming from the wsgi layer
         :retval a mapping of the following form::
@@ -74,10 +85,9 @@
             'container_format': <CONTAINER_FORMAT>,
             'checksum': <CHECKSUM>
             }
-
         """
         params = self._get_query_params(req)
-        images = db_api.image_get_all_public(None, **params)
+        images = self._get_images(req.context, **params)
 
         results = []
         for image in images:
@@ -88,7 +98,8 @@
         return dict(images=results)
 
     def detail(self, req):
-        """Return a filtered list of public, non-deleted images in detail
+        """
+        Return a filtered list of public, non-deleted images in detail
 
         :param req: the Request object coming from the wsgi layer
         :retval a mapping of the following form::
@@ -97,11 +108,10 @@
 
         Where image_list is a sequence of mappings containing
         all image model fields.
-
         """
         params = self._get_query_params(req)
-        images = db_api.image_get_all_public(None, **params)
 
+        images = self._get_images(req.context, **params)
         image_dicts = [make_image_dict(i) for i in images]
         return dict(images=image_dicts)
 
@@ -127,15 +137,20 @@
         return params
 
     def _get_filters(self, req):
-        """Return a dictionary of query param filters from the request
+        """
+        Return a dictionary of query param filters from the request
 
         :param req: the Request object coming from the wsgi layer
         :retval a dict of key/value filters
-
         """
         filters = {}
         properties = {}
 
+        if req.context.is_admin:
+            # Only admin gets to look for non-public images
+            filters['is_public'] = self._get_is_public(req)
+        else:
+            filters['is_public'] = True
         for param in req.str_params:
             if param in SUPPORTED_FILTERS:
                 filters[param] = req.str_params.get(param)
@@ -191,12 +206,36 @@
             raise exc.HTTPBadRequest(explanation=msg)
         return sort_dir
 
+    def _get_is_public(self, req):
+        """Parse is_public into something usable."""
+        is_public = req.str_params.get('is_public', None)
+
+        if is_public is None:
+            # NOTE(vish): This preserves the default value of showing only
+            #             public images.
+            return True
+        is_public = is_public.lower()
+        if is_public == 'none':
+            return None
+        elif is_public == 'true' or is_public == '1':
+            return True
+        elif is_public == 'false' or is_public == '0':
+            return False
+        else:
+            raise exc.HTTPBadRequest("is_public must be None, True, or False")
+
     def show(self, req, id):
         """Return data about the given image id."""
         try:
-            image = db_api.image_get(None, id)
+            image = db_api.image_get(req.context, id)
         except exception.NotFound:
             raise exc.HTTPNotFound()
+        except exception.NotAuthorized:
+            # If it's private and doesn't belong to them, don't let on
+            # that it exists
+            logger.info("Access by %s to image %s denied" %
+                        (req.context.user, id))
+            raise exc.HTTPNotFound()
 
         return dict(image=make_image_dict(image))
 
@@ -208,13 +247,20 @@
         :param id:  The opaque internal identifier for the image
 
         :retval Returns 200 if delete was successful, a fault if not.
+        """
+        if req.context.read_only:
+            raise exc.HTTPForbidden()
 
-        """
-        context = None
         try:
-            db_api.image_destroy(context, id)
+            db_api.image_destroy(req.context, id)
         except exception.NotFound:
             return exc.HTTPNotFound()
+        except exception.NotAuthorized:
+            # If it's private and doesn't belong to them, don't let on
+            # that it exists
+            logger.info("Access by %s to image %s denied" %
+                        (req.context.user, id))
+            raise exc.HTTPNotFound()
 
     def create(self, req, body):
         """
@@ -226,16 +272,21 @@
         :retval Returns the newly-created image information as a mapping,
                 which will include the newly-created image's internal id
                 in the 'id' field
-
         """
+        if req.context.read_only:
+            raise exc.HTTPForbidden()
+
         image_data = body['image']
 
         # Ensure the image has a status set
         image_data.setdefault('status', 'active')
 
-        context = None
+        # Set up the image owner
+        if not req.context.is_admin or 'owner' not in image_data:
+            image_data['owner'] = req.context.owner
+
         try:
-            image_data = db_api.image_create(context, image_data)
+            image_data = db_api.image_create(req.context, image_data)
             return dict(image=make_image_dict(image_data))
         except exception.Duplicate:
             msg = ("Image with identifier %s already exists!" % id)
@@ -247,27 +298,34 @@
             return exc.HTTPBadRequest(msg)
 
     def update(self, req, id, body):
-        """Updates an existing image with the registry.
+        """
+        Updates an existing image with the registry.
 
         :param req: wsgi Request object
         :param body: Dictionary of information about the image
         :param id:  The opaque internal identifier for the image
 
         :retval Returns the updated image information as a mapping,
-
         """
+        if req.context.read_only:
+            raise exc.HTTPForbidden()
+
         image_data = body['image']
 
+        # Prohibit modification of 'owner'
+        if not req.context.is_admin and 'owner' in image_data:
+            del image_data['owner']
+
         purge_props = req.headers.get("X-Glance-Registry-Purge-Props", "false")
-        context = None
         try:
             logger.debug("Updating image %(id)s with metadata: %(image_data)r"
                          % locals())
             if purge_props == "true":
-                updated_image = db_api.image_update(context, id, image_data,
-                                                        True)
+                updated_image = db_api.image_update(req.context, id,
+                                                    image_data, True)
             else:
-                updated_image = db_api.image_update(context, id, image_data)
+                updated_image = db_api.image_update(req.context, id,
+                                                    image_data)
             return dict(image=make_image_dict(updated_image))
         except exception.Invalid, e:
             msg = ("Failed to update image metadata. "
@@ -278,6 +336,14 @@
             raise exc.HTTPNotFound(body='Image not found',
                                request=req,
                                content_type='text/plain')
+        except exception.NotAuthorized:
+            # If it's private and doesn't belong to them, don't let on
+            # that it exists
+            logger.info("Access by %s to image %s denied" %
+                        (req.context.user, id))
+            raise exc.HTTPNotFound(body='Image not found',
+                               request=req,
+                               content_type='text/plain')
 
 
 def create_resource(options):

=== modified file 'glance/store/__init__.py'
--- glance/store/__init__.py	2011-03-23 14:16:10 +0000
+++ glance/store/__init__.py	2011-07-26 10:13:35 +0000
@@ -15,11 +15,17 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import logging
 import optparse
 import os
 import urlparse
 
-from glance.common import exception
+from glance import registry
+from glance.common import config, exception
+from glance.store import location
+
+
+logger = logging.getLogger('glance.store')
 
 
 # TODO(sirp): should this be moved out to common/utils.py ?
@@ -74,70 +80,48 @@
 def get_from_backend(uri, **kwargs):
     """Yields chunks of data from backend specified by uri"""
 
-    parsed_uri = urlparse.urlparse(uri)
-    scheme = parsed_uri.scheme
-
-    backend_class = get_backend_class(scheme)
-
-    return backend_class.get(parsed_uri, **kwargs)
+    loc = location.get_location_from_uri(uri)
+    backend_class = get_backend_class(loc.store_name)
+
+    return backend_class.get(loc, **kwargs)
 
 
 def delete_from_backend(uri, **kwargs):
     """Removes chunks of data from backend specified by uri"""
 
-    parsed_uri = urlparse.urlparse(uri)
-    scheme = parsed_uri.scheme
-
-    backend_class = get_backend_class(scheme)
+    loc = location.get_location_from_uri(uri)
+    backend_class = get_backend_class(loc.store_name)
 
     if hasattr(backend_class, 'delete'):
-        return backend_class.delete(parsed_uri, **kwargs)
-
-
-def get_store_from_location(location):
+        return backend_class.delete(loc, **kwargs)
+
+
+def get_store_from_location(uri):
     """
     Given a location (assumed to be a URL), attempt to determine
     the store from the location.  We use here a simple guess that
     the scheme of the parsed URL is the store...
 
-    :param location: Location to check for the store
-    """
-    loc_pieces = urlparse.urlparse(location)
-    return loc_pieces.scheme
-
-
-def parse_uri_tokens(parsed_uri, example_url):
-    """
-    Given a URI and an example_url, attempt to parse the uri to assemble an
-    authurl. This method returns the user, key, authurl, referenced container,
-    and the object we're looking for in that container.
-
-    Parsing the uri is three phases:
-        1) urlparse to split the tokens
-        2) use RE to split on @ and /
-        3) reassemble authurl
-
-    """
-    path = parsed_uri.path.lstrip('//')
-    netloc = parsed_uri.netloc
-
-    try:
+    :param uri: Location to check for the store
+    """
+    loc = location.get_location_from_uri(uri)
+    return loc.store_name
+
+
+def schedule_delete_from_backend(uri, options, context, id, **kwargs):
+    """
+    Given a uri and a time, schedule the deletion of an image.
+    """
+    use_delay = config.get_option(options, 'delayed_delete', type='bool',
+                                  default=False)
+    if not use_delay:
+        registry.update_image_metadata(options, context, id,
+                                       {'status': 'deleted'})
         try:
-            creds, netloc = netloc.split('@')
-        except ValueError:
-            # Python 2.6.1 compat
-            # see lp659445 and Python issue7904
-            creds, path = path.split('@')
-        user, key = creds.split(':')
-        path_parts = path.split('/')
-        obj = path_parts.pop()
-        container = path_parts.pop()
-    except (ValueError, IndexError):
-        raise BackendException(
-             "Expected four values to unpack in: %s:%s. "
-             "Should have received something like: %s."
-             % (parsed_uri.scheme, parsed_uri.path, example_url))
-
-    authurl = "https://%s" % '/'.join(path_parts)
-
-    return user, key, authurl, container, obj
+            return delete_from_backend(uri, **kwargs)
+        except (UnsupportedBackend, exception.NotFound):
+            msg = "Failed to delete image from store (%s). "
+            logger.error(msg % uri)
+
+    registry.update_image_metadata(options, context, id,
+                                   {'status': 'pending_delete'})

=== modified file 'glance/store/filesystem.py'
--- glance/store/filesystem.py	2011-03-08 15:22:44 +0000
+++ glance/store/filesystem.py	2011-07-26 10:13:35 +0000
@@ -26,9 +26,39 @@
 
 from glance.common import exception
 import glance.store
+import glance.store.location
 
 logger = logging.getLogger('glance.store.filesystem')
 
+glance.store.location.add_scheme_map({'file': 'filesystem'})
+
+
+class StoreLocation(glance.store.location.StoreLocation):
+
+    """Class describing a Filesystem URI"""
+
+    def process_specs(self):
+        self.scheme = self.specs.get('scheme', 'file')
+        self.path = self.specs.get('path')
+
+    def get_uri(self):
+        return "file://%s" % self.path
+
+    def parse_uri(self, uri):
+        """
+        Parse URLs. This method fixes an issue where credentials specified
+        in the URL are interpreted differently in Python 2.6.1+ than prior
+        versions of Python.
+        """
+        pieces = urlparse.urlparse(uri)
+        assert pieces.scheme == 'file'
+        self.scheme = pieces.scheme
+        path = (pieces.netloc + pieces.path).strip()
+        if path == '':
+            reason = "No path specified"
+            raise exception.BadStoreUri(uri, reason)
+        self.path = path
+
 
 class ChunkedFile(object):
 
@@ -64,13 +94,19 @@
 
 class FilesystemBackend(glance.store.Backend):
     @classmethod
-    def get(cls, parsed_uri, expected_size=None, options=None):
-        """ Filesystem-based backend
-
-        file:///path/to/file.tar.gz.0
-        """
-
-        filepath = parsed_uri.path
+    def get(cls, location, expected_size=None, options=None):
+        """
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file, and returns a generator to use in
+        reading the image file.
+
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
+
+        :raises NotFound if file does not exist
+        """
+        loc = location.store_location
+        filepath = loc.path
         if not os.path.exists(filepath):
             raise exception.NotFound("Image file %s not found" % filepath)
         else:
@@ -79,17 +115,19 @@
             return ChunkedFile(filepath)
 
     @classmethod
-    def delete(cls, parsed_uri):
+    def delete(cls, location):
         """
-        Removes a file from the filesystem backend.
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file to delete
 
-        :param parsed_uri: Parsed pieces of URI in form of::
-            file:///path/to/filename.ext
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
 
         :raises NotFound if file does not exist
         :raises NotAuthorized if cannot delete because of permissions
         """
-        fn = parsed_uri.path
+        loc = location.store_location
+        fn = loc.path
         if os.path.exists(fn):
             try:
                 logger.debug("Deleting image at %s", fn)

=== modified file 'glance/store/http.py'
--- glance/store/http.py	2011-03-17 19:57:47 +0000
+++ glance/store/http.py	2011-07-26 10:13:35 +0000
@@ -16,31 +16,104 @@
 #    under the License.
 
 import httplib
+import urlparse
 
+from glance.common import exception
 import glance.store
+import glance.store.location
+
+glance.store.location.add_scheme_map({'http': 'http',
+                                      'https': 'http'})
+
+
+class StoreLocation(glance.store.location.StoreLocation):
+
+    """Class describing an HTTP(S) URI"""
+
+    def process_specs(self):
+        self.scheme = self.specs.get('scheme', 'http')
+        self.netloc = self.specs['netloc']
+        self.user = self.specs.get('user')
+        self.password = self.specs.get('password')
+        self.path = self.specs.get('path')
+
+    def _get_credstring(self):
+        if self.user:
+            return '%s:%s@' % (self.user, self.password)
+        return ''
+
+    def get_uri(self):
+        return "%s://%s%s%s" % (
+            self.scheme,
+            self._get_credstring(),
+            self.netloc,
+            self.path)
+
+    def parse_uri(self, uri):
+        """
+        Parse URLs. This method fixes an issue where credentials specified
+        in the URL are interpreted differently in Python 2.6.1+ than prior
+        versions of Python.
+        """
+        pieces = urlparse.urlparse(uri)
+        assert pieces.scheme in ('https', 'http')
+        self.scheme = pieces.scheme
+        netloc = pieces.netloc
+        path = pieces.path
+        try:
+            if '@' in netloc:
+                creds, netloc = netloc.split('@')
+            else:
+                creds = None
+        except ValueError:
+            # Python 2.6.1 compat
+            # see lp659445 and Python issue7904
+            if '@' in path:
+                creds, path = path.split('@')
+            else:
+                creds = None
+        if creds:
+            try:
+                self.user, self.password = creds.split(':')
+            except ValueError:
+                reason = ("Credentials '%s' not well-formatted."
+                          % "".join(creds))
+                raise exception.BadStoreUri(uri, reason)
+        else:
+            self.user = None
+        if netloc == '':
+            reason = "No address specified in HTTP URL"
+            raise exception.BadStoreUri(uri, reason)
+        self.netloc = netloc
+        self.path = path
 
 
 class HTTPBackend(glance.store.Backend):
     """ An implementation of the HTTP Backend Adapter """
 
     @classmethod
-    def get(cls, parsed_uri, expected_size, options=None, conn_class=None):
-        """Takes a parsed uri for an HTTP resource, fetches it, and yields the
-        data.
+    def get(cls, location, expected_size, options=None, conn_class=None):
         """
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file, and returns a generator from Swift
+        provided by Swift client's get_object() method.
 
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
+        """
+        loc = location.store_location
         if conn_class:
             pass  # use the conn_class passed in
-        elif parsed_uri.scheme == "http":
+        elif loc.scheme == "http":
             conn_class = httplib.HTTPConnection
-        elif parsed_uri.scheme == "https":
+        elif loc.scheme == "https":
             conn_class = httplib.HTTPSConnection
         else:
             raise glance.store.BackendException(
                 "scheme '%s' not supported for HTTPBackend")
 
-        conn = conn_class(parsed_uri.netloc)
-        conn.request("GET", parsed_uri.path, "", {})
+        conn = conn_class(loc.netloc)
+        conn.request("GET", loc.path, "", {})
 
         try:
             return glance.store._file_iter(conn.getresponse(), cls.CHUNKSIZE)

=== added file 'glance/store/location.py'
--- glance/store/location.py	1970-01-01 00:00:00 +0000
+++ glance/store/location.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,182 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2011 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+A class that describes the location of an image in Glance.
+
+In Glance, an image can either be **stored** in Glance, or it can be
+**registered** in Glance but actually be stored somewhere else.
+
+We needed a class that could support the various ways that Glance
+describes where exactly an image is stored.
+
+An image in Glance has two location properties: the image URI
+and the image storage URI.
+
+The image URI is essentially the permalink identifier for the image.
+It is displayed in the output of various Glance API calls and,
+while read-only, is entirely user-facing. It shall **not** contain any
+security credential information at all. The Glance image URI shall
+be the host:port of that Glance API server along with /images/<IMAGE_ID>.
+
+The Glance storage URI is an internal URI structure that Glance
+uses to maintain critical information about how to access the images
+that it stores in its storage backends. It **does contain** security
+credentials and is **not** user-facing.
+"""
+
+import logging
+import urlparse
+
+from glance.common import exception
+from glance.common import utils
+
+logger = logging.getLogger('glance.store.location')
+
+SCHEME_TO_STORE_MAP = {}
+
+
+def get_location_from_uri(uri):
+    """
+    Given a URI, return a Location object that has had an appropriate
+    store parse the URI.
+
+    :param uri: A URI that could come from the end-user in the Location
+                attribute/header
+
+    Example URIs:
+        https://user:[e-mail]:80/images/some-id
+        http://images.oracle.com/123456
+        swift://user:account:[e-mail]/container/obj-id
+        swift+http://user:account:[e-mail]/container/obj-id
+        s3://accesskey:[e-mail]/bucket/key-id
+        s3+https://accesskey:[e-mail]/bucket/key-id
+        file:///var/lib/glance/images/1
+    """
+    # Add known stores to mapping... this gets past circular import
+    # issues. There's a better way to do this, but that's for another
+    # patch...
+    # TODO(jaypipes) Clear up these imports in refactor-stores blueprint
+    import glance.store.filesystem
+    import glance.store.http
+    import glance.store.s3
+    import glance.store.swift
+    pieces = urlparse.urlparse(uri)
+    if pieces.scheme not in SCHEME_TO_STORE_MAP.keys():
+        raise exception.UnknownScheme(pieces.scheme)
+    loc = Location(pieces.scheme, uri=uri)
+    return loc
+
+
+def add_scheme_map(scheme_map):
+    """
+    Given a mapping of 'scheme' to store_name, adds the mapping to the
+    known list of schemes.
+
+    Each store should call this method and let Glance know about which
+    schemes to map to a store name.
+    """
+    SCHEME_TO_STORE_MAP.update(scheme_map)
+
+
+class Location(object):
+
+    """
+    Class describing the location of an image that Glance knows about
+    """
+
+    def __init__(self, store_name, uri=None, image_id=None, store_specs=None):
+        """
+        Create a new Location object.
+
+        :param store_name: The string identifier of the storage backend
+        :param image_id: The identifier of the image in whatever storage
+                         backend is used.
+        :param uri: Optional URI to construct location from
+        :param store_specs: Dictionary of information about the location
+                            of the image that is dependent on the backend
+                            store
+        """
+        self.store_name = store_name
+        self.image_id = image_id
+        self.store_specs = store_specs or {}
+        self.store_location = self._get_store_location()
+        if uri:
+            self.store_location.parse_uri(uri)
+
+    def _get_store_location(self):
+        """
+        We find the store module and then grab an instance of the store's
+        StoreLocation class which handles store-specific location information
+        """
+        try:
+            cls = utils.import_class('glance.store.%s.StoreLocation'
+                                     % SCHEME_TO_STORE_MAP[self.store_name])
+            return cls(self.store_specs)
+        except exception.NotFound:
+            logger.error("Unable to find StoreLocation class in store %s",
+                         self.store_name)
+            return None
+
+    def get_store_uri(self):
+        """
+        Returns the Glance image URI, which is the host:port of the API server
+        along with /images/<IMAGE_ID>
+        """
+        return self.store_location.get_uri()
+
+    def get_uri(self):
+        return None
+
+
+class StoreLocation(object):
+
+    """
+    Base class that must be implemented by each store
+    """
+
+    def __init__(self, store_specs):
+        self.specs = store_specs
+        if self.specs:
+            self.process_specs()
+
+    def process_specs(self):
+        """
+        Subclasses should implement any processing of the self.specs collection
+        such as storing credentials and possibly establishing connections.
+        """
+        pass
+
+    def get_uri(self):
+        """
+        Subclasses should implement a method that returns an internal URI that,
+        when supplied to the StoreLocation instance, can be interpreted by the
+        StoreLocation's parse_uri() method. The URI returned from this method
+        shall never be public and only used internally within Glance, so it is
+        fine to encode credentials in this URI.
+        """
+        raise NotImplementedError("StoreLocation subclass must implement "
+                                  "get_uri()")
+
+    def parse_uri(self, uri):
+        """
+        Subclasses should implement a method that accepts a string URI and
+        sets appropriate internal fields such that a call to get_uri() will
+        return a proper internal URI
+        """
+        raise NotImplementedError("StoreLocation subclass must implement "
+                                  "parse_uri()")

=== modified file 'glance/store/s3.py'
--- glance/store/s3.py	2011-01-27 04:19:13 +0000
+++ glance/store/s3.py	2011-07-26 10:13:35 +0000
@@ -17,7 +17,101 @@
 
 """The s3 backend adapter"""
 
+import urlparse
+
+from glance.common import exception
 import glance.store
+import glance.store.location
+
+glance.store.location.add_scheme_map({'s3': 's3',
+                                      's3+http': 's3',
+                                      's3+https': 's3'})
+
+
+class StoreLocation(glance.store.location.StoreLocation):
+
+    """
+    Class describing an S3 URI. An S3 URI can look like any of
+    the following:
+
+        s3://accesskey:[e-mail]/bucket/key-id
+        s3+http://accesskey:[e-mail]/bucket/key-id
+        s3+https://accesskey:[e-mail]/bucket/key-id
+
+    The s3+https:// URIs indicate there is an HTTPS s3service URL
+    """
+
+    def process_specs(self):
+        self.scheme = self.specs.get('scheme', 's3')
+        self.accesskey = self.specs.get('accesskey')
+        self.secretkey = self.specs.get('secretkey')
+        self.s3serviceurl = self.specs.get('s3serviceurl')
+        self.bucket = self.specs.get('bucket')
+        self.key = self.specs.get('key')
+
+    def _get_credstring(self):
+        if self.accesskey:
+            return '%s:%s@' % (self.accesskey, self.secretkey)
+        return ''
+
+    def get_uri(self):
+        return "%s://%s%s/%s/%s" % (
+            self.scheme,
+            self._get_credstring(),
+            self.s3serviceurl,
+            self.bucket,
+            self.key)
+
+    def parse_uri(self, uri):
+        """
+        Parse URLs. This method fixes an issue where credentials specified
+        in the URL are interpreted differently in Python 2.6.1+ than prior
+        versions of Python.
+
+        Note that an Amazon AWS secret key can contain the forward slash,
+        which is entirely retarded, and breaks urlparse miserably.
+        This function works around that issue.
+        """
+        pieces = urlparse.urlparse(uri)
+        assert pieces.scheme in ('s3', 's3+http', 's3+https')
+        self.scheme = pieces.scheme
+        path = pieces.path.strip('/')
+        netloc = pieces.netloc.strip('/')
+        entire_path = (netloc + '/' + path).strip('/')
+
+        if '@' in uri:
+            creds, path = entire_path.split('@')
+            cred_parts = creds.split(':')
+
+            try:
+                access_key = cred_parts[0]
+                secret_key = cred_parts[1]
+                # NOTE(jaypipes): Need to encode to UTF-8 here because of a
+                # bug in the HMAC library that boto uses.
+                # See: http://bugs.python.org/issue5285
+                # See: http://trac.edgewall.org/ticket/8083
+                access_key = access_key.encode('utf-8')
+                secret_key = secret_key.encode('utf-8')
+                self.accesskey = access_key
+                self.secretkey = secret_key
+            except IndexError:
+                reason = "Badly formed S3 credentials %s" % creds
+                raise exception.BadStoreUri(uri, reason)
+        else:
+            self.accesskey = None
+            path = entire_path
+        try:
+            path_parts = path.split('/')
+            self.key = path_parts.pop()
+            self.bucket = path_parts.pop()
+            if len(path_parts) > 0:
+                self.s3serviceurl = '/'.join(path_parts)
+            else:
+                reason = "Badly formed S3 URI. Missing s3 service URL."
+                raise exception.BadStoreUri(uri, reason)
+        except IndexError:
+            reason = "Badly formed S3 URI"
+            raise exception.BadStoreUri(uri, reason)
 
 
 class S3Backend(glance.store.Backend):
@@ -26,29 +120,30 @@
     EXAMPLE_URL = "s3://ACCESS_KEY:SECRET_KEY@s3_url/bucket/file.gz.0"
 
     @classmethod
-    def get(cls, parsed_uri, expected_size, conn_class=None):
-        """
-        Takes a parsed_uri in the format of:
-        s3://access_key:[e-mail]/bucket/file.gz.0, connects
-        to s3 and downloads the file. Returns the generator resp_body provided
-        by get_object.
-        """
+    def get(cls, location, expected_size, conn_class=None):
+        """
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file, and returns a generator from S3
+        provided by S3's key object
 
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
+        """
         if conn_class:
             pass
         else:
             import boto.s3.connection
             conn_class = boto.s3.connection.S3Connection
 
-        (access_key, secret_key, host, bucket, obj) = \
-            cls._parse_s3_tokens(parsed_uri)
+        loc = location.store_location
 
         # Close the connection when we're through.
-        with conn_class(access_key, secret_key, host=host) as s3_conn:
-            bucket = cls._get_bucket(s3_conn, bucket)
+        with conn_class(loc.accesskey, loc.secretkey,
+                        host=loc.s3serviceurl) as s3_conn:
+            bucket = cls._get_bucket(s3_conn, loc.bucket)
 
             # Close the key when we're through.
-            with cls._get_key(bucket, obj) as key:
+            with cls._get_key(bucket, loc.obj) as key:
                 if not key.size == expected_size:
                     raise glance.store.BackendException(
                         "Expected %s bytes, got %s" %
@@ -59,28 +154,28 @@
                     yield chunk
 
     @classmethod
-    def delete(cls, parsed_uri, conn_class=None):
-        """
-        Takes a parsed_uri in the format of:
-        s3://access_key:[e-mail]/bucket/file.gz.0, connects
-        to s3 and deletes the file. Returns whatever boto.s3.key.Key.delete()
-        returns.
-        """
+    def delete(cls, location, conn_class=None):
+        """
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file to delete
 
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
+        """
         if conn_class:
             pass
         else:
             conn_class = boto.s3.connection.S3Connection
 
-        (access_key, secret_key, host, bucket, obj) = \
-            cls._parse_s3_tokens(parsed_uri)
+        loc = location.store_location
 
         # Close the connection when we're through.
-        with conn_class(access_key, secret_key, host=host) as s3_conn:
-            bucket = cls._get_bucket(s3_conn, bucket)
+        with conn_class(loc.accesskey, loc.secretkey,
+                        host=loc.s3serviceurl) as s3_conn:
+            bucket = cls._get_bucket(s3_conn, loc.bucket)
 
             # Close the key when we're through.
-            with cls._get_key(bucket, obj) as key:
+            with cls._get_key(bucket, loc.obj) as key:
                 return key.delete()
 
     @classmethod
@@ -102,8 +197,3 @@
         if not key:
             raise glance.store.BackendException("Could not get key: %s" % key)
         return key
-
-    @classmethod
-    def _parse_s3_tokens(cls, parsed_uri):
-        """Parse tokens from the parsed_uri"""
-        return glance.store.parse_uri_tokens(parsed_uri, cls.EXAMPLE_URL)

=== added file 'glance/store/scrubber.py'
--- glance/store/scrubber.py	1970-01-01 00:00:00 +0000
+++ glance/store/scrubber.py	2011-07-26 10:13:35 +0000
@@ -0,0 +1,90 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 OpenStack, LLC
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+import eventlet
+import logging
+
+from glance import registry
+from glance import store
+from glance.common import config
+from glance.common import context
+from glance.common import exception
+from glance.registry.db import api as db_api
+
+
+logger = logging.getLogger('glance.store.scrubber')
+
+
+class Daemon(object):
+    def __init__(self, wakeup_time=300, threads=1000):
+        logger.info("Starting Daemon: " +
+                    "wakeup_time=%s threads=%s" % (wakeup_time, threads))
+        self.wakeup_time = wakeup_time
+        self.event = eventlet.event.Event()
+        self.pool = eventlet.greenpool.GreenPool(threads)
+
+    def start(self, application):
+        self._run(application)
+
+    def wait(self):
+        try:
+            self.event.wait()
+        except KeyboardInterrupt:
+            logger.info("Daemon Shutdown on KeyboardInterrupt")
+
+    def _run(self, application):
+        logger.debug("Runing application")
+        self.pool.spawn_n(application.run, self.pool, self.event)
+        eventlet.spawn_after(self.wakeup_time, self._run, application)
+        logger.debug("Next run scheduled in %s seconds" % self.wakeup_time)
+
+
+class Scrubber(object):
+    def __init__(self, options):
+        logger.info("Initializing scrubber with options: %s" % options)
+        self.options = options
+        scrub_time = config.get_option(options, 'scrub_time', type='int',
+                                       default=0)
+        logger.info("Scrub interval set to %s seconds" % scrub_time)
+        self.scrub_time = datetime.timedelta(seconds=scrub_time)
+        db_api.configure_db(options)
+
+    def run(self, pool, event=None):
+        delete_time = datetime.datetime.utcnow() - self.scrub_time
+        logger.info("Getting images deleted before %s" % delete_time)
+        pending = db_api.image_get_all_pending_delete(None, delete_time)
+        logger.info("Deleting %s images" % len(pending))
+        delete_work = [(p['id'], p['location']) for p in pending]
+        pool.starmap(self._delete, delete_work)
+
+    def _delete(self, id, location):
+        try:
+            logger.debug("Deleting %s" % location)
+            store.delete_from_backend(location)
+        except (store.UnsupportedBackend, exception.NotFound):
+            msg = "Failed to delete image from store (%s). "
+            logger.error(msg % uri)
+
+        ctx = context.RequestContext(is_admin=True, show_deleted=True)
+        db_api.image_update(ctx, id, {'status': 'deleted'})
+
+
+def app_factory(global_config, **local_conf):
+    conf = global_config.copy()
+    conf.update(local_conf)
+    return Scrubber(conf)

=== modified file 'glance/store/swift.py'
--- glance/store/swift.py	2011-04-13 23:18:26 +0000
+++ glance/store/swift.py	2011-07-26 10:13:35 +0000
@@ -21,50 +21,150 @@
 
 import httplib
 import logging
+import urlparse
 
 from glance.common import config
 from glance.common import exception
 import glance.store
+import glance.store.location
 
 DEFAULT_SWIFT_CONTAINER = 'glance'
 
 logger = logging.getLogger('glance.store.swift')
 
+glance.store.location.add_scheme_map({'swift': 'swift',
+                                      'swift+http': 'swift',
+                                      'swift+https': 'swift'})
+
+
+class StoreLocation(glance.store.location.StoreLocation):
+
+    """
+    Class describing a Swift URI. A Swift URI can look like any of
+    the following:
+
+        swift://user:[e-mail]/container/obj-id
+        swift+http://user:[e-mail]/container/obj-id
+        swift+https://user:[e-mail]/container/obj-id
+
+    The swift+https:// URIs indicate there is an HTTPS authentication URL
+    """
+
+    def process_specs(self):
+        self.scheme = self.specs.get('scheme', 'swift+https')
+        self.user = self.specs.get('user')
+        self.key = self.specs.get('key')
+        self.authurl = self.specs.get('authurl')
+        self.container = self.specs.get('container')
+        self.obj = self.specs.get('obj')
+
+    def _get_credstring(self):
+        if self.user:
+            return '%s:%s@' % (self.user, self.key)
+        return ''
+
+    def get_uri(self):
+        return "%s://%s%s/%s/%s" % (
+            self.scheme,
+            self._get_credstring(),
+            self.authurl,
+            self.container,
+            self.obj)
+
+    def parse_uri(self, uri):
+        """
+        Parse URLs. This method fixes an issue where credentials specified
+        in the URL are interpreted differently in Python 2.6.1+ than prior
+        versions of Python. It also deals with the peculiarity that new-style
+        Swift URIs have where a username can contain a ':', like so:
+
+            swift://account:user:[e-mail]/container/obj
+        """
+        pieces = urlparse.urlparse(uri)
+        assert pieces.scheme in ('swift', 'swift+http', 'swift+https')
+        self.scheme = pieces.scheme
+        netloc = pieces.netloc
+        path = pieces.path.lstrip('/')
+        if netloc != '':
+            # > Python 2.6.1
+            if '@' in netloc:
+                creds, netloc = netloc.split('@')
+            else:
+                creds = None
+        else:
+            # Python 2.6.1 compat
+            # see lp659445 and Python issue7904
+            if '@' in path:
+                creds, path = path.split('@')
+            else:
+                creds = None
+            netloc = path[0:path.find('/')].strip('/')
+            path = path[path.find('/'):].strip('/')
+        if creds:
+            cred_parts = creds.split(':')
+
+            # User can be account:user, in which case cred_parts[0:2] will be
+            # the account and user. Combine them into a single username of
+            # account:user
+            if len(cred_parts) == 1:
+                reason = "Badly formed credentials '%s' in Swift URI" % creds
+                raise exception.BadStoreUri(uri, reason)
+            elif len(cred_parts) == 3:
+                user = ':'.join(cred_parts[0:2])
+            else:
+                user = cred_parts[0]
+            key = cred_parts[-1]
+            self.user = user
+            self.key = key
+        else:
+            self.user = None
+        path_parts = path.split('/')
+        try:
+            self.obj = path_parts.pop()
+            self.container = path_parts.pop()
+            self.authurl = netloc
+            if len(path_parts) > 0:
+                self.authurl = netloc + '/' + '/'.join(path_parts).strip('/')
+        except IndexError:
+            reason = "Badly formed Swift URI"
+            raise exception.BadStoreUri(uri, reason)
+
 
 class SwiftBackend(glance.store.Backend):
-    """
-    An implementation of the swift backend adapter.
-    """
+    """An implementation of the swift backend adapter."""
+
     EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>"
 
     CHUNKSIZE = 65536
 
     @classmethod
-    def get(cls, parsed_uri, expected_size=None, options=None):
+    def get(cls, location, expected_size=None, options=None):
         """
-        Takes a parsed_uri in the format of:
-        swift://user:password@auth_url/container/file.gz.0, connects to the
-        swift instance at auth_url and downloads the file. Returns the
-        generator resp_body provided by get_object.
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file, and returns a generator from Swift
+        provided by Swift client's get_object() method.
+
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
         """
         from swift.common import client as swift_client
-        (user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
 
         # TODO(sirp): snet=False for now, however, if the instance of
         # swift we're talking to is within our same region, we should set
         # snet=True
+        loc = location.store_location
         swift_conn = swift_client.Connection(
-            authurl=authurl, user=user, key=key, snet=False)
+            authurl=loc.authurl, user=loc.user, key=loc.key, snet=False)
 
         try:
             (resp_headers, resp_body) = swift_conn.get_object(
-                container=container, obj=obj, resp_chunk_size=cls.CHUNKSIZE)
+                container=loc.container, obj=loc.obj,
+                resp_chunk_size=cls.CHUNKSIZE)
         except swift_client.ClientException, e:
             if e.http_status == httplib.NOT_FOUND:
-                location = format_swift_location(user, key, authurl,
-                                                 container, obj)
+                uri = location.get_store_uri()
                 raise exception.NotFound("Swift could not find image at "
-                                         "location %(location)s" % locals())
+                                         "uri %(uri)s" % locals())
 
         if expected_size:
             obj_size = int(resp_headers['content-length'])
@@ -99,6 +199,10 @@
             <CONTAINER> = ``swift_store_container``
             <ID> = The id of the image being added
 
+        :note Swift auth URLs by default use HTTPS. To specify an HTTP
+              auth URL, you can specify http://someurl.com for the
+              swift_store_auth_address config option
+
         :param id: The opaque image identifier
         :param data: The image data to write, as a file-like object
         :param options: Conf mapping
@@ -120,9 +224,14 @@
         user = cls._option_get(options, 'swift_store_user')
         key = cls._option_get(options, 'swift_store_key')
 
-        full_auth_address = auth_address
-        if not full_auth_address.startswith('http'):
-            full_auth_address = 'https://' + full_auth_address
+        scheme = 'swift+https'
+        if auth_address.startswith('http://'):
+            scheme = 'swift+http'
+            full_auth_address = auth_address
+        elif auth_address.startswith('https://'):
+            full_auth_address = auth_address
+        else:
+            full_auth_address = 'https://' + auth_address  # Defaults https
 
         swift_conn = swift_client.Connection(
             authurl=full_auth_address, user=user, key=key, snet=False)
@@ -134,8 +243,13 @@
         create_container_if_missing(container, swift_conn, options)
 
         obj_name = str(id)
-        location = format_swift_location(user, key, auth_address,
-                                         container, obj_name)
+        location = StoreLocation({'scheme': scheme,
+                                  'container': container,
+                                  'obj': obj_name,
+                                  'authurl': auth_address,
+                                  'user': user,
+                                  'key': key})
+
         try:
             obj_etag = swift_conn.put_object(container, obj_name, data)
 
@@ -153,7 +267,7 @@
             # header keys are lowercased by Swift
             if 'content-length' in resp_headers:
                 size = int(resp_headers['content-length'])
-            return (location, size, obj_etag)
+            return (location.get_uri(), size, obj_etag)
         except swift_client.ClientException, e:
             if e.http_status == httplib.CONFLICT:
                 raise exception.Duplicate("Swift already has an image at "
@@ -163,89 +277,34 @@
             raise glance.store.BackendException(msg)
 
     @classmethod
-    def delete(cls, parsed_uri):
+    def delete(cls, location):
         """
-        Deletes the swift object(s) at the parsed_uri location
+        Takes a `glance.store.location.Location` object that indicates
+        where to find the image file to delete
+
+        :location `glance.store.location.Location` object, supplied
+                  from glance.store.location.get_location_from_uri()
         """
         from swift.common import client as swift_client
-        (user, key, authurl, container, obj) = parse_swift_tokens(parsed_uri)
 
         # TODO(sirp): snet=False for now, however, if the instance of
         # swift we're talking to is within our same region, we should set
         # snet=True
+        loc = location.store_location
         swift_conn = swift_client.Connection(
-            authurl=authurl, user=user, key=key, snet=False)
+            authurl=loc.authurl, user=loc.user, key=loc.key, snet=False)
 
         try:
-            swift_conn.delete_object(container, obj)
+            swift_conn.delete_object(loc.container, loc.obj)
         except swift_client.ClientException, e:
             if e.http_status == httplib.NOT_FOUND:
-                location = format_swift_location(user, key, authurl,
-                                                 container, obj)
+                uri = location.get_store_uri()
                 raise exception.NotFound("Swift could not find image at "
-                                         "location %(location)s" % locals())
+                                         "uri %(uri)s" % locals())
             else:
                 raise
 
 
-def parse_swift_tokens(parsed_uri):
-    """
-    Return the various tokens used by Swift.
-
-    :param parsed_uri: The pieces of a URI returned by urlparse
-    :retval A tuple of (user, key, auth_address, container, obj_name)
-    """
-    path = parsed_uri.path.lstrip('//')
-    netloc = parsed_uri.netloc
-
-    try:
-        try:
-            creds, netloc = netloc.split('@')
-            path = '/'.join([netloc, path])
-        except ValueError:
-            # Python 2.6.1 compat
-            # see lp659445 and Python issue7904
-            creds, path = path.split('@')
-
-        cred_parts = creds.split(':')
-
-        # User can be account:user, in which case cred_parts[0:2] will be
-        # the account and user. Combine them into a single username of
-        # account:user
-        if len(cred_parts) == 3:
-            user = ':'.join(cred_parts[0:2])
-        else:
-            user = cred_parts[0]
-        key = cred_parts[-1]
-        path_parts = path.split('/')
-        obj = path_parts.pop()
-        container = path_parts.pop()
-    except (ValueError, IndexError):
-        raise glance.store.BackendException(
-             "Expected four values to unpack in: swift:%s. "
-             "Should have received something like: %s."
-             % (parsed_uri.path, SwiftBackend.EXAMPLE_URL))
-
-    authurl = "https://%s" % '/'.join(path_parts)
-
-    return user, key, authurl, container, obj
-
-
-def format_swift_location(user, key, auth_address, container, obj_name):
-    """
-    Returns the swift URI in the format:
-        swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<OBJNAME>
-
-    :param user: The swift user to authenticate with
-    :param key: The auth key for the authenticating user
-    :param auth_address: The base URL for the authentication service
-    :param container: The name of the container
-    :param obj_name: The name of the object
-    """
-    return "swift://%(user)s:%(key)s@%(auth_address)s/"\
-           "%(container)s/%(obj_name)s" % locals()
-
-
 def create_container_if_missing(container, swift_conn, options):
     """
     Creates a missing container in Swift if the

=== modified file 'glance/utils.py'
--- glance/utils.py	2011-06-11 18:18:03 +0000
+++ glance/utils.py	2011-07-26 10:13:35 +0000
@@ -18,6 +18,12 @@
 """
 A few utility routines used throughout Glance
 """
+import errno
+import logging
+
+import xattr
+
+logger = logging.getLogger('glance.utils')
 
 
 def image_meta_to_http_headers(image_meta):
@@ -99,3 +105,158 @@
     :param req:  Webob.Request object
     """
     return req.content_length or 'transfer-encoding' in req.headers
+
+
+def chunkiter(fp, chunk_size=65536):
+    """Return an iterator to a file-like obj which yields fixed size chunks
+
+    :param fp: a file-like object
+    :param chunk_size: maximum size of chunk
+    """
+    while True:
+        chunk = fp.read(chunk_size)
+        if chunk:
+            yield chunk
+        else:
+            break
+
+
+class PrettyTable(object):
+    """Creates an ASCII art table for use in bin/glance
+
+    Example:
+
+        ID  Name              Size         Hits
+        --- ----------------- ------------ -----
+        122 image                       22     0
+    """
+    def __init__(self):
+        self.columns = []
+
+    def add_column(self, width, label="", just='l'):
+        """Add a column to the table
+
+        :param width: number of characters wide the column should be
+        :param label: column heading
+        :param just: justification for the column, 'l' for left,
+                     'r' for right
+        """
+        self.columns.append((width, label, just))
+
+    def make_header(self):
+        label_parts = []
+        break_parts = []
+        for width, label, _ in self.columns:
+            # NOTE(sirp): headers are always left justified
+            label_part = self._clip_and_justify(label, width, 'l')
+            label_parts.append(label_part)
+
+            break_part = '-' * width
+            break_parts.append(break_part)
+
+        label_line = ' '.join(label_parts)
+        break_line = ' '.join(break_parts)
+        return '\n'.join([label_line, break_line])
+
+    def make_row(self, *args):
+        row = args
+        row_parts = []
+        for data, (width, _, just) in zip(row, self.columns):
+            row_part = self._clip_and_justify(data, width, just)
+            row_parts.append(row_part)
+
+        row_line = ' '.join(row_parts)
+        return row_line
+
+    @staticmethod
+    def _clip_and_justify(data, width, just):
+        # clip field to column width
+        clipped_data = str(data)[:width]
+
+        if just == 'r':
+            # right justify
+            justified = clipped_data.rjust(width)
+        else:
+            # left justify
+            justified = clipped_data.ljust(width)
+
+        return justified
+
+
+def _make_namespaced_xattr_key(key, namespace='user'):
+    """Create a fully-qualified xattr-key by including the intended namespace.
+
+    Namespacing differs among OSes[1]:
+
+        FreeBSD: user, system
+        Linux: user, system, trusted, security
+        MacOS X: not needed
+
+    Mac OS X won't break if we include a namespace qualifier, so, for
+    simplicity, we always include it.
+
+    --
+    [1] http://en.wikipedia.org/wiki/Extended_file_attributes
+    """
+    namespaced_key = ".".join([namespace, key])
+    return namespaced_key
+
+
+def get_xattr(path, key, **kwargs):
+    """Return the value for a particular xattr
+
+    If the key doesn't not exist, or xattrs aren't supported by the file
+    system then a KeyError will be raised, that is, unless you specify a
+    default using kwargs.
+    """
+    namespaced_key = _make_namespaced_xattr_key(key)
+    entry_xattr = xattr.xattr(path)
+    try:
+        return entry_xattr[namespaced_key]
+    except KeyError:
+        if 'default' in kwargs:
+            return kwargs['default']
+        else:
+            raise
+
+
+def set_xattr(path, key, value):
+    """Set the value of a specified xattr.
+
+    If xattrs aren't supported by the file-system, we skip setting the value.
+    """
+    namespaced_key = _make_namespaced_xattr_key(key)
+    entry_xattr = xattr.xattr(path)
+    try:
+        entry_xattr.set(namespaced_key, str(value))
+    except IOError as e:
+        if e.errno == errno.EOPNOTSUPP:
+            logger.warn("xattrs not supported, skipping...")
+        else:
+            raise
+
+
+def inc_xattr(path, key, n=1):
+    """Increment the value of an xattr (assuming it is an integer).
+
+    BEWARE, this code *does* have a RACE CONDITION, since the
+    read/update/write sequence is not atomic.
+
+    Since the use-case for this function is collecting stats--not critical--
+    the benefits of simple, lock-free code out-weighs the possibility of an
+    occasional hit not being counted.
+    """
+    try:
+        count = int(get_xattr(path, key))
+    except KeyError:
+        # NOTE(sirp): a KeyError is generated in two cases:
+        # 1) xattrs is not supported by the filesystem
+        # 2) the key is not present on the file
+        #
+        # In either case, just ignore it...
+        pass
+    else:
+        # NOTE(sirp): only try to bump the count if xattrs is supported
+        # and the key is present
+        count += n
+        set_xattr(path, key, str(count))

=== modified file 'run_tests.py'
--- run_tests.py	2011-06-24 18:44:34 +0000
+++ run_tests.py	2011-07-26 10:13:35 +0000
@@ -38,7 +38,8 @@
 #    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 #    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-"""Unittest runner for glance
+"""
+Unittest runner for glance
 
 To run all test::
     python run_tests.py
@@ -179,7 +180,8 @@
         self._last_case = None
         self.colorizer = None
         # NOTE(vish, tfukushima): reset stdout for the terminal check
-        stdout = sys.__stdout__
+        stdout = sys.stdout
+        sys.stdout = sys.__stdout__
         for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
             if colorizer.supported():
                 self.colorizer = colorizer(self.stream)
@@ -211,7 +213,8 @@
 
     # NOTE(vish, tfukushima): copied from unittest with edit to add color
     def addError(self, test, err):
-        """Overrides normal addError to add support for errorClasses.
+        """
+        Overrides normal addError to add support for errorClasses.
         If the exception is a registered class, the error will be added
         to the list for that class, not errors.
         """
@@ -279,7 +282,8 @@
 
     c = config.Config(stream=sys.stdout,
                       env=os.environ,
-                      verbosity=3)
+                      verbosity=3,
+                      plugins=core.DefaultPluginManager())
 
     runner = GlanceTestRunner(stream=c.stream,
                             verbosity=c.verbosity,

=== modified file 'setup.py'
--- setup.py	2011-04-15 09:34:17 +0000
+++ setup.py	2011-07-26 10:13:35 +0000
@@ -87,6 +87,9 @@
     ],
     scripts=['bin/glance',
              'bin/glance-api',
+             'bin/glance-cache-prefetcher',
+             'bin/glance-cache-pruner',
+             'bin/glance-cache-reaper',
              'bin/glance-control',
              'bin/glance-manage',
              'bin/glance-registry',

=== modified file 'tests/functional/__init__.py'
--- tests/functional/__init__.py	2011-05-27 02:24:55 +0000
+++ tests/functional/__init__.py	2011-07-26 10:13:35 +0000
@@ -127,7 +127,7 @@
     Server object that starts/stops/manages the API server
     """
 
-    def __init__(self, test_dir, port, registry_port):
+    def __init__(self, test_dir, port, registry_port, delayed_delete=False):
         super(ApiServer, self).__init__(test_dir, port)
         self.server_name = 'api'
         self.default_store = 'file'
@@ -137,6 +137,7 @@
                                          "api.pid")
         self.log_file = os.path.join(self.test_dir, "api.log")
         self.registry_port = registry_port
+        self.delayed_delete = delayed_delete
         self.conf_base = """[DEFAULT]
 verbose = %(verbose)s
 debug = %(debug)s
@@ -147,9 +148,10 @@
 registry_host = 0.0.0.0
 registry_port = %(registry_port)s
 log_file = %(log_file)s
+delayed_delete = %(delayed_delete)s
 
 [pipeline:glance-api]
-pipeline = versionnegotiation apiv1app
+pipeline = versionnegotiation context apiv1app
 
 [pipeline:versions]
 pipeline = versionsapp
@@ -162,6 +164,9 @@
 
 [filter:versionnegotiation]
 paste.filter_factory = glance.api.middleware.version_negotiation:filter_factory
+
+[filter:context]
+paste.filter_factory = glance.common.context:filter_factory
 """
 
 
@@ -175,11 +180,7 @@
         super(RegistryServer, self).__init__(test_dir, port)
         self.server_name = 'registry'
 
-        # NOTE(sirp): in-memory DBs don't play well with sqlalchemy migrate
-        # (see http://code.google.com/p/sqlalchemy-migrate/
-        #            issues/detail?id=72)
-        self.db_file = os.path.join(self.test_dir, 'test_glance_api.sqlite')
-        default_sql_connection = 'sqlite:///%s' % self.db_file
+        default_sql_connection = 'sqlite:///'
         self.sql_connection = os.environ.get('GLANCE_TEST_SQL_CONNECTION',
                                              default_sql_connection)
 
@@ -195,8 +196,44 @@
 sql_connection = %(sql_connection)s
 sql_idle_timeout = 3600
 
-[app:glance-registry]
+[pipeline:glance-registry]
+pipeline = context registryapp
+
+[app:registryapp]
 paste.app_factory = glance.registry.server:app_factory
+
+[filter:context]
+paste.filter_factory = glance.common.context:filter_factory
+"""
+
+
+class ScrubberDaemon(Server):
+    """
+    Server object that starts/stops/manages the Scrubber server
+    """
+
+    def __init__(self, test_dir, sql_connection, daemon=False):
+        # NOTE(jkoelker): Set the port to 0 since we actually don't listen
+        super(ScrubberDaemon, self).__init__(test_dir, 0)
+        self.server_name = 'scrubber'
+        self.daemon = daemon
+
+        self.sql_connection = sql_connection
+
+        self.pid_file = os.path.join(self.test_dir, "scrubber.pid")
+        self.log_file = os.path.join(self.test_dir, "scrubber.log")
+        self.conf_base = """[DEFAULT]
+verbose = %(verbose)s
+debug = %(debug)s
+log_file = %(log_file)s
+scrub_time = 5
+daemon = %(daemon)s
+wakeup_time = 2
+sql_connection = %(sql_connection)s
+sql_idle_timeout = 3600
+
+[app:glance-scrubber]
+paste.app_factory = glance.store.scrubber:app_factory
 """
 
 
@@ -221,8 +258,13 @@
         self.registry_server = RegistryServer(self.test_dir,
                                               self.registry_port)
 
+        registry_db = self.registry_server.sql_connection
+        self.scrubber_daemon = ScrubberDaemon(self.test_dir,
+                                              sql_connection=registry_db)
+
         self.pid_files = [self.api_server.pid_file,
-                          self.registry_server.pid_file]
+                          self.registry_server.pid_file,
+                          self.scrubber_daemon.pid_file]
         self.files_to_destroy = []
 
     def tearDown(self):
@@ -308,6 +350,13 @@
                          "Got: %s" % err)
         self.assertTrue("Starting glance-registry with" in out)
 
+        exitcode, out, err = self.scrubber_daemon.start(**kwargs)
+
+        self.assertEqual(0, exitcode,
+                         "Failed to spin up the Scrubber daemon. "
+                         "Got: %s" % err)
+        self.assertTrue("Starting glance-scrubber with" in out)
+
         self.wait_for_servers()
 
     def ping_server(self, port):
@@ -365,6 +414,10 @@
                          "Failed to spin down the Registry server. "
                          "Got: %s" % err)
 
+        exitcode, out, err = self.scrubber_daemon.stop()
+        self.assertEqual(0, exitcode,
+                         "Failed to spin down the Scrubber daemon. "
+                         "Got: %s" % err)
         # If all went well, then just remove the test directory.
         # We only want to check the logs and stuff if something
         # went wrong...

=== modified file 'tests/functional/test_bin_glance.py'
--- tests/functional/test_bin_glance.py	2011-05-05 23:12:21 +0000
+++ tests/functional/test_bin_glance.py	2011-07-26 10:13:35 +0000
@@ -40,7 +40,6 @@
             3. Delete the image
             4. Verify no longer in index
         """
-
         self.cleanup()
         self.start_servers()
 
@@ -106,7 +105,6 @@
             5. Update the image's Name attribute
             6. Verify the updated name is shown
         """
-
         self.cleanup()
         self.start_servers()
 
@@ -192,7 +190,6 @@
             3. Verify the status of the image is displayed in the show output
                and is in status 'killed'
         """
-
         self.cleanup()
 
         # Start servers with a Swift backend and a bad auth URL
@@ -253,7 +250,6 @@
             3. Verify no public images found
             4. Run SQL against DB to verify no undeleted properties
         """
-
         self.cleanup()
         self.start_servers()
 

=== modified file 'tests/functional/test_curl_api.py'
--- tests/functional/test_curl_api.py	2011-06-26 19:57:51 +0000
+++ tests/functional/test_curl_api.py	2011-07-26 10:13:35 +0000
@@ -63,7 +63,6 @@
         11. PUT /images/1
         - Add a previously deleted property.
         """