Merge lp:~mdragon/nova/system-usages into lp:~hudson-openstack/nova/trunk

[Monsyne Dragon]

This adds system usage notifications using the notifications framework.
These are designed to feed an external billing or similar system that subscribes to the nova feed and does the analysis.

[Trey Morris (tr3buchet)]

all my comments were discussed verbally. looks good.

review: Approve

[Ed Leafe (ed-leafe)]

lgtm

review: Approve

[Matt Dietz (cerberus)]

Good work on this! Tests look good.

However, looks like there's some room for cleanup

Couldn't we just take lines:

152 + usage_info = dict(
153 + tenant_id=instance_ref['project_id'],
154 + user_id=instance_ref['user_id'],
155 + instance_id=instance_ref['id'],
156 + instance_type=instance_ref['instance_type']['name'],
157 + instance_type_id=instance_ref['instance_type_id'],
158 + display_name=instance_ref['display_name'],
159 + created_at=str(instance_ref['created_at']),
160 + launched_at=str(instance_ref['launched_at']) \
161 + if instance_ref['launched_at'] else '',
162 + image_ref=instance_ref['image_ref'])

And make a utility method, like "usage_from_instance" ? You repeat the pattern 3 times.

review: Needs Fixing

[lp:~mdragon/nova/system-usages]

1086. By Monsyne Dragon on 2011-06-28

Refactored usage generation

[Monsyne Dragon (mdragon)]

ok, I refactored that method. Was debating that anyway.

[Matt Dietz (cerberus)]

Cool beans

review: Approve

=== added file 'bin/instance-usage-audit'
--- bin/instance-usage-audit	1970-01-01 00:00:00 +0000
+++ bin/instance-usage-audit	2011-06-28 20:37:32 +0000
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2011 Openstack, LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Cron script to generate usage notifications for instances neither created
+   nor destroyed in a given time period.
+
+   Together with the notifications generated by compute on instance
+   create/delete/resize, over that ime period, this allows an external
+   system consuming usage notification feeds to calculate instance usage
+   for each tenant.
+
+   Time periods are specified like so:
+   <number>[mdy]
+
+   1m = previous month. If the script is run April 1, it will generate usages
+        for March 1 thry March 31.
+   3m = 3 previous months.
+   90d = previous 90 days.
+   1y = previous year. If run on Jan 1, it generates usages for
+        Jan 1 thru Dec 31 of the previous year.
+"""
+
+import datetime
+import gettext
+import os
+import sys
+import time
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+POSSIBLE_TOPDIR = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(POSSIBLE_TOPDIR, 'nova', '__init__.py')):
+    sys.path.insert(0, POSSIBLE_TOPDIR)
+
+gettext.install('nova', unicode=1)
+
+
+from nova import context
+from nova import db
+from nova import exception
+from nova import flags
+from nova import log as logging
+from nova import utils
+
+from nova.notifier import api as notifier_api
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string('instance_usage_audit_period', '1m',
+                    'time period to generate instance usages for.')
+
+
+def time_period(period):
+    today = datetime.date.today()
+    unit = period[-1]
+    if unit not in 'mdy':
+        raise ValueError('Time period must be m, d, or y')
+    n = int(period[:-1])
+    if unit == 'm':
+        year = today.year - (n // 12)
+        n = n % 12
+        if n >= today.month:
+            year -= 1
+            month = 12 + (today.month - n)
+        else:
+            month = today.month - n
+        begin = datetime.datetime(day=1, month=month, year=year)
+        end = datetime.datetime(day=1, month=today.month, year=today.year)
+
+    elif unit == 'y':
+        begin = datetime.datetime(day=1, month=1, year=today.year - n)
+        end = datetime.datetime(day=1, month=1, year=today.year)
+
+    elif unit == 'd':
+        b = today - datetime.timedelta(days=n)
+        begin = datetime.datetime(day=b.day, month=b.month, year=b.year)
+        end = datetime.datetime(day=today.day,
+                               month=today.month,
+                               year=today.year)
+
+    return (begin, end)
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    flags.FLAGS(sys.argv)
+    logging.setup()
+    begin, end = time_period(FLAGS.instance_usage_audit_period)
+    print "Creating usages for %s until %s" % (str(begin), str(end))
+    instances = db.instance_get_active_by_window(context.get_admin_context(),
+                                                 begin,
+                                                 end)
+    print "%s instances" % len(instances)
+    for instance_ref in instances:
+        usage_info = utils.usage_from_instance(instance_ref,
+                              audit_period_begining=str(begin),
+                              audit_period_ending=str(end))
+        notifier_api.notify('compute.%s' % FLAGS.host,
+                            'compute.instance.exists',
+                            notifier_api.INFO,
+                            usage_info)

=== modified file 'nova/compute/manager.py'
--- nova/compute/manager.py	2011-06-24 12:01:51 +0000
+++ nova/compute/manager.py	2011-06-28 20:37:32 +0000
@@ -49,10 +49,12 @@
 from nova import log as logging
 from nova import manager
 from nova import network
+from nova import notifier
 from nova import rpc
 from nova import utils
 from nova import volume
 from nova.compute import power_state
+from nova.notifier import api as notifier_api
 from nova.compute.utils import terminate_volumes
 from nova.virt import driver
 
@@ -343,6 +345,11 @@
 
             self._update_launched_at(context, instance_id)
             self._update_state(context, instance_id)
+            usage_info = utils.usage_from_instance(instance_ref)
+            notifier_api.notify('compute.%s' % self.host,
+                                'compute.instance.create',
+                                notifier_api.INFO,
+                                usage_info)
         except exception.InstanceNotFound:
             # FIXME(wwolf): We are just ignoring InstanceNotFound
             # exceptions here in case the instance was immediately
@@ -421,9 +428,15 @@
     def terminate_instance(self, context, instance_id):
         """Terminate an instance on this host."""
         self._shutdown_instance(context, instance_id, 'Terminating')
+        instance_ref = self.db.instance_get(context.elevated(), instance_id)
 
         # TODO(ja): should we keep it in a terminated state for a bit?
         self.db.instance_destroy(context, instance_id)
+        usage_info = utils.usage_from_instance(instance_ref)
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.delete',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -460,6 +473,12 @@
         self._update_image_ref(context, instance_id, image_ref)
         self._update_launched_at(context, instance_id)
         self._update_state(context, instance_id)
+        usage_info = utils.usage_from_instance(instance_ref,
+                                               image_ref=image_ref)
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.rebuild',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -637,6 +656,11 @@
         context = context.elevated()
         instance_ref = self.db.instance_get(context, instance_id)
         self.driver.destroy(instance_ref)
+        usage_info = utils.usage_from_instance(instance_ref)
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.resize.confirm',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -684,6 +708,11 @@
         self.driver.revert_resize(instance_ref)
         self.db.migration_update(context, migration_id,
                 {'status': 'reverted'})
+        usage_info = utils.usage_from_instance(instance_ref)
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.resize.revert',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock
@@ -720,6 +749,13 @@
                        'migration_id': migration_ref['id'],
                        'instance_id': instance_id, },
                 })
+        usage_info = utils.usage_from_instance(instance_ref,
+                              new_instance_type=instance_type['name'],
+                              new_instance_type_id=instance_type['id'])
+        notifier_api.notify('compute.%s' % self.host,
+                            'compute.instance.resize.prep',
+                            notifier_api.INFO,
+                            usage_info)
 
     @exception.wrap_exception
     @checks_instance_lock

=== modified file 'nova/db/api.py'
--- nova/db/api.py	2011-06-28 13:04:19 +0000
+++ nova/db/api.py	2011-06-28 20:37:32 +0000
@@ -442,6 +442,11 @@
     return IMPL.instance_get_all(context)
 
 
+def instance_get_active_by_window(context, begin, end=None):
+    """Get instances active during a certain time window."""
+    return IMPL.instance_get_active_by_window(context, begin, end)
+
+
 def instance_get_all_by_user(context, user_id):
     """Get all instances."""
     return IMPL.instance_get_all_by_user(context, user_id)

=== modified file 'nova/db/sqlalchemy/api.py'
--- nova/db/sqlalchemy/api.py	2011-06-28 13:04:19 +0000
+++ nova/db/sqlalchemy/api.py	2011-06-28 20:37:32 +0000
@@ -956,6 +956,24 @@
 
 
 @require_admin_context
+def instance_get_active_by_window(context, begin, end=None):
+    """Return instances that were continuously active over the given window"""
+    session = get_session()
+    query = session.query(models.Instance).\
+                   options(joinedload_all('fixed_ip.floating_ips')).\
+                   options(joinedload('security_groups')).\
+                   options(joinedload_all('fixed_ip.network')).\
+                   options(joinedload('instance_type')).\
+                   filter(models.Instance.launched_at < begin)
+    if end:
+        query = query.filter(or_(models.Instance.terminated_at == None,
+                                 models.Instance.terminated_at > end))
+    else:
+        query = query.filter(models.Instance.terminated_at == None)
+    return query.all()
+
+
+@require_admin_context
 def instance_get_all_by_user(context, user_id):
     session = get_session()
     return session.query(models.Instance).\

=== added file 'nova/notifier/test_notifier.py'
--- nova/notifier/test_notifier.py	1970-01-01 00:00:00 +0000
+++ nova/notifier/test_notifier.py	2011-06-28 20:37:32 +0000
@@ -0,0 +1,28 @@
+# Copyright 2011 OpenStack LLC.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+
+from nova import flags
+from nova import log as logging
+
+FLAGS = flags.FLAGS
+
+NOTIFICATIONS = []
+
+
+def notify(message):
+    """Test notifier, stores notifications in memory for unittests."""
+    NOTIFICATIONS.append(message)

=== modified file 'nova/tests/test_compute.py'
--- nova/tests/test_compute.py	2011-06-24 12:01:51 +0000
+++ nova/tests/test_compute.py	2011-06-28 20:37:32 +0000
@@ -37,6 +37,7 @@
 from nova import rpc
 from nova import test
 from nova import utils
+from nova.notifier import test_notifier
 
 LOG = logging.getLogger('nova.tests.compute')
 FLAGS = flags.FLAGS
@@ -62,6 +63,7 @@
         super(ComputeTestCase, self).setUp()
         self.flags(connection_type='fake',
                    stub_network=True,
+                   notification_driver='nova.notifier.test_notifier',
                    network_manager='nova.network.manager.FlatManager')
         self.compute = utils.import_object(FLAGS.compute_manager)
         self.compute_api = compute.API()
@@ -69,6 +71,7 @@
         self.user = self.manager.create_user('fake', 'fake', 'fake')
         self.project = self.manager.create_project('fake', 'fake', 'fake')
         self.context = context.RequestContext('fake', 'fake', False)
+        test_notifier.NOTIFICATIONS = []
 
         def fake_show(meh, context, id):
             return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}}
@@ -327,6 +330,50 @@
         self.assert_(console)
         self.compute.terminate_instance(self.context, instance_id)
 
+    def test_run_instance_usage_notification(self):
+        """Ensure run instance generates apropriate usage notification"""
+        instance_id = self._create_instance()
+        self.compute.run_instance(self.context, instance_id)
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.create')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_ref'], '1')
+        self.compute.terminate_instance(self.context, instance_id)
+
+    def test_terminate_usage_notification(self):
+        """Ensure terminate_instance generates apropriate usage notification"""
+        instance_id = self._create_instance()
+        self.compute.run_instance(self.context, instance_id)
+        test_notifier.NOTIFICATIONS = []
+        self.compute.terminate_instance(self.context, instance_id)
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.delete')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_ref'], '1')
+
     def test_run_instance_existing(self):
         """Ensure failure when running an instance that already exists"""
         instance_id = self._create_instance()
@@ -378,6 +425,36 @@
 
         self.compute.terminate_instance(self.context, instance_id)
 
+    def test_resize_instance_notification(self):
+        """Ensure notifications on instance migrate/resize"""
+        instance_id = self._create_instance()
+        context = self.context.elevated()
+
+        self.compute.run_instance(self.context, instance_id)
+        test_notifier.NOTIFICATIONS = []
+
+        db.instance_update(self.context, instance_id, {'host': 'foo'})
+        self.compute.prep_resize(context, instance_id, 1)
+        migration_ref = db.migration_get_by_instance_and_status(context,
+                instance_id, 'pre-migrating')
+
+        self.assertEquals(len(test_notifier.NOTIFICATIONS), 1)
+        msg = test_notifier.NOTIFICATIONS[0]
+        self.assertEquals(msg['priority'], 'INFO')
+        self.assertEquals(msg['event_type'], 'compute.instance.resize.prep')
+        payload = msg['payload']
+        self.assertEquals(payload['tenant_id'], self.project.id)
+        self.assertEquals(payload['user_id'], self.user.id)
+        self.assertEquals(payload['instance_id'], instance_id)
+        self.assertEquals(payload['instance_type'], 'm1.tiny')
+        type_id = instance_types.get_instance_type_by_name('m1.tiny')['id']
+        self.assertEquals(str(payload['instance_type_id']), str(type_id))
+        self.assertTrue('display_name' in payload)
+        self.assertTrue('created_at' in payload)
+        self.assertTrue('launched_at' in payload)
+        self.assertEquals(payload['image_ref'], '1')
+        self.compute.terminate_instance(context, instance_id)
+
     def test_resize_instance(self):
         """Ensure instance can be migrated/resized"""
         instance_id = self._create_instance()

=== modified file 'nova/utils.py'
--- nova/utils.py	2011-06-28 14:43:25 +0000
+++ nova/utils.py	2011-06-28 20:37:32 +0000
@@ -282,6 +282,22 @@
                            'ABCDEFGHJKLMNPQRSTUVWXYZ')  # Removed: I, O
 
 
+def usage_from_instance(instance_ref, **kw):
+    usage_info = dict(
+          tenant_id=instance_ref['project_id'],
+          user_id=instance_ref['user_id'],
+          instance_id=instance_ref['id'],
+          instance_type=instance_ref['instance_type']['name'],
+          instance_type_id=instance_ref['instance_type_id'],
+          display_name=instance_ref['display_name'],
+          created_at=str(instance_ref['created_at']),
+          launched_at=str(instance_ref['launched_at']) \
+                      if instance_ref['launched_at'] else '',
+          image_ref=instance_ref['image_ref'])
+    usage_info.update(kw)
+    return usage_info
+
+
 def generate_password(length=20, symbols=DEFAULT_PASSWORD_SYMBOLS):
     """Generate a random password from the supplied symbols.