Merge lp:~vishvananda/nova/fix-exceptions into lp:~hudson-openstack/nova/trunk

[Vish Ishaya]

More changes to volume to fix concurrency issues. Also testing updates.

volumes now store a set of available shelf/blades in the datastore, instead of depending on a filesystem glob. This avoids a race condition that occurs where two different volumes are attempting to export to the same location.

The general idea of pooled resources needs to be abstracted out into the datamodel. It is used for vpn ports and volumes now, and should be uses for ip addresses as well.

[Vish Ishaya (vishvananda)]

Hmm, looks like there were some weird merge errors, Merged in trunk to fix.

Vish

On Thu, Aug 5, 2010 at 12:23 PM, vishvananda <email address hidden> wrote:

> vishvananda has proposed merging lp:~vishvananda/nova/fix-exceptions into
> lp:nova.
>
> Requested reviews:
> Nova Core (nova-core)
>
>
> More changes to volume to fix concurrency issues. Also testing updates.
>
> volumes now store a set of available shelf/blades in the datastore, instead
> of depending on a filesystem glob. This avoids a race condition that occurs
> where two different volumes are attempting to export to the same location.
>
> The general idea of pooled resources needs to be abstracted out into the
> datamodel. It is used for vpn ports and volumes now, and should be uses for
> ip addresses as well.
>
>
> --
> https://code.launchpad.net/~vishvananda/nova/fix-exceptions/+merge/31877
> You are the owner of lp:~vishvananda/nova/fix-exceptions.
>
> === modified file 'nova/tests/volume_unittest.py'
> --- nova/tests/volume_unittest.py 2010-07-31 02:33:07 +0000
> +++ nova/tests/volume_unittest.py 2010-08-05 19:23:40 +0000
> @@ -17,6 +17,10 @@
> # under the License.
>
> import logging
> +import shutil
> +import tempfile
> +
> +from twisted.internet import defer
>
> from nova import compute
> from nova import exception
> @@ -34,10 +38,16 @@
> super(VolumeTestCase, self).setUp()
> self.compute = compute.service.ComputeService()
> self.volume = None
> + self.tempdir = tempfile.mkdtemp()
> self.flags(connection_type='fake',
> - fake_storage=True)
> + fake_storage=True,
> + aoe_export_dir=self.tempdir)
> self.volume = volume_service.VolumeService()
>
> + def tearDown(self):
> + shutil.rmtree(self.tempdir)
> +
> + @defer.inlineCallbacks
> def test_run_create_volume(self):
> vol_size = '0'
> user_id = 'fake'
> @@ -48,34 +58,52 @@
> volume_service.get_volume(volume_id)['volume_id'])
>
> rv = self.volume.delete_volume(volume_id)
> +<<<<<<< TREE
> self.assertFailure(volume_service.get_volume(volume_id),
> exception.Error)
> +=======
> + self.assertRaises(exception.Error, volume_service.get_volume,
> volume_id)
> +>>>>>>> MERGE-SOURCE
>
> + @defer.inlineCallbacks
> def test_too_big_volume(self):
> vol_size = '1001'
> user_id = 'fake'
> project_id = 'fake'
> - self.assertRaises(TypeError,
> - self.volume.create_volume,
> - vol_size, user_id, project_id)
> + try:
> + yield self.volume.create_volume(vol_size, user_id, project_id)
> + self.fail("Should have thrown TypeError")
> + except TypeError:
> + pass
>
> + @defer.inlineCallbacks
> def test_too_many_volumes(self):
> vol_size = '1'
> user_id = 'fake'
> project_id = 'fake'
> num_shelves = FLAGS.last_shelf_id - FLAGS.first_shelf_id + 1
> - total_slots = FLAGS.slots_per_shelf * num_shelves
> + total_slots = FLAGS.blades_per_sh...

[Eric Day (eday)]

lgtm!

review: Approve

[OpenStack Infra (hudson-openstack)]

There is no resulting diff between lp:~vishvananda/nova/fix-exceptions and lp:nova.

=== modified file 'nova/tests/volume_unittest.py'
--- nova/tests/volume_unittest.py	2010-07-31 02:33:07 +0000
+++ nova/tests/volume_unittest.py	2010-08-06 22:56:00 +0000
@@ -17,6 +17,10 @@
 #    under the License.
 
 import logging
+import shutil
+import tempfile
+
+from twisted.internet import defer
 
 from nova import compute
 from nova import exception
@@ -34,10 +38,16 @@
         super(VolumeTestCase, self).setUp()
         self.compute = compute.service.ComputeService()
         self.volume = None
+        self.tempdir = tempfile.mkdtemp()
         self.flags(connection_type='fake',
-                   fake_storage=True)
+                   fake_storage=True,
+                   aoe_export_dir=self.tempdir)
         self.volume = volume_service.VolumeService()
 
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    @defer.inlineCallbacks
     def test_run_create_volume(self):
         vol_size = '0'
         user_id = 'fake'
@@ -48,34 +58,40 @@
                          volume_service.get_volume(volume_id)['volume_id'])
 
         rv = self.volume.delete_volume(volume_id)
-        self.assertFailure(volume_service.get_volume(volume_id),
-                           exception.Error)
+        self.assertRaises(exception.Error, volume_service.get_volume, volume_id)
 
+    @defer.inlineCallbacks
     def test_too_big_volume(self):
         vol_size = '1001'
         user_id = 'fake'
         project_id = 'fake'
-        self.assertRaises(TypeError,
-                          self.volume.create_volume,
-                          vol_size, user_id, project_id)
+        try:
+            yield self.volume.create_volume(vol_size, user_id, project_id)
+            self.fail("Should have thrown TypeError")
+        except TypeError:
+            pass
 
+    @defer.inlineCallbacks
     def test_too_many_volumes(self):
         vol_size = '1'
         user_id = 'fake'
         project_id = 'fake'
         num_shelves = FLAGS.last_shelf_id - FLAGS.first_shelf_id + 1
-        total_slots = FLAGS.slots_per_shelf * num_shelves
+        total_slots = FLAGS.blades_per_shelf * num_shelves
         vols = []
+        from nova import datastore
+        redis = datastore.Redis.instance()
         for i in xrange(total_slots):
             vid = yield self.volume.create_volume(vol_size, user_id, project_id)
             vols.append(vid)
         self.assertFailure(self.volume.create_volume(vol_size,
                                                      user_id,
                                                      project_id),
-                           volume_service.NoMoreVolumes)
+                           volume_service.NoMoreBlades)
         for id in vols:
             yield self.volume.delete_volume(id)
 
+    @defer.inlineCallbacks
     def test_run_attach_detach_volume(self):
         # Create one volume and one compute to test with
         instance_id = "storage-test"
@@ -84,22 +100,26 @@
         project_id = 'fake'
         mountpoint = "/dev/sdf"
         volume_id = yield self.volume.create_volume(vol_size, user_id, project_id)
-
         volume_obj = volume_service.get_volume(volume_id)
         volume_obj.start_attach(instance_id, mountpoint)
-        rv = yield self.compute.attach_volume(volume_id,
-                                          instance_id,
-                                          mountpoint)
+        if FLAGS.fake_tests:
+            volume_obj.finish_attach()
+        else:
+            rv = yield self.compute.attach_volume(instance_id,
+                                                  volume_id,
+                                                  mountpoint)
         self.assertEqual(volume_obj['status'], "in-use")
-        self.assertEqual(volume_obj['attachStatus'], "attached")
+        self.assertEqual(volume_obj['attach_status'], "attached")
         self.assertEqual(volume_obj['instance_id'], instance_id)
         self.assertEqual(volume_obj['mountpoint'], mountpoint)
 
-        self.assertRaises(exception.Error,
-                          self.volume.delete_volume,
-                          volume_id)
-
-        rv = yield self.volume.detach_volume(volume_id)
+        self.assertFailure(self.volume.delete_volume(volume_id), exception.Error)
+        volume_obj.start_detach()
+        if FLAGS.fake_tests:
+            volume_obj.finish_detach()
+        else:
+            rv = yield self.volume.detach_volume(instance_id,
+                                                 volume_id)
         volume_obj = volume_service.get_volume(volume_id)
         self.assertEqual(volume_obj['status'], "available")
 
@@ -108,6 +128,27 @@
                           volume_service.get_volume,
                           volume_id)
 
+    @defer.inlineCallbacks
+    def test_multiple_volume_race_condition(self):
+        vol_size = "5"
+        user_id = "fake"
+        project_id = 'fake'
+        shelf_blades = []
+        def _check(volume_id):
+            vol = volume_service.get_volume(volume_id)
+            shelf_blade = '%s.%s' % (vol['shelf_id'], vol['blade_id'])
+            self.assert_(shelf_blade not in shelf_blades)
+            shelf_blades.append(shelf_blade)
+            logging.debug("got %s" % shelf_blade)
+            vol.destroy()
+        deferreds = []
+        for i in range(5):
+            d = self.volume.create_volume(vol_size, user_id, project_id)
+            d.addCallback(_check)
+            d.addErrback(self.fail)
+            deferreds.append(d)
+        yield defer.DeferredList(deferreds)
+
     def test_multi_node(self):
         # TODO(termie): Figure out how to test with two nodes,
         # each of them having a different FLAG for storage_node

=== modified file 'nova/virt/libvirt_conn.py'
--- nova/virt/libvirt_conn.py	2010-08-06 21:27:48 +0000
+++ nova/virt/libvirt_conn.py	2010-08-06 22:56:00 +0000
@@ -114,7 +114,8 @@
     def _cleanup(self, instance):
         target = os.path.abspath(instance.datamodel['basepath'])
         logging.info("Deleting instance files at %s", target)
-        shutil.rmtree(target)
+        if os.path.exists(target):
+            shutil.rmtree(target)
 
 
     @defer.inlineCallbacks

=== modified file 'nova/volume/service.py'
--- nova/volume/service.py	2010-07-31 02:33:07 +0000
+++ nova/volume/service.py	2010-08-06 22:56:00 +0000
@@ -22,12 +22,8 @@
 Currently uses Ata-over-Ethernet.
 """
 
-import glob
 import logging
 import os
-import shutil
-import socket
-import tempfile
 
 from twisted.internet import defer
 
@@ -47,9 +43,6 @@
                     'Name for the VG that will contain exported volumes')
 flags.DEFINE_string('aoe_eth_dev', 'eth0',
                     'Which device to export the volumes on')
-flags.DEFINE_string('storage_name',
-                    socket.gethostname(),
-                    'name of this service')
 flags.DEFINE_integer('first_shelf_id',
                     utils.last_octet(utils.get_my_ip()) * 10,
                     'AoE starting shelf_id for this service')
@@ -59,9 +52,9 @@
 flags.DEFINE_string('aoe_export_dir',
                     '/var/lib/vblade-persist/vblades',
                     'AoE directory where exports are created')
-flags.DEFINE_integer('slots_per_shelf',
+flags.DEFINE_integer('blades_per_shelf',
                     16,
-                    'Number of AoE slots per shelf')
+                    'Number of AoE blades per shelf')
 flags.DEFINE_string('storage_availability_zone',
                     'nova',
                     'availability zone of this service')
@@ -69,7 +62,7 @@
                      'Should we make real storage volumes to attach?')
 
 
-class NoMoreVolumes(exception.Error):
+class NoMoreBlades(exception.Error):
     pass
 
 def get_volume(volume_id):
@@ -77,8 +70,9 @@
     volume_class = Volume
     if FLAGS.fake_storage:
         volume_class = FakeVolume
-    if datastore.Redis.instance().sismember('volumes', volume_id):
-        return volume_class(volume_id=volume_id)
+    vol = volume_class.lookup(volume_id)
+    if vol:
+        return vol
     raise exception.Error("Volume does not exist")
 
 class VolumeService(service.Service):
@@ -91,18 +85,9 @@
         super(VolumeService, self).__init__()
         self.volume_class = Volume
         if FLAGS.fake_storage:
-            FLAGS.aoe_export_dir = tempfile.mkdtemp()
             self.volume_class = FakeVolume
         self._init_volume_group()
 
-    def __del__(self):
-        # TODO(josh): Get rid of this destructor, volumes destroy themselves
-        if FLAGS.fake_storage:
-            try:
-                shutil.rmtree(FLAGS.aoe_export_dir)
-            except Exception, err:
-                pass
-
     @defer.inlineCallbacks
     @validate.rangetest(size=(0, 1000))
     def create_volume(self, size, user_id, project_id):
@@ -113,8 +98,6 @@
         """
         logging.debug("Creating volume of size: %s" % (size))
         vol = yield self.volume_class.create(size, user_id, project_id)
-        datastore.Redis.instance().sadd('volumes', vol['volume_id'])
-        datastore.Redis.instance().sadd('volumes:%s' % (FLAGS.storage_name), vol['volume_id'])
         logging.debug("restarting exports")
         yield self._restart_exports()
         defer.returnValue(vol['volume_id'])
@@ -134,21 +117,19 @@
     def delete_volume(self, volume_id):
         logging.debug("Deleting volume with id of: %s" % (volume_id))
         vol = get_volume(volume_id)
-        if vol['status'] == "attached":
+        if vol['attach_status'] == "attached":
             raise exception.Error("Volume is still attached")
-        if vol['node_name'] != FLAGS.storage_name:
+        if vol['node_name'] != FLAGS.node_name:
             raise exception.Error("Volume is not local to this node")
         yield vol.destroy()
-        datastore.Redis.instance().srem('volumes', vol['volume_id'])
-        datastore.Redis.instance().srem('volumes:%s' % (FLAGS.storage_name), vol['volume_id'])
         defer.returnValue(True)
 
     @defer.inlineCallbacks
     def _restart_exports(self):
         if FLAGS.fake_storage:
             return
-        yield process.simple_execute("sudo vblade-persist auto all")
-        # NOTE(vish): this command sometimes sends output to stderr for warnings
+        # NOTE(vish): these commands sometimes sends output to stderr for warnings
+        yield process.simple_execute("sudo vblade-persist auto all", error_ok=1)
         yield process.simple_execute("sudo vblade-persist start all", error_ok=1)
 
     @defer.inlineCallbacks
@@ -172,14 +153,15 @@
         return self.volume_id
 
     def default_state(self):
-        return {"volume_id": self.volume_id}
+        return {"volume_id": self.volume_id,
+                "node_name": "unassigned"}
 
     @classmethod
     @defer.inlineCallbacks
     def create(cls, size, user_id, project_id):
         volume_id = utils.generate_uid('vol')
         vol = cls(volume_id)
-        vol['node_name'] = FLAGS.storage_name
+        vol['node_name'] = FLAGS.node_name
         vol['size'] = size
         vol['user_id'] = user_id
         vol['project_id'] = project_id
@@ -225,14 +207,31 @@
         self['attach_status'] = "detached"
         self.save()
 
+    def save(self):
+        is_new = self.is_new_record()
+        super(Volume, self).save()
+        if is_new:
+            redis = datastore.Redis.instance()
+            key = self.__devices_key
+            # TODO(vish): these should be added by admin commands
+            more = redis.scard(self._redis_association_name("node",
+                                                            self['node_name']))
+            if (not redis.exists(key) and not more):
+                for shelf_id in range(FLAGS.first_shelf_id,
+                                      FLAGS.last_shelf_id + 1):
+                    for blade_id in range(FLAGS.blades_per_shelf):
+                        redis.sadd(key, "%s.%s" % (shelf_id, blade_id))
+            self.associate_with("node", self['node_name'])
+
     @defer.inlineCallbacks
     def destroy(self):
-        try:
-            yield self._remove_export()
-        except Exception as ex:
-            logging.debug("Ingnoring failure to remove export %s" % ex)
-            pass
+        yield self._remove_export()
         yield self._delete_lv()
+        self.unassociate_with("node", self['node_name'])
+        if self.get('shelf_id', None) and self.get('blade_id', None):
+            redis = datastore.Redis.instance()
+            key = self.__devices_key
+            redis.sadd(key, "%s.%s" % (self['shelf_id'], self['blade_id']))
         super(Volume, self).destroy()
 
     @defer.inlineCallbacks
@@ -244,66 +243,72 @@
         yield process.simple_execute(
                 "sudo lvcreate -L %s -n %s %s" % (sizestr,
                                                   self['volume_id'],
-                                                  FLAGS.volume_group))
+                                                  FLAGS.volume_group),
+                error_ok=1)
 
     @defer.inlineCallbacks
     def _delete_lv(self):
         yield process.simple_execute(
                 "sudo lvremove -f %s/%s" % (FLAGS.volume_group,
-                                            self['volume_id']))
+                                            self['volume_id']), error_ok=1)
+
+    @property
+    def __devices_key(self):
+        return 'volume_devices:%s' % FLAGS.node_name
 
     @defer.inlineCallbacks
     def _setup_export(self):
-        (shelf_id, blade_id) = get_next_aoe_numbers()
+        redis = datastore.Redis.instance()
+        key = self.__devices_key
+        device = redis.spop(key)
+        if not device:
+            raise NoMoreBlades()
+        (shelf_id, blade_id) = device.split('.')
         self['aoe_device'] = "e%s.%s" % (shelf_id, blade_id)
         self['shelf_id'] = shelf_id
         self['blade_id'] = blade_id
         self.save()
-        yield self._exec_export()
+        yield self._exec_setup_export()
 
     @defer.inlineCallbacks
-    def _exec_export(self):
+    def _exec_setup_export(self):
         yield process.simple_execute(
                 "sudo vblade-persist setup %s %s %s /dev/%s/%s" %
                 (self['shelf_id'],
                  self['blade_id'],
                  FLAGS.aoe_eth_dev,
                  FLAGS.volume_group,
-                 self['volume_id']))
+                 self['volume_id']), error_ok=1)
 
     @defer.inlineCallbacks
     def _remove_export(self):
+        if not self.get('shelf_id', None) or not self.get('blade_id', None):
+            defer.returnValue(False)
+        yield self._exec_remove_export()
+        defer.returnValue(True)
+
+    @defer.inlineCallbacks
+    def _exec_remove_export(self):
         yield process.simple_execute(
                 "sudo vblade-persist stop %s %s" % (self['shelf_id'],
-                                                    self['blade_id']))
+                                                    self['blade_id']), error_ok=1)
         yield process.simple_execute(
                 "sudo vblade-persist destroy %s %s" % (self['shelf_id'],
-                                                       self['blade_id']))
+                                                       self['blade_id']), error_ok=1)
+
 
 
 class FakeVolume(Volume):
     def _create_lv(self):
         pass
 
-    def _exec_export(self):
+    def _exec_setup_export(self):
         fname = os.path.join(FLAGS.aoe_export_dir, self['aoe_device'])
         f = file(fname, "w")
         f.close()
 
-    def _remove_export(self):
-        pass
+    def _exec_remove_export(self):
+        os.unlink(os.path.join(FLAGS.aoe_export_dir, self['aoe_device']))
 
     def _delete_lv(self):
         pass
-
-def get_next_aoe_numbers():
-    for shelf_id in xrange(FLAGS.first_shelf_id, FLAGS.last_shelf_id + 1):
-        aoes = glob.glob("%s/e%s.*" % (FLAGS.aoe_export_dir, shelf_id))
-        if not aoes:
-            blade_id = 0
-        else:
-            blade_id = int(max([int(a.rpartition('.')[2]) for a in aoes])) + 1
-        if blade_id < FLAGS.slots_per_shelf:
-            logging.debug("Next shelf.blade is %s.%s", shelf_id, blade_id)
-            return (shelf_id, blade_id)
-    raise NoMoreVolumes()