Merge lp:~hazmat/txaws/fix-s3-port-and-bucket-op into lp:txaws

[Kapil Thangavelu]

This builds on clint's s3-port-fix branch to solve two other openstack compatibility issues.

- Bucket/collection operations need to have the bucket urls with a '/' appended. Confirmed this also done by boto.

- Security group descriptions may omit port information. This is a workaround for nova bug: 829609.
  This change revents txaws from blowing up an trying to turn a None value into an integer.

[Thomas Herve (therve)]

Looks good, +1!

review: Approve

=== modified file 'txaws/ec2/client.py'
--- txaws/ec2/client.py	2011-05-12 14:48:26 +0000
+++ txaws/ec2/client.py	2011-08-29 20:27:23 +0000
@@ -658,9 +658,20 @@
             if ip_permissions is None:
                 ip_permissions = ()
             for ip_permission in ip_permissions:
+
+                # openstack doesn't handle self authorized groups properly
+                # XXX this is an upstream problem and should be addressed there
+                # lp bug #829609
                 ip_protocol = ip_permission.findtext("ipProtocol")
-                from_port = int(ip_permission.findtext("fromPort"))
-                to_port = int(ip_permission.findtext("toPort"))
+                from_port = ip_permission.findtext("fromPort")
+                to_port = ip_permission.findtext("toPort")
+
+                if from_port:
+                    from_port = int(from_port)
+
+                if to_port:
+                    to_port = int(to_port)
+
                 for groups in ip_permission.findall("groups/item") or ():
                     user_id = groups.findtext("userId")
                     group_name = groups.findtext("groupName")

=== modified file 'txaws/ec2/tests/test_client.py'
--- txaws/ec2/tests/test_client.py	2011-05-12 14:38:37 +0000
+++ txaws/ec2/tests/test_client.py	2011-08-29 20:27:23 +0000
@@ -522,6 +522,38 @@
         d = ec2.describe_security_groups("WebServers")
         return d.addCallback(check_result)
 
+    def test_describe_security_groups_with_openstack(self):
+        """
+        L{EC2Client.describe_security_groups} can work with openstack
+        responses, which may lack proper port information for
+        self-referencing group. Verifying that the response doesn't
+        cause an internal error, workaround for nova launchpad bug
+        #829609.
+        """
+        class StubQuery(object):
+
+            def __init__(stub, action="", creds=None, endpoint=None,
+                         other_params={}):
+                self.assertEqual(action, "DescribeSecurityGroups")
+                self.assertEqual(creds.access_key, "foo")
+                self.assertEqual(creds.secret_key, "bar")
+                self.assertEqual(other_params, {"GroupName.1": "WebServers"})
+
+            def submit(self):
+                return succeed(
+                    payload.sample_describe_security_groups_with_openstack)
+
+        def check_result(security_groups):
+            [security_group] = security_groups
+            self.assertEquals(security_group.name, "WebServers")
+            self.assertEqual(
+                security_group.allowed_groups[0].group_name, "WebServers")
+
+        creds = AWSCredentials("foo", "bar")
+        ec2 = client.EC2Client(creds, query_factory=StubQuery)
+        d = ec2.describe_security_groups("WebServers")
+        return d.addCallback(check_result)
+
     def test_create_security_group(self):
         """
         L{EC2Client.create_security_group} returns a C{Deferred} that

=== modified file 'txaws/s3/client.py'
--- txaws/s3/client.py	2011-08-29 20:27:23 +0000
+++ txaws/s3/client.py	2011-08-29 20:27:23 +0000
@@ -54,6 +54,8 @@
             if not self.object_name.startswith("/"):
                 path += "/"
             path += self.object_name
+        elif self.bucket is not None and not path.endswith("/"):
+            path += "/"
         return path
 
     def get_url(self):

=== modified file 'txaws/s3/tests/test_client.py'
--- txaws/s3/tests/test_client.py	2011-08-29 20:27:23 +0000
+++ txaws/s3/tests/test_client.py	2011-08-29 20:27:23 +0000
@@ -34,7 +34,7 @@
 
     def test_get_path_with_bucket(self):
         url_context = client.URLContext(self.endpoint, bucket="mystuff")
-        self.assertEquals(url_context.get_path(), "/mystuff")
+        self.assertEquals(url_context.get_path(), "/mystuff/")
 
     def test_get_path_with_bucket_and_object(self):
         url_context = client.URLContext(

=== modified file 'txaws/testing/payload.py'
--- txaws/testing/payload.py	2011-03-26 12:40:36 +0000
+++ txaws/testing/payload.py	2011-08-29 20:27:23 +0000
@@ -178,6 +178,43 @@
 """ % (version.ec2_api,)
 
 
+sample_describe_security_groups_with_openstack = """\
+<?xml version="1.0"?>
+<DescribeSecurityGroupsResponse xmlns="http://ec2.amazonaws.com/doc/%s/">
+  <requestId>7d4e4dbd-0a33-4d3a-864a-b5ce0f1c9cbf</requestId>
+  <securityGroupInfo>
+    <item>
+      <ipPermissions>
+        <item>
+          <toPort>22</toPort>
+          <ipProtocol>tcp</ipProtocol>
+          <ipRanges>
+             <item><cidrIp>0.0.0.0/0</cidrIp></item>
+          </ipRanges>
+          <groups/>
+          <fromPort>22</fromPort>
+        </item>
+        <item>
+         <toPort/>
+         <ipProtocol/>
+         <ipRanges/>
+         <groups>
+            <item>
+              <groupName>WebServers</groupName>
+              <userId>UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM</userId>
+            </item>
+         </groups>
+         <fromPort/>
+        </item>
+      </ipPermissions>
+      <groupName>WebServers</groupName>
+      <groupDescription>Web servers</groupDescription>
+      <ownerId>UYY3TLBUXIEON5NQVUUX6OMPWBZIQNFM</ownerId>
+    </item>
+  </securityGroupInfo>
+</DescribeSecurityGroupsResponse>
+""" % (version.ec2_api,)
+
 sample_describe_security_groups_result = """\
 <?xml version="1.0"?>
 <DescribeSecurityGroupsResponse xmlns="http://ec2.amazonaws.com/doc/%s/">