Merge lp:~fwereade/pyjuju/webdav-storage into lp:pyjuju

[William Reade]

WebDAV FileStorage class now exists, and is hooked up to the orchestra MachineProvider; as a bonus, so are save_state and load_state, because they're trivial now.

NOTE: also depends on lp:~fwereade/ensemble/generic-state-ops

[lp:~fwereade/pyjuju/webdav-storage]

291. By William Reade on 2011-07-31

merge from lp:~fwereade/ensemble/generic-state-ops

292. By William Reade on 2011-07-31

merge and fix from lp:~fwereade/ensemble/cobbler-instance-ids

[Gustavo Niemeyer (niemeyer)]

Very nice code as well, thanks. +1, with one constraint:

[1]

The new entry points in orchestra/__init__.py need some quick
test coverage to ensure they basically work at least.

review: Approve

[lp:~fwereade/pyjuju/webdav-storage]

293. By William Reade on 2011-08-02

storage-url is no longer mandatory in config; if not present, assume %(orchestra-server)s/webdav

294. By William Reade on 2011-08-02

merge lp:~fwereade/ensemble/generic-state-ops

295. By William Reade on 2011-08-02

merge lp:~fwereade/ensemble/cobbler-instance-ids

296. By William Reade on 2011-08-02

per review, direct verification tests for the happy path of orchestra.MachineProvider's .save_state and .load_state

297. By William Reade on 2011-08-03

merge trunk

[Gustavo Niemeyer (niemeyer)]

William, if you are reading this without a second review, please go ahead and merge it.

The branch looks good, and has been waiting for the second review for long enough.

review: Approve

=== modified file 'ensemble/environment/config.py'
--- ensemble/environment/config.py	2011-07-20 23:32:19 +0000
+++ ensemble/environment/config.py	2011-08-03 06:46:34 +0000
@@ -50,7 +50,9 @@
                               "orchestra-pass": String(),
                               "admin-secret": String(),
                               "acquired-mgmt-class": String(),
-                              "available-mgmt-class": String()}),
+                              "available-mgmt-class": String(),
+                              "storage-url": String()},
+                             optional=["storage-url"]),
         "dummy": KeyDict({})}))},
     optional=["default"])
 

=== modified file 'ensemble/environment/tests/test_config.py'
--- ensemble/environment/tests/test_config.py	2011-07-21 08:15:51 +0000
+++ ensemble/environment/tests/test_config.py	2011-08-03 06:46:34 +0000
@@ -36,6 +36,7 @@
     admin-secret: garden
     acquired-mgmt-class: acquired
     available-mgmt-class: available
+    storage-url: http://somewhereel.se
 """
 
 class EnvironmentsConfigTestBase(TestCase):
@@ -554,7 +555,7 @@
     def test_orchestra_schema_requires(self):
         requires = (
             "type orchestra-server orchestra-user orchestra-pass "
-            "admin-secret acquired-mgmt-class available-mgmt-class").split()
+            "admin-secret acquired-mgmt-class available-mgmt-class ").split()
         for require in requires:
             config = yaml.load(SAMPLE_ORCHESTRA)
             del config["environments"]["sample"][require]

=== modified file 'ensemble/providers/orchestra/__init__.py'
--- ensemble/providers/orchestra/__init__.py	2011-07-20 07:49:20 +0000
+++ ensemble/providers/orchestra/__init__.py	2011-08-03 06:46:34 +0000
@@ -1,9 +1,11 @@
-from twisted.internet.defer import fail, succeed
+from twisted.internet.defer import fail
 
 from ensemble.errors import EnvironmentNotFound
 from ensemble.providers.common.bootstrap import Bootstrap
+from ensemble.providers.common.state import SaveState, LoadState
 
 from .cobbler import CobblerConnect
+from .files import FileStorage
 from .launch import OrchestraLaunchMachine
 
 
@@ -70,23 +72,27 @@
         raise NotImplementedError()
 
     def save_state(self, state):
-        """
-        Save state to the provider.
+        """Save state to the provider.
 
         @param state
         @type dict
         """
-        return succeed(None)
+        serializer = SaveState(self)
+        return serializer.run(state)
 
     def load_state(self):
-        """
-        Load state from the provider, returns a dictionary.
-        """
-        raise NotImplementedError()
+        """Load state from the provider.
+
+        @return: a dictionary.
+        """
+        deserializer = LoadState(self)
+        return deserializer.run()
 
     def get_file_storage(self):
         """Retrieve the provider C{FileStorage} abstraction."""
-        raise NotImplementedError()
+        if "storage-url" not in self.config:
+            return FileStorage("http://%(orchestra-server)s/webdav" % self.config)
+        return FileStorage(self.config["storage-url"])
 
     def get_serialization_data(self):
         """Return a dictionary serialization of the provider configuration.
@@ -124,14 +130,3 @@
         @raise: EnvironmentNotFound or EnvironmentPending
         """
         return fail(EnvironmentNotFound())
-
-
-class FileStorage(object):
-
-    def get(self, name):
-        """Returns open file or fails with FileNotFound"""
-        raise NotImplementedError()
-
-    def put(self, remote_path, local_path):
-        """Copies contents of local_path to remote_path"""
-        raise NotImplementedError()

=== added file 'ensemble/providers/orchestra/files.py'
--- ensemble/providers/orchestra/files.py	1970-01-01 00:00:00 +0000
+++ ensemble/providers/orchestra/files.py	2011-08-03 06:46:34 +0000
@@ -0,0 +1,35 @@
+from cStringIO import StringIO
+
+from twisted.web.client import getPage
+from twisted.web.error import Error
+
+from ensemble.errors import FileNotFound
+
+
+class FileStorage(object):
+
+    def __init__(self, base_url):
+        self._base_url = base_url
+
+    def _key_to_url(self, name):
+        return "%s/%s" % (self._base_url, name)
+
+    def get(self, name):
+        url = self._key_to_url(name)
+        d = getPage(url)
+        d.addCallback(StringIO)
+
+        def convert_404(failure):
+            failure.trap(Error)
+            if failure.value.status == "404":
+                raise FileNotFound(url)
+            return failure
+        d.addErrback(convert_404)
+        return d
+
+    def put(self, remote_path, file_object):
+        url = self._key_to_url(remote_path)
+        data = file_object.read()
+        d = getPage(url, method="PUT", postdata=data)
+        d.addCallback(lambda _: True)
+        return d

=== modified file 'ensemble/providers/orchestra/tests/test_bootstrap.py'
--- ensemble/providers/orchestra/tests/test_bootstrap.py	2011-08-02 08:07:12 +0000
+++ ensemble/providers/orchestra/tests/test_bootstrap.py	2011-08-03 06:46:34 +0000
@@ -1,4 +1,5 @@
 from xmlrpclib import Fault
+from yaml import dump
 
 from twisted.internet.defer import fail, succeed
 from twisted.web.xmlrpc import Proxy
@@ -14,6 +15,8 @@
           "orchestra-pass": "pass",
           "acquired-mgmt-class": "acquired",
           "available-mgmt-class": "available",
+          "admin-secret": "SEEKRIT",
+          "storage-url": "http://somewhe.re/webdav",
           "admin-secret": "SEEKRIT"}
 
 
@@ -92,6 +95,12 @@
             "TOKEN")
         self.mocker.result(succeed("[some-timestamp]_power"))
 
+    def mock_save_state(self):
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/provider-state",
+                method="PUT", postdata=dump({"zookeeper-instances": ["winston-uid"]}))
+        self.mocker.result(succeed(None))
+
     def test_no_machines_available(self):
         self.mock_proxy()
         self.mock_get_systems(acceptable=False)
@@ -137,6 +146,7 @@
         self.mock_acquire_system()
         self.mock_set_ks_meta()
         self.mock_start_system()
+        self.mock_save_state()
         self.mocker.replay()
 
         provider = MachineProvider("tetrascape", CONFIG)

=== added file 'ensemble/providers/orchestra/tests/test_files.py'
--- ensemble/providers/orchestra/tests/test_files.py	1970-01-01 00:00:00 +0000
+++ ensemble/providers/orchestra/tests/test_files.py	2011-08-03 06:46:34 +0000
@@ -0,0 +1,101 @@
+from cStringIO import StringIO
+
+from twisted.internet.defer import fail, succeed
+from twisted.web.error import Error
+
+from ensemble.errors import FileNotFound
+from ensemble.lib.testing import TestCase
+from ensemble.providers.orchestra import MachineProvider
+
+
+def get_file_storage(with_storage_url=True):
+    config = {"storage-url": "http://somewhe.re/webdav",
+              "orchestra-server": "somewhereel.se",
+              "orchestra-user": "user",
+              "orchestra-pass": "pass",
+              "acquired-mgmt-class": "acquired",
+              "available-mgmt-class": "available"}
+    if not with_storage_url:
+        del config["storage-url"]
+    provider = MachineProvider("blah", config)
+    return provider.get_file_storage()
+
+
+class FileStorageTest(TestCase):
+
+    def test_get_works_no_storage_url(self):
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhereel.se/webdav/rubber/chicken")
+        self.mocker.result(succeed("pulley"))
+        self.mocker.replay()
+
+        fs = get_file_storage(with_storage_url=False)
+        d = fs.get("rubber/chicken")
+
+        def verify(result):
+            self.assertEquals(result.read(), "pulley")
+        d.addCallback(verify)
+        return d
+
+    def test_get_works(self):
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/rubber/chicken")
+        self.mocker.result(succeed("pulley"))
+        self.mocker.replay()
+
+        fs = get_file_storage()
+        d = fs.get("rubber/chicken")
+
+        def verify(result):
+            self.assertEquals(result.read(), "pulley")
+        d.addCallback(verify)
+        return d
+
+    def test_get_fails(self):
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/rubber/chicken")
+        self.mocker.result(fail(Error("404")))
+        self.mocker.replay()
+
+        fs = get_file_storage()
+        d = fs.get("rubber/chicken")
+        self.assertFailure(d, FileNotFound)
+        return d
+
+    def test_get_errors(self):
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/rubber/chicken")
+        self.mocker.result(fail(Error("500")))
+        self.mocker.replay()
+
+        fs = get_file_storage()
+        d = fs.get("rubber/chicken")
+        self.assertFailure(d, Error)
+        return d
+
+    def test_put_works(self):
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/rubber/chicken",
+                method="PUT", postdata="pulley")
+        self.mocker.result(succeed(None))
+        self.mocker.replay()
+
+        fs = get_file_storage()
+        d = fs.put("rubber/chicken", StringIO("pulley"))
+
+        def verify(result):
+            self.assertEquals(result, True)
+        d.addCallback(verify)
+        return d
+
+    def test_put_errors(self):
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/rubber/chicken",
+                method="PUT", postdata="pulley")
+        self.mocker.result(fail(Error("500")))
+        self.mocker.replay()
+
+        fs = get_file_storage()
+        d = fs.put("rubber/chicken", StringIO("pulley"))
+        self.assertFailure(d, Error)
+        return d

=== modified file 'ensemble/providers/orchestra/tests/test_provider.py'
--- ensemble/providers/orchestra/tests/test_provider.py	2011-07-30 11:50:10 +0000
+++ ensemble/providers/orchestra/tests/test_provider.py	2011-08-03 06:46:34 +0000
@@ -15,5 +15,3 @@
             "tetrascape", CONFIG)
         self.assertEquals(provider.environment_name, "tetrascape")
         self.assertEquals(provider.config, CONFIG)
-
-

=== added file 'ensemble/providers/orchestra/tests/test_state.py'
--- ensemble/providers/orchestra/tests/test_state.py	1970-01-01 00:00:00 +0000
+++ ensemble/providers/orchestra/tests/test_state.py	2011-08-03 06:46:34 +0000
@@ -0,0 +1,49 @@
+from yaml import dump
+
+from twisted.internet.defer import succeed
+
+from ensemble.lib.testing import TestCase
+from ensemble.providers.orchestra import MachineProvider
+
+
+def get_provider():
+    config = {"orchestra-server": "somewhe.re",
+              "orchestra-user": "user",
+              "orchestra-pass": "pass",
+              "acquired-mgmt-class": "acquired",
+              "available-mgmt-class": "available"}
+    return MachineProvider("tetrascape", config)
+
+
+class StateTest(TestCase):
+
+    def test_save(self):
+        state = {"foo": "blah blah"}
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/provider-state",
+                method="PUT", postdata=dump(state))
+        self.mocker.result(succeed(None))
+        self.mocker.replay()
+
+        provider = get_provider()
+        d = provider.save_state(state)
+
+        def verify(result):
+            self.assertEquals(result, True)
+        d.addCallback(verify)
+        return d
+
+    def test_load(self):
+        expect_state = {"foo": "blah blah"}
+        getPage = self.mocker.replace("twisted.web.client.getPage")
+        getPage("http://somewhe.re/webdav/provider-state")
+        self.mocker.result(succeed(dump(expect_state)))
+        self.mocker.replay()
+
+        provider = get_provider()
+        d = provider.load_state()
+
+        def verify(state):
+            self.assertEquals(state, expect_state)
+        d.addCallback(verify)
+        return d