MAAS

Merge lp:~blake-rouse/maas/cache-cluster-boot-images into lp:~maas-committers/maas/trunk

  1. cache-cluster-boot-images
  2. Merge into trunk
Proposed by Blake Rouse
Status: Merged
Approved by: Blake Rouse
Approved revision: no longer in the source branch.
Merged at revision: 3430
Proposed branch: lp:~blake-rouse/maas/cache-cluster-boot-images
Merge into: lp:~maas-committers/maas/trunk
Diff against target: 176 lines (+73/-3)
4 files modified
src/maasserver/clusterrpc/tests/test_boot_images.py (+3/-0)
src/provisioningserver/rpc/boot_images.py (+25/-2)
src/provisioningserver/rpc/tests/test_boot_images.py (+43/-1)
src/provisioningserver/rpc/tests/test_clusterservice.py (+2/-0)
To merge this branch: bzr merge lp:~blake-rouse/maas/cache-cluster-boot-images
Reviewer Review Type Date Requested Status
Jason Hobbs (community) Approve
Review via email: mp+244754@code.launchpad.net

Commit message

Cache the list of boot images on the cluster. Instead of reading from the disk on every request of the boot images, it is only updated when the contents of the disk have changed from an import.

To post a comment you must log in.
Revision history for this message
Jason Hobbs (jason-hobbs) wrote :

lgtm.

review: Approve
Revision history for this message
MAAS Lander (maas-lander) wrote :
Download full text (24.6 KiB)

The attempt to merge lp:~blake-rouse/maas/cache-cluster-boot-images into lp:maas failed. Below is the output from the failed tests.

Ign http://security.ubuntu.com trusty-security InRelease
Hit http://security.ubuntu.com trusty-security Release.gpg
Ign http://nova.clouds.archive.ubuntu.com trusty InRelease
Hit http://security.ubuntu.com trusty-security Release
Ign http://nova.clouds.archive.ubuntu.com trusty-updates InRelease
Hit http://nova.clouds.archive.ubuntu.com trusty Release.gpg
Get:1 http://nova.clouds.archive.ubuntu.com trusty-updates Release.gpg [933 B]
Hit http://nova.clouds.archive.ubuntu.com trusty Release
Get:2 http://nova.clouds.archive.ubuntu.com trusty-updates Release [62.0 kB]
Hit http://security.ubuntu.com trusty-security/main Sources
Hit http://security.ubuntu.com trusty-security/universe Sources
Hit http://nova.clouds.archive.ubuntu.com trusty/main Sources
Hit http://security.ubuntu.com trusty-security/main amd64 Packages
Hit http://nova.clouds.archive.ubuntu.com trusty/universe Sources
Hit http://security.ubuntu.com trusty-security/universe amd64 Packages
Hit http://nova.clouds.archive.ubuntu.com trusty/main amd64 Packages
Hit http://security.ubuntu.com trusty-security/main Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty/universe amd64 Packages
Hit http://security.ubuntu.com trusty-security/universe Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty/main Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty/universe Translation-en
Ign http://nova.clouds.archive.ubuntu.com trusty/main Translation-en_US
Ign http://nova.clouds.archive.ubuntu.com trusty/universe Translation-en_US
Get:3 http://nova.clouds.archive.ubuntu.com trusty-updates/main Sources [148 kB]
Get:4 http://nova.clouds.archive.ubuntu.com trusty-updates/universe Sources [94.7 kB]
Get:5 http://nova.clouds.archive.ubuntu.com trusty-updates/main amd64 Packages [384 kB]
Get:6 http://nova.clouds.archive.ubuntu.com trusty-updates/universe amd64 Packages [227 kB]
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/main Translation-en
Hit http://nova.clouds.archive.ubuntu.com trusty-updates/universe Translation-en
Fetched 917 kB in 2s (321 kB/s)
Reading package lists...
sudo DEBIAN_FRONTEND=noninteractive apt-get -y \
     --no-install-recommends install apache2 authbind bind9 bind9utils build-essential bzr-builddeb curl daemontools debhelper dh-apport distro-info dnsutils firefox freeipmi-tools gjs ipython isc-dhcp-common libjs-raphael libjs-yui3-full libjs-yui3-min libpq-dev make pep8 postgresql pyflakes python-amqplib python-bzrlib python-celery python-convoy python-crochet python-cssselect python-curtin python-dev python-distro-info python-django python-django-piston python-django-south python-djorm-ext-pgarray python-docutils python-extras python-fixtures python-flake8 python-formencode python-hivex python-httplib2 python-jinja2 python-jsonschema python-lockfile python-lxml python-mimeparse python-mock python-netaddr python-netifaces python-nose python-oauth python-oops python-oops-amqp python-oops-datedir-repo python-oops-twisted python-oops-wsgi python-openssl python-paramiko python-pexpect python-pip python-pocket-lin...

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'src/maasserver/clusterrpc/tests/test_boot_images.py'
--- src/maasserver/clusterrpc/tests/test_boot_images.py 2014-10-30 13:39:50 +0000
+++ src/maasserver/clusterrpc/tests/test_boot_images.py 2014-12-16 18:59:29 +0000
@@ -144,6 +144,7 @@
144 resource_dir = self.make_dir()144 resource_dir = self.make_dir()
145 self.tftproot = os.path.join(resource_dir, 'current')145 self.tftproot = os.path.join(resource_dir, 'current')
146 os.mkdir(self.tftproot)146 os.mkdir(self.tftproot)
147 self.patch(boot_images, 'CACHED_BOOT_IMAGES', None)
147 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', resource_dir)148 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', resource_dir)
148149
149 def test_returns_boot_images(self):150 def test_returns_boot_images(self):
@@ -172,6 +173,7 @@
172 resource_dir = self.make_dir()173 resource_dir = self.make_dir()
173 self.tftproot = os.path.join(resource_dir, 'current')174 self.tftproot = os.path.join(resource_dir, 'current')
174 os.mkdir(self.tftproot)175 os.mkdir(self.tftproot)
176 self.patch(boot_images, 'CACHED_BOOT_IMAGES', None)
175 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', resource_dir)177 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', resource_dir)
176178
177 def test_returns_boot_images_for_one_cluster(self):179 def test_returns_boot_images_for_one_cluster(self):
@@ -261,6 +263,7 @@
261 resource_dir = self.make_dir()263 resource_dir = self.make_dir()
262 self.tftproot = os.path.join(resource_dir, 'current')264 self.tftproot = os.path.join(resource_dir, 'current')
263 os.mkdir(self.tftproot)265 os.mkdir(self.tftproot)
266 self.patch(boot_images, 'CACHED_BOOT_IMAGES', None)
264 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', resource_dir)267 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', resource_dir)
265268
266 def make_boot_images(self):269 def make_boot_images(self):
267270
=== modified file 'src/provisioningserver/rpc/boot_images.py'
--- src/provisioningserver/rpc/boot_images.py 2014-10-30 13:17:51 +0000
+++ src/provisioningserver/rpc/boot_images.py 2014-12-16 18:59:29 +0000
@@ -31,9 +31,28 @@
31from twisted.internet.threads import deferToThread31from twisted.internet.threads import deferToThread
3232
3333
34CACHED_BOOT_IMAGES = None
35
36
34def list_boot_images():37def list_boot_images():
35 """List the boot images that exist on the cluster."""38 """List the boot images that exist on the cluster.
36 return tftppath.list_boot_images(39
40 This return value of this function is cached. This helps reduce the amount
41 of IO, as this function is called often. To update the cache call
42 `reload_boot_images`.
43 """
44 global CACHED_BOOT_IMAGES
45 if CACHED_BOOT_IMAGES is None:
46 CACHED_BOOT_IMAGES = tftppath.list_boot_images(
47 os.path.join(BOOT_RESOURCES_STORAGE, 'current'))
48 return CACHED_BOOT_IMAGES
49
50
51def reload_boot_images():
52 """Update the cached boot images so `list_boot_images` returns the
53 most up-to-date boot images list."""
54 global CACHED_BOOT_IMAGES
55 CACHED_BOOT_IMAGES = tftppath.list_boot_images(
37 os.path.join(BOOT_RESOURCES_STORAGE, 'current'))56 os.path.join(BOOT_RESOURCES_STORAGE, 'current'))
3857
3958
@@ -67,6 +86,10 @@
67 with environment_variables(variables):86 with environment_variables(variables):
68 boot_resources.import_images(sources)87 boot_resources.import_images(sources)
6988
89 # Update the boot images cache so `list_boot_images` returns the
90 # correct information.
91 reload_boot_images()
92
7093
71def import_boot_images(sources, http_proxy=None, https_proxy=None):94def import_boot_images(sources, http_proxy=None, https_proxy=None):
72 """Imports the boot images from the given sources."""95 """Imports the boot images from the given sources."""
7396
=== modified file 'src/provisioningserver/rpc/tests/test_boot_images.py'
--- src/provisioningserver/rpc/tests/test_boot_images.py 2014-10-30 13:17:51 +0000
+++ src/provisioningserver/rpc/tests/test_boot_images.py 2014-12-16 18:59:29 +0000
@@ -23,7 +23,10 @@
23 MockNotCalled,23 MockNotCalled,
24 )24 )
25from maastesting.testcase import MAASTwistedRunTest25from maastesting.testcase import MAASTwistedRunTest
26from mock import sentinel26from mock import (
27 ANY,
28 sentinel,
29 )
27from provisioningserver import concurrency30from provisioningserver import concurrency
28from provisioningserver.boot import tftppath31from provisioningserver.boot import tftppath
29from provisioningserver.config import BOOT_RESOURCES_STORAGE32from provisioningserver.config import BOOT_RESOURCES_STORAGE
@@ -35,6 +38,7 @@
35 import_boot_images,38 import_boot_images,
36 is_import_boot_images_running,39 is_import_boot_images_running,
37 list_boot_images,40 list_boot_images,
41 reload_boot_images,
38 )42 )
39from provisioningserver.testing.config import BootSourcesFixture43from provisioningserver.testing.config import BootSourcesFixture
40from provisioningserver.testing.testcase import PservTestCase44from provisioningserver.testing.testcase import PservTestCase
@@ -61,6 +65,7 @@
61class TestListBootImages(PservTestCase):65class TestListBootImages(PservTestCase):
6266
63 def test__calls_list_boot_images_with_boot_resource_storage(self):67 def test__calls_list_boot_images_with_boot_resource_storage(self):
68 self.patch(boot_images, 'CACHED_BOOT_IMAGES', None)
64 mock_list_boot_images = self.patch(tftppath, 'list_boot_images')69 mock_list_boot_images = self.patch(tftppath, 'list_boot_images')
65 list_boot_images()70 list_boot_images()
66 self.assertThat(71 self.assertThat(
@@ -68,6 +73,36 @@
68 MockCalledOnceWith(73 MockCalledOnceWith(
69 os.path.join(BOOT_RESOURCES_STORAGE, "current")))74 os.path.join(BOOT_RESOURCES_STORAGE, "current")))
7075
76 def test__calls_list_boot_images_when_cache_is_None(self):
77 self.patch(boot_images, 'CACHED_BOOT_IMAGES', None)
78 mock_list_boot_images = self.patch(tftppath, 'list_boot_images')
79 list_boot_images()
80 self.assertThat(
81 mock_list_boot_images,
82 MockCalledOnceWith(ANY))
83
84 def test__doesnt_call_list_boot_images_when_cache_is_not_None(self):
85 fake_boot_images = [factory.make_name('image') for _ in range(3)]
86 self.patch(boot_images, 'CACHED_BOOT_IMAGES', fake_boot_images)
87 mock_list_boot_images = self.patch(tftppath, 'list_boot_images')
88 self.expectThat(list_boot_images(), Equals(fake_boot_images))
89 self.expectThat(
90 mock_list_boot_images,
91 MockNotCalled())
92
93
94class TestReloadBootImages(PservTestCase):
95
96 def test__sets_CACHED_BOOT_IMAGES(self):
97 self.patch(
98 boot_images, 'CACHED_BOOT_IMAGES', factory.make_name('old_cache'))
99 fake_boot_images = [factory.make_name('image') for _ in range(3)]
100 mock_list_boot_images = self.patch(tftppath, 'list_boot_images')
101 mock_list_boot_images.return_value = fake_boot_images
102 reload_boot_images()
103 self.assertEquals(
104 boot_images.CACHED_BOOT_IMAGES, fake_boot_images)
105
71106
72class TestGetHostsFromSources(PservTestCase):107class TestGetHostsFromSources(PservTestCase):
73108
@@ -141,6 +176,13 @@
141 _run_import(sources=sources)176 _run_import(sources=sources)
142 self.assertThat(fake, MockCalledOnceWith(sources))177 self.assertThat(fake, MockCalledOnceWith(sources))
143178
179 def test__run_import_calls_reload_boot_images(self):
180 fake_reload = self.patch(boot_images, 'reload_boot_images')
181 self.patch(boot_resources, 'import_images')
182 sources, _ = make_sources()
183 _run_import(sources=sources)
184 self.assertThat(fake_reload, MockCalledOnceWith())
185
144186
145class TestImportBootImages(PservTestCase):187class TestImportBootImages(PservTestCase):
146188
147189
=== modified file 'src/provisioningserver/rpc/tests/test_clusterservice.py'
--- src/provisioningserver/rpc/tests/test_clusterservice.py 2014-12-12 00:56:22 +0000
+++ src/provisioningserver/rpc/tests/test_clusterservice.py 2014-12-16 18:59:29 +0000
@@ -227,6 +227,7 @@
227227
228 @inlineCallbacks228 @inlineCallbacks
229 def test_list_boot_images_can_be_called(self):229 def test_list_boot_images_can_be_called(self):
230 self.patch(boot_images, 'CACHED_BOOT_IMAGES', None)
230 list_boot_images = self.patch(tftppath, "list_boot_images")231 list_boot_images = self.patch(tftppath, "list_boot_images")
231 list_boot_images.return_value = []232 list_boot_images.return_value = []
232233
@@ -256,6 +257,7 @@
256 os.makedirs(os.path.join(current_dir, *options))257 os.makedirs(os.path.join(current_dir, *options))
257 make_osystem(self, options[0], purposes)258 make_osystem(self, options[0], purposes)
258 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', tftpdir)259 self.patch(boot_images, 'BOOT_RESOURCES_STORAGE', tftpdir)
260 self.patch(boot_images, 'CACHED_BOOT_IMAGES', None)
259261
260 expected_images = [262 expected_images = [
261 {263 {