Merge lp:~ltrager/maas/lp1554636_2.0 into lp:maas/2.0

Proposed by Lee Trager on 2016-11-02
Status: Merged
Approved by: Lee Trager on 2016-11-02
Approved revision: 5203
Merged at revision: 5203
Proposed branch: lp:~ltrager/maas/lp1554636_2.0
Merge into: lp:maas/2.0
Diff against target: 256 lines (+178/-7)
2 files modified
src/provisioningserver/import_images/boot_resources.py (+23/-6)
src/provisioningserver/import_images/tests/test_boot_resources.py (+155/-1)
To merge this branch: bzr merge lp:~ltrager/maas/lp1554636_2.0
Reviewer Review Type Date Requested Status
Lee Trager Approve on 2016-11-02
Review via email: mp+309880@code.launchpad.net

Commit message

Update iSCSI targets on every import (in case last time failed to update).

To post a comment you must log in.
MAAS Lander (maas-lander) wrote :
Download full text (1.3 MiB)

The attempt to merge lp:~ltrager/maas/lp1554636_2.0 into lp:maas/2.0 failed. Below is the output from the failed tests.

Hit:1 http://prodstack-zone-2.clouds.archive.ubuntu.com/ubuntu xenial InRelease
Get:2 http://security.ubuntu.com/ubuntu xenial-security InRelease [94.5 kB]
Get:3 http://prodstack-zone-2.clouds.archive.ubuntu.com/ubuntu xenial-updates InRelease [95.7 kB]
Hit:4 http://prodstack-zone-2.clouds.archive.ubuntu.com/ubuntu xenial-backports InRelease
Get:5 http://prodstack-zone-2.clouds.archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages [417 kB]
Get:6 http://prodstack-zone-2.clouds.archive.ubuntu.com/ubuntu xenial-updates/universe amd64 Packages [357 kB]
Fetched 964 kB in 2s (476 kB/s)
Reading package lists...
sudo DEBIAN_FRONTEND=noninteractive apt-get -y \
    --no-install-recommends install apache2 archdetect-deb authbind bash bind9 bind9utils build-essential bzr bzr-builddeb chromium-browser chromium-chromedriver curl daemontools debhelper dh-apport dh-systemd distro-info dnsutils firefox freeipmi-tools git gjs ipython isc-dhcp-common libjs-angularjs libjs-jquery libjs-jquery-hotkeys libjs-yui3-full libjs-yui3-min libpq-dev make nodejs-legacy npm postgresql pxelinux python3-all python3-apt python3-bson python3-convoy python3-crochet python3-cssselect python3-curtin python3-dev python3-distro-info python3-django python3-django-nose python3-django-piston3 python3-dnspython python3-docutils python3-formencode python3-hivex python3-httplib2 python3-jinja2 python3-jsonschema python3-lxml python3-netaddr python3-netifaces python3-novaclient python3-oauth python3-oauthlib python3-openssl python3-paramiko python3-petname python3-pexpect python3-psycopg2 python3-pyinotify python3-pyparsing python3-pyvmomi python3-requests python3-seamicroclient python3-setuptools python3-simplestreams python3-sphinx python3-tempita python3-twisted python3-txtftp python3-tz python3-yaml python3-zope.interface python-bson python-crochet python-django python-django-piston python-djorm-ext-pgarray python-formencode python-lxml python-netaddr python-netifaces python-pocket-lint python-psycopg2 python-simplejson python-tempita python-twisted python-yaml socat syslinux-common tgt ubuntu-cloudimage-keyring wget xvfb
Reading package lists...
Building dependency tree...
Reading state information...
authbind is already the newest version (2.1.1+nmu1).
build-essential is already the newest version (12.1ubuntu2).
debhelper is already the newest version (9.20160115ubuntu3).
distro-info is already the newest version (0.14build1).
freeipmi-tools is already the newest version (1.4.11-1ubuntu1).
git is already the newest version (1:2.7.4-0ubuntu1).
libjs-angularjs is already the newest version (1.2.28-1ubuntu2).
libjs-jquery is already the newest version (1.11.3+dfsg-4).
libjs-yui3-full is already the newest version (3.5.1-1ubuntu3).
libjs-yui3-min is already the newest version (3.5.1-1ubuntu3).
make is already the newest version (4.1-6).
postgresql is already the newest version (9.5+173).
pxelinux is already the newest version (3:6.03+dfsg-11ubuntu1).
python-formencode is already the newest version (1.3.0-0ubuntu5).
python-lxml is already the newest version (3....

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'src/provisioningserver/import_images/boot_resources.py'
--- src/provisioningserver/import_images/boot_resources.py 2016-07-07 21:57:57 +0000
+++ src/provisioningserver/import_images/boot_resources.py 2016-11-02 18:09:22 +0000
@@ -235,6 +235,11 @@
235 return BootSources.parse(StringIO(sources_yaml))235 return BootSources.parse(StringIO(sources_yaml))
236236
237237
238def update_iscsi_targets(snapshot_path):
239 maaslog.info("Updating boot image iSCSI targets.")
240 update_targets_conf(snapshot_path)
241
242
238def import_images(sources):243def import_images(sources):
239 """Import images. Callable from the command line.244 """Import images. Callable from the command line.
240245
@@ -246,6 +251,9 @@
246 maaslog.warning("Can't import: region did not provide a source.")251 maaslog.warning("Can't import: region did not provide a source.")
247 return False252 return False
248253
254 with ClusterConfiguration.open() as config:
255 storage = FilePath(config.tftp_root).parent().path
256
249 with tempdir('keyrings') as keyrings_path:257 with tempdir('keyrings') as keyrings_path:
250 # XXX: Band-aid to ensure that the keyring_data is bytes. Future task:258 # XXX: Band-aid to ensure that the keyring_data is bytes. Future task:
251 # try to figure out why this sometimes happens.259 # try to figure out why this sometimes happens.
@@ -261,15 +269,15 @@
261269
262 image_descriptions = download_all_image_descriptions(sources)270 image_descriptions = download_all_image_descriptions(sources)
263 if image_descriptions.is_empty():271 if image_descriptions.is_empty():
272 update_iscsi_targets(os.path.join(storage, 'current'))
264 maaslog.warning(273 maaslog.warning(
265 "Finished importing boot images, the region does not have "274 "Finished importing boot images, the region does not have "
266 "any boot images available.")275 "any boot images available.")
267 return False276 return False
268277
269 with ClusterConfiguration.open() as config:
270 storage = FilePath(config.tftp_root).parent().path
271 meta_file_content = image_descriptions.dump_json()278 meta_file_content = image_descriptions.dump_json()
272 if meta_contains(storage, meta_file_content):279 if meta_contains(storage, meta_file_content):
280 update_iscsi_targets(os.path.join(storage, 'current'))
273 maaslog.info(281 maaslog.info(
274 "Finished importing boot images, the region does not "282 "Finished importing boot images, the region does not "
275 "have any new images.")283 "have any new images.")
@@ -277,8 +285,17 @@
277285
278 product_mapping = map_products(image_descriptions)286 product_mapping = map_products(image_descriptions)
279287
280 snapshot_path = download_all_boot_resources(288 try:
281 sources, storage, product_mapping)289 snapshot_path = download_all_boot_resources(
290 sources, storage, product_mapping)
291 except:
292 update_iscsi_targets(os.path.join(storage, 'current'))
293 # Cleanup snapshots and cache since download failed.
294 maaslog.warning(
295 "Unable to import boot images; cleaning up failed snapshot "
296 "and cache.")
297 cleanup_snapshots_and_cache(storage)
298 raise
282299
283 maaslog.info("Writing boot image metadata and iSCSI targets.")300 maaslog.info("Writing boot image metadata and iSCSI targets.")
284 write_snapshot_metadata(snapshot_path, meta_file_content)301 write_snapshot_metadata(snapshot_path, meta_file_content)
@@ -289,8 +306,8 @@
289306
290 # If we got here, all went well. This is now truly the "current" snapshot.307 # If we got here, all went well. This is now truly the "current" snapshot.
291 update_current_symlink(storage, snapshot_path)308 update_current_symlink(storage, snapshot_path)
292 maaslog.info("Updating boot image iSCSI targets.")309
293 update_targets_conf(snapshot_path)310 update_iscsi_targets(snapshot_path)
294311
295 # Now cleanup the old snapshots and cache.312 # Now cleanup the old snapshots and cache.
296 maaslog.info('Cleaning up old snapshots and cache.')313 maaslog.info('Cleaning up old snapshots and cache.')
297314
=== modified file 'src/provisioningserver/import_images/tests/test_boot_resources.py'
--- src/provisioningserver/import_images/tests/test_boot_resources.py 2016-07-07 21:57:57 +0000
+++ src/provisioningserver/import_images/tests/test_boot_resources.py 2016-11-02 18:09:22 +0000
@@ -19,11 +19,15 @@
19 Popen,19 Popen,
20)20)
21from unittest import mock21from unittest import mock
22from unittest.mock import call22from unittest.mock import (
23 call,
24 MagicMock,
25)
2326
24from maastesting.factory import factory27from maastesting.factory import factory
25from maastesting.matchers import (28from maastesting.matchers import (
26 MockAnyCall,29 MockAnyCall,
30 MockCalledOnce,
27 MockCalledOnceWith,31 MockCalledOnceWith,
28 MockCalledWith,32 MockCalledWith,
29 MockCallsMatch,33 MockCallsMatch,
@@ -511,6 +515,7 @@
511 BootImageMapping())515 BootImageMapping())
512 self.patch_maaslog()516 self.patch_maaslog()
513 self.patch(boot_resources, 'RepoWriter')517 self.patch(boot_resources, 'RepoWriter')
518 self.patch(boot_resources, 'update_iscsi_targets')
514 args = self.make_args(sources_file=sources_fixture.filename)519 args = self.make_args(sources_file=sources_fixture.filename)
515520
516 boot_resources.main(args)521 boot_resources.main(args)
@@ -657,3 +662,152 @@
657 boot_resources.import_images(sources)662 boot_resources.import_images(sources)
658 self.assertThat(663 self.assertThat(
659 fake_write_all_keyrings, MockCalledWith(mock.ANY, sources))664 fake_write_all_keyrings, MockCalledWith(mock.ANY, sources))
665
666 def test__returns_false_when_no_images(self):
667 # Stop import_images() from actually doing anything.
668 self.patch(boot_resources, 'maaslog')
669 fake_download_all_image_descriptions = self.patch(
670 boot_resources, 'download_all_image_descriptions')
671 fake_download_all_image_descriptions.return_value = MagicMock()
672 fake_update_iscsi_targets = self.patch(
673 boot_resources, 'update_iscsi_targets')
674
675 self.patch(boot_resources, 'write_all_keyrings')
676 sources = [
677 {
678 'keyring_data': self.getUniqueString(),
679 'url': factory.make_name("something"),
680 'selections': [
681 {
682 'os': factory.make_name("os"),
683 'release': factory.make_name("release"),
684 'arches': [factory.make_name("arch")],
685 'subarches': [factory.make_name("subarch")],
686 'labels': [factory.make_name("label")],
687 },
688 ],
689 },
690 ],
691 self.assertFalse(boot_resources.import_images(sources))
692 self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
693
694 def test__returns_false_when_no_new_images(self):
695 # Stop import_images() from actually doing anything.
696 self.patch(boot_resources, 'maaslog')
697 fake_download_all_image_descriptions = self.patch(
698 boot_resources, 'download_all_image_descriptions')
699 fake_image_descriptions = MagicMock()
700 fake_image_descriptions.is_empty.return_value = False
701 fake_download_all_image_descriptions.return_value = (
702 fake_image_descriptions)
703 self.patch(boot_resources, 'meta_contains').return_value = True
704 fake_update_iscsi_targets = self.patch(
705 boot_resources, 'update_iscsi_targets')
706
707 self.patch(boot_resources, 'write_all_keyrings')
708 sources = [
709 {
710 'keyring_data': self.getUniqueString(),
711 'url': factory.make_name("something"),
712 'selections': [
713 {
714 'os': factory.make_name("os"),
715 'release': factory.make_name("release"),
716 'arches': [factory.make_name("arch")],
717 'subarches': [factory.make_name("subarch")],
718 'labels': [factory.make_name("label")],
719 },
720 ],
721 },
722 ],
723 self.assertFalse(boot_resources.import_images(sources))
724 self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
725
726 def test__cleans_up_on_failure(self):
727 # Stop import_images() from actually doing anything.
728 self.patch(boot_resources, 'maaslog')
729 fake_download_all_image_descriptions = self.patch(
730 boot_resources, 'download_all_image_descriptions')
731 fake_image_descriptions = MagicMock()
732 fake_image_descriptions.is_empty.return_value = False
733 fake_download_all_image_descriptions.return_value = (
734 fake_image_descriptions)
735 self.patch(boot_resources, 'meta_contains').return_value = False
736 self.patch(boot_resources, 'map_products')
737 self.patch(
738 boot_resources, 'download_all_boot_resources'
739 ).side_effect = Exception
740 fake_update_iscsi_targets = self.patch(
741 boot_resources, 'update_iscsi_targets')
742 fake_cleanup_snapshots_and_cache = self.patch(
743 boot_resources, 'cleanup_snapshots_and_cache')
744
745 self.patch(boot_resources, 'write_all_keyrings')
746 sources = [
747 {
748 'keyring_data': self.getUniqueString(),
749 'url': factory.make_name("something"),
750 'selections': [
751 {
752 'os': factory.make_name("os"),
753 'release': factory.make_name("release"),
754 'arches': [factory.make_name("arch")],
755 'subarches': [factory.make_name("subarch")],
756 'labels': [factory.make_name("label")],
757 },
758 ],
759 },
760 ],
761 self.assertRaises(
762 Exception, boot_resources.import_images, sources)
763 self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
764 self.assertThat(fake_cleanup_snapshots_and_cache, MockCalledOnce())
765
766 def test__runs_import_and_returns_true(self):
767 # Stop import_images() from actually doing anything.
768 self.patch(boot_resources, 'maaslog')
769 fake_download_all_image_descriptions = self.patch(
770 boot_resources, 'download_all_image_descriptions')
771 fake_image_descriptions = MagicMock()
772 fake_image_descriptions.is_empty.return_value = False
773 fake_download_all_image_descriptions.return_value = (
774 fake_image_descriptions)
775 self.patch(boot_resources, 'meta_contains').return_value = False
776 self.patch(boot_resources, 'map_products')
777 self.patch(boot_resources, 'download_all_boot_resources')
778 fake_write_snapshot_metadata = self.patch(
779 boot_resources, 'write_snapshot_metadata')
780 fake_targets_conf = self.patch(
781 boot_resources, 'write_targets_conf')
782 fake_install_boot_loaders = self.patch(
783 boot_resources, 'install_boot_loaders')
784 fake_update_current_symlink = self.patch(
785 boot_resources, 'update_current_symlink')
786 fake_update_iscsi_targets = self.patch(
787 boot_resources, 'update_iscsi_targets')
788 fake_cleanup_snapshots_and_cache = self.patch(
789 boot_resources, 'cleanup_snapshots_and_cache')
790
791 self.patch(boot_resources, 'write_all_keyrings')
792 sources = [
793 {
794 'keyring_data': self.getUniqueString(),
795 'url': factory.make_name("something"),
796 'selections': [
797 {
798 'os': factory.make_name("os"),
799 'release': factory.make_name("release"),
800 'arches': [factory.make_name("arch")],
801 'subarches': [factory.make_name("subarch")],
802 'labels': [factory.make_name("label")],
803 },
804 ],
805 },
806 ],
807 self.assertTrue(boot_resources.import_images(sources))
808 self.assertThat(fake_write_snapshot_metadata, MockCalledOnce())
809 self.assertThat(fake_targets_conf, MockCalledOnce())
810 self.assertThat(fake_install_boot_loaders, MockCalledOnce())
811 self.assertThat(fake_update_current_symlink, MockCalledOnce())
812 self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
813 self.assertThat(fake_cleanup_snapshots_and_cache, MockCalledOnce())

Subscribers

People subscribed via source and target branches

to all changes: