Merge lp:~ltrager/maas/remove_tgt into lp:~maas-committers/maas/trunk

Proposed by Lee Trager
Status: Rejected
Rejected by: MAAS Lander
Proposed branch: lp:~ltrager/maas/remove_tgt
Merge into: lp:~maas-committers/maas/trunk
Prerequisite: lp:~ltrager/maas/rootfs_over_http
Diff against target: 1720 lines (+17/-1158)
25 files modified
snap/bin/run-rackd (+0/-10)
snap/bin/run-tgtd (+0/-37)
snap/conf/supervisord.conf.template (+0/-10)
snap/snapcraft.yaml (+0/-4)
src/maasserver/bootresources.py (+1/-2)
src/maasserver/enum.py (+3/-3)
src/maasserver/forms/settings.py (+0/-8)
src/maasserver/models/config.py (+0/-1)
src/maasserver/models/service.py (+0/-2)
src/maasserver/static/js/angular/controllers/tests/test_node_details.js (+1/-3)
src/maasserver/static/partials/node-details.html (+0/-7)
src/provisioningserver/import_images/boot_resources.py (+1/-130)
src/provisioningserver/import_images/download_resources.py (+0/-69)
src/provisioningserver/import_images/tests/test_boot_resources.py (+1/-171)
src/provisioningserver/import_images/tests/test_download_resources.py (+0/-32)
src/provisioningserver/import_images/tests/test_uec2roottar.py (+0/-370)
src/provisioningserver/import_images/uec2roottar.py (+0/-187)
src/provisioningserver/kernel_opts.py (+9/-48)
src/provisioningserver/rpc/region.py (+0/-1)
src/provisioningserver/service_monitor.py (+0/-10)
src/provisioningserver/support_dump.py (+0/-3)
src/provisioningserver/tests/test_kernel_opts.py (+0/-11)
src/provisioningserver/tests/test_service_monitor.py (+1/-13)
src/provisioningserver/tests/test_upgrade_cluster.py (+0/-17)
src/provisioningserver/upgrade_cluster.py (+0/-9)
To merge this branch: bzr merge lp:~ltrager/maas/remove_tgt
Reviewer Review Type Date Requested Status
Mike Pontillo (community) Approve
Review via email: mp+325675@code.launchpad.net

Commit message

Remove TGT configuration and monitoring.

To post a comment you must log in.
Revision history for this message
Mike Pontillo (mpontillo) wrote :

Nice work! I bet it felt good to remove all that code. ;-)

Did you already test this end-to-end in conjunction with the packaging changes you proposed?

review: Needs Information
Revision history for this message
Lee Trager (ltrager) wrote :

Yes it did :) I tested this with all three branches locally and am working on fixing the CI tests.

Revision history for this message
Mike Pontillo (mpontillo) wrote :

Sounds great. If we have an IPv6 environment ready to go, we should test that, too. (Unfortunately, though I do have an IPv6 test network, I don't have any test devices that will boot from it.)

review: Approve
lp:~ltrager/maas/remove_tgt updated
6088. By Lee Trager

Merge trunk

6089. By Lee Trager

Merge rootfs_over_http

Revision history for this message
Blake Rouse (blake-rouse) wrote :

You will need to convert this to a git branch, as we will be in git before this gets to land.

Revision history for this message
MAAS Lander (maas-lander) wrote :

Transitioned to Git.

lp:maas has now moved from Bzr to Git.
Please propose your branches with Launchpad using Git.

git clone https://git.launchpad.net/maas

Unmerged revisions

6089. By Lee Trager

Merge rootfs_over_http

6088. By Lee Trager

Merge trunk

6087. By Lee Trager

Remove TGT configuration and monitoring.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'snap/bin/run-rackd'
2--- snap/bin/run-rackd 2017-05-12 15:08:50 +0000
3+++ snap/bin/run-rackd 2017-06-21 22:15:36 +0000
4@@ -24,15 +24,5 @@
5 export MAAS_ROOT="$SNAP_DATA"
6 export MAAS_CLUSTER_CONFIG="$SNAP_DATA/rackd.conf"
7
8-# Setup language and perl5 correctly. Needed by tgt-admin written in
9-# Perl, yes really!
10-#
11-# XXX blake_r: Fix the hardcoded x86_64-linux-gnu to work for other
12-# architectures.
13-export LANGUAGE="C.UTF-8"
14-export LC_ALL="C.UTF-8"
15-export LANG="C.UTF-8"
16-export PERL5LIB="$SNAP/usr/lib/x86_64-linux-gnu/perl/5.22:$SNAP/usr/share/perl/5.22:$SNAP/usr/share/perl5"
17-
18 # Run the rackd.
19 exec $SNAP/bin/twistd --logger=provisioningserver.logger.EventLogger --nodaemon --pidfile= maas-rackd
20
21=== removed file 'snap/bin/run-tgtd'
22--- snap/bin/run-tgtd 2017-04-05 20:37:42 +0000
23+++ snap/bin/run-tgtd 1970-01-01 00:00:00 +0000
24@@ -1,37 +0,0 @@
25-#!/bin/bash
26-# Copyright 2017 Canonical Ltd. This software is licensed under the
27-# GNU Affero General Public License version 3 (see the file LICENSE).
28-
29-set -e
30-trap "kill -- -$$" EXIT
31-
32-# Configure tgtd socket.
33-export TGT_IPC_SOCKET="$SNAP_DATA/tgtd-socket"
34-
35-# Setup language and perl5 correctly. Needed by tgt-admin written in
36-# Perl, yes really!
37-#
38-# XXX blake_r: Fix the hardcoded x86_64-linux-gnu to work for other
39-# architectures.
40-export LANGUAGE="C.UTF-8"
41-export LC_ALL="C.UTF-8"
42-export LANG="C.UTF-8"
43-export PERL5LIB="$SNAP/usr/lib/x86_64-linux-gnu/perl/5.22:$SNAP/usr/share/perl/5.22:$SNAP/usr/share/perl5"
44-
45-# Configuration paths.
46-TGTD_CONFIG="$SNAP_DATA/var/lib/maas/boot-resources/current/maas.tgt"
47-if [ ! -e "$TGTD_CONFIG" ]
48-then
49- TGTD_CONFIG="$SNAP/usr/share/maas/empty.tgt"
50-fi
51-
52-# Spawn tgtd.
53-trap 'kill $PID; wait $PID' TERM INT
54-$SNAP/usr/sbin/tgtd -f &
55-PID=$!
56-
57-# Perform post start commands.
58-$SNAP/usr/sbin/tgtadm --op update --mode sys --name State -v offline
59-$SNAP/usr/sbin/tgt-admin -e -c "$TGTD_CONFIG"
60-$SNAP/usr/sbin/tgtadm --op update --mode sys --name State -v ready
61-wait $PID
62
63=== modified file 'snap/conf/supervisord.conf.template'
64--- snap/conf/supervisord.conf.template 2017-05-12 16:23:16 +0000
65+++ snap/conf/supervisord.conf.template 2017-06-21 22:15:36 +0000
66@@ -90,16 +90,6 @@
67 redirect_stderr=true
68 stdout_logfile=%(ENV_SNAP_COMMON)s/log/dhcpd6.log
69
70-[program:tgt]
71-process_name=tgt
72-command=%(ENV_SNAP)s/bin/run-tgtd
73-stopasgroup=true
74-killasgroup=true
75-redirect_stderr=true
76-stdout_logfile=%(ENV_SNAP_COMMON)s/log/tgt.log
77-{{endif}}
78-
79-
80 {{if rackd or regiond}}
81 [program:ntp]
82 process_name=ntp
83
84=== modified file 'snap/snapcraft.yaml'
85--- snap/snapcraft.yaml 2017-05-30 15:57:19 +0000
86+++ snap/snapcraft.yaml 2017-06-21 22:15:36 +0000
87@@ -35,7 +35,6 @@
88 - iproute2
89 - isc-dhcp-client
90 - isc-dhcp-server
91- - libconfig-general-perl
92 - libjs-angularjs
93 - libjs-jquery
94 - libjs-yui3-full
95@@ -46,7 +45,6 @@
96 - postgresql
97 - squid
98 - tcpdump
99- - tgt
100 - ubuntu-cloudimage-keyring
101 install: |
102 ln -s ../usr/lib/postgresql/9.5/bin/initdb $SNAPCRAFT_PART_INSTALL/bin/initdb
103@@ -96,7 +94,6 @@
104 - -usr/share/doc-base
105 - -usr/share/lintian
106 - -usr/share/man
107- - -usr/share/perl5
108 organize:
109 lib/python3.5/site-packages/etc: etc
110 lib/python3.5/site-packages/usr: usr
111@@ -121,7 +118,6 @@
112 source: snap
113 organize:
114 bind: usr/share/maas/bind
115- conf/empty.tgt: usr/share/maas/empty.tgt
116 conf/ntp.conf: usr/share/maas/ntp.conf
117 conf/supervisord.conf.template: usr/share/maas/supervisord.conf.template
118 stage:
119
120=== modified file 'src/maasserver/bootresources.py'
121--- src/maasserver/bootresources.py 2017-05-03 00:56:19 +0000
122+++ src/maasserver/bootresources.py 2017-06-21 22:15:36 +0000
123@@ -651,8 +651,7 @@
124
125 # A ROOT_IMAGE may already be downloaded for the release if the stream
126 # switched from one not containg SquashFS images to one that does. We
127- # want to use the SquashFS image but if both are available TGT will
128- # fail to start because both images will be shared with the same name.
129+ # want to use the SquashFS image so ignore the tgz.
130 if product['ftype'] == BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE:
131 resource_set.files.filter(
132 filetype=BOOT_RESOURCE_FILE_TYPE.ROOT_IMAGE).delete()
133
134=== modified file 'src/maasserver/enum.py'
135--- src/maasserver/enum.py 2017-03-10 08:46:42 +0000
136+++ src/maasserver/enum.py 2017-06-21 22:15:36 +0000
137@@ -406,13 +406,13 @@
138 #: Root image in SquashFS form, does not need to be converted
139 SQUASHFS_IMAGE = 'squashfs'
140
141- #: Boot Kernel (ISCSI kernel)
142+ #: Boot Kernel
143 BOOT_KERNEL = 'boot-kernel'
144
145- #: Boot Initrd (ISCSI initrd)
146+ #: Boot Initrd
147 BOOT_INITRD = 'boot-initrd'
148
149- #: Boot DTB (ISCSI dtb)
150+ #: Boot DTB
151 BOOT_DTB = 'boot-dtb'
152
153 #: An uncompressed bootloader (PXELinux, GRUB, etc)
154
155=== modified file 'src/maasserver/forms/settings.py'
156--- src/maasserver/forms/settings.py 2017-06-21 22:15:36 +0000
157+++ src/maasserver/forms/settings.py 2017-06-21 22:15:36 +0000
158@@ -538,14 +538,6 @@
159 'min_value': 1,
160 },
161 },
162- 'http_boot': {
163- 'default': False,
164- 'form': forms.BooleanField,
165- 'form_kwargs': {
166- 'label': "When true all ephemeral environments boot over HTTP.",
167- 'required': False
168- }
169- },
170 }
171
172
173
174=== modified file 'src/maasserver/models/config.py'
175--- src/maasserver/models/config.py 2017-06-21 22:15:36 +0000
176+++ src/maasserver/models/config.py 2017-06-21 22:15:36 +0000
177@@ -102,7 +102,6 @@
178 'max_node_installation_results': 1,
179 # Notifications.
180 'subnet_ip_exhaustion_threshold_count': 16,
181- 'http_boot': False,
182 }
183
184
185
186=== modified file 'src/maasserver/models/service.py'
187--- src/maasserver/models/service.py 2017-06-01 12:40:27 +0000
188+++ src/maasserver/models/service.py 2017-06-21 22:15:36 +0000
189@@ -37,7 +37,6 @@
190 "rackd",
191 "http",
192 "tftp",
193- "tgt",
194 "dhcpd",
195 "dhcpd6",
196 "ntp_rack",
197@@ -52,7 +51,6 @@
198 "rackd": SERVICE_STATUS.DEAD,
199 "http": SERVICE_STATUS.DEAD,
200 "tftp": SERVICE_STATUS.DEAD,
201- "tgt": SERVICE_STATUS.UNKNOWN,
202 "dhcpd": SERVICE_STATUS.DEAD,
203 "dhcpd6": SERVICE_STATUS.DEAD,
204 "ntp_region": SERVICE_STATUS.DEAD,
205
206=== modified file 'src/maasserver/static/js/angular/controllers/tests/test_node_details.js'
207--- src/maasserver/static/js/angular/controllers/tests/test_node_details.js 2017-06-02 21:36:12 +0000
208+++ src/maasserver/static/js/angular/controllers/tests/test_node_details.js 2017-06-21 22:15:36 +0000
209@@ -340,7 +340,6 @@
210
211 it("updateServices sets $scope.services when node is loaded", function() {
212 spyOn(ControllersManager, "getServices").and.returnValue([
213- { "status": "unknown", "name": "tgt" },
214 { "status": "running", "name": "rackd" }
215 ]);
216 spyOn(ControllersManager, "setActiveItem").and.returnValue(
217@@ -359,8 +358,7 @@
218
219 expect(ControllersManager.getServices).toHaveBeenCalledWith(node);
220 expect($scope.services).not.toBeNull();
221- expect(Object.keys($scope.services).length).toBe(2);
222- expect($scope.services.tgt.status).toBe('unknown');
223+ expect(Object.keys($scope.services).length).toBe(1);
224 expect($scope.services.rackd.status).toBe('running');
225 });
226
227
228=== modified file 'src/maasserver/static/partials/node-details.html'
229--- src/maasserver/static/partials/node-details.html 2017-06-14 09:56:32 +0000
230+++ src/maasserver/static/partials/node-details.html 2017-06-21 22:15:36 +0000
231@@ -575,13 +575,6 @@
232 <span data-ng-if="services.dhcpd6.status_info"> &mdash; {$ services.dhcpd6.status_info $}</span>
233 </span>
234 </li>
235- <li class="list__item">
236- <span>
237- <span class="icon icon--{$ getServiceClass(services.tgt) $}"></span>
238- &nbsp;tgt
239- <span data-ng-if="services.tgt.status_info"> &mdash; {$ services.tgt.status_info $}</span>
240- </span>
241- </li>
242 <li class="list__item" data-ng-if="services.ntp_rack">
243 <span>
244 <span class="icon icon--{$ getServiceClass(services.ntp_rack) $}"></span>
245
246=== modified file 'src/provisioningserver/import_images/boot_resources.py'
247--- src/provisioningserver/import_images/boot_resources.py 2017-05-03 00:56:19 +0000
248+++ src/provisioningserver/import_images/boot_resources.py 2017-06-21 22:15:36 +0000
249@@ -12,10 +12,8 @@
250 import errno
251 from io import StringIO
252 import os
253-from textwrap import dedent
254
255 from provisioningserver.boot import BootMethodRegistry
256-from provisioningserver.boot.tftppath import list_boot_images
257 from provisioningserver.config import (
258 BootSources,
259 ClusterConfiguration,
260@@ -36,20 +34,13 @@
261 from provisioningserver.import_images.helpers import maaslog
262 from provisioningserver.import_images.keyrings import write_all_keyrings
263 from provisioningserver.import_images.product_mapping import map_products
264-from provisioningserver.path import get_path
265 from provisioningserver.rpc import getRegionClient
266-from provisioningserver.service_monitor import service_monitor
267-from provisioningserver.utils import sudo
268 from provisioningserver.utils.fs import (
269 atomic_symlink,
270 atomic_write,
271 read_text_file,
272 tempdir,
273 )
274-from provisioningserver.utils.shell import (
275- call_and_check,
276- ExternalProcessError,
277-)
278 from twisted.internet.defer import inlineCallbacks
279 from twisted.python.filepath import FilePath
280
281@@ -58,47 +49,6 @@
282 """Raised when the config file for the script doesn't exist."""
283
284
285-def tgt_entry(osystem, arch, subarch, release, label, image):
286- """Generate tgt target used to commission arch/subarch with release
287-
288- Tgt target used to commission arch/subarch machine with a specific Ubuntu
289- release should have the following name: ephemeral-arch-subarch-release.
290- This function creates target description in a format used by tgt-admin.
291- It uses arch, subarch and release to generate target name and image as
292- a path to image file which should be shared. Tgt target is marked as
293- read-only. Tgt target has 'allow-in-use' option enabled because this
294- script actively uses hardlinks to do image management and root images
295- in different folders may point to the same inode. Tgt doesn't allow us to
296- use the same inode for different tgt targets (even read-only targets which
297- looks like a bug to me) without this option enabled.
298-
299- :param osystem: Operating System name we generate tgt target for
300- :param arch: Architecture name we generate tgt target for
301- :param subarch: Subarchitecture name we generate tgt target for
302- :param release: Ubuntu release we generate tgt target for
303- :param label: The images' label
304- :param image: Path to the image which should be shared via tgt/iscsi
305- :return Tgt entry which can be written to tgt-admin configuration file
306- """
307- prefix = 'iqn.2004-05.com.ubuntu:maas'
308- target_name = 'ephemeral-%s-%s-%s-%s-%s' % (
309- osystem,
310- arch,
311- subarch,
312- release,
313- label
314- )
315- entry = dedent("""\
316- <target {prefix}:{target_name}>
317- readonly 1
318- allow-in-use yes
319- backing-store "{image}"
320- driver iscsi
321- </target>
322- """).format(prefix=prefix, target_name=target_name, image=image)
323- return entry
324-
325-
326 def link_bootloaders(destination):
327 """Link the all the required file from each bootloader method.
328 :param destination: Directory where the loaders should be stored.
329@@ -127,42 +77,6 @@
330 return parser
331
332
333-def compose_targets_conf(snapshot_path):
334- """Produce the contents of a snapshot's tgt conf file.
335-
336- :param snapshot_path: Filesystem path to a snapshot of current upstream
337- boot resources.
338- :return: Contents for a `targets.conf` file.
339- :rtype: bytes
340- """
341- # Use a set to make sure we don't register duplicate entries in tgt.
342- entries = set()
343- for item in list_boot_images(snapshot_path):
344- osystem = item['osystem']
345- arch = item['architecture']
346- subarch = item['subarchitecture']
347- release = item['release']
348- label = item['label']
349- entries.add((osystem, arch, subarch, release, label))
350- tgt_entries = []
351- for osystem, arch, subarch, release, label in sorted(entries):
352- base_path = os.path.join(
353- snapshot_path, osystem, arch, subarch,
354- release, label)
355- root_image = os.path.join(base_path, 'root-image')
356- if os.path.isfile(root_image):
357- entry = tgt_entry(
358- osystem, arch, subarch, release, label, root_image)
359- tgt_entries.append(entry)
360- squashfs_image = os.path.join(base_path, 'squashfs')
361- if os.path.isfile(squashfs_image):
362- entry = tgt_entry(
363- osystem, arch, subarch, release, label, squashfs_image)
364- tgt_entries.append(entry)
365- text = ''.join(tgt_entries)
366- return text.encode('utf-8')
367-
368-
369 def meta_contains(storage, content):
370 """Does the `maas.meta` file match `content`?
371
372@@ -195,38 +109,6 @@
373 atomic_write(meta_file_content.encode("ascii"), meta_file, mode=0o644)
374
375
376-def write_targets_conf(snapshot):
377- """Write "maas.tgt" file."""
378- targets_conf = os.path.join(snapshot, 'maas.tgt')
379- targets_conf_content = compose_targets_conf(snapshot)
380- atomic_write(targets_conf_content, targets_conf, mode=0o644)
381-
382-
383-def update_targets_conf(snapshot):
384- """Runs tgt-admin to update the new targets from "maas.tgt"."""
385- # Ensure that tgt is running before tgt-admin is used.
386- service_monitor.ensureService("tgt").wait(30)
387-
388- # Update the tgt config.
389- targets_conf = os.path.join(snapshot, 'maas.tgt')
390-
391- # The targets_conf may not exist in the event the BootSource is broken
392- # and images havn't been imported yet. This fixes LP:1655721
393- if not os.path.exists(targets_conf):
394- return
395-
396- try:
397- call_and_check(sudo([
398- get_path('/usr/sbin/tgt-admin'),
399- '--conf', targets_conf,
400- '--update', 'ALL',
401- ]))
402- except ExternalProcessError as e:
403- msg = "Unable to update TGT config: %s" % e
404- try_send_rack_event(EVENT_TYPES.RACK_IMPORT_WARNING, msg)
405- maaslog.warning(msg)
406-
407-
408 def read_sources(sources_yaml):
409 """Read boot resources config file.
410
411@@ -253,11 +135,6 @@
412 return BootSources.parse(StringIO(sources_yaml))
413
414
415-def update_iscsi_targets(snapshot_path):
416- maaslog.info("Updating boot image iSCSI targets.")
417- update_targets_conf(snapshot_path)
418-
419-
420 def import_images(sources):
421 """Import images. Callable from the command line.
422
423@@ -297,7 +174,6 @@
424 image_descriptions = download_all_image_descriptions(
425 sources, validate_products=False)
426 if image_descriptions.is_empty():
427- update_iscsi_targets(os.path.join(storage, 'current'))
428 msg = (
429 "Finished importing boot images, the region does not have "
430 "any boot images available.")
431@@ -307,7 +183,6 @@
432
433 meta_file_content = image_descriptions.dump_json()
434 if meta_contains(storage, meta_file_content):
435- update_iscsi_targets(os.path.join(storage, 'current'))
436 maaslog.info(
437 "Finished importing boot images, the region does not "
438 "have any new images.")
439@@ -324,7 +199,6 @@
440 try_send_rack_event(
441 EVENT_TYPES.RACK_IMPORT_ERROR,
442 "Unable to import boot images: %s" % e)
443- update_iscsi_targets(os.path.join(storage, 'current'))
444 maaslog.error(
445 "Unable to import boot images; cleaning up failed snapshot "
446 "and cache.")
447@@ -332,9 +206,8 @@
448 cleanup_snapshots_and_cache(storage)
449 raise
450
451- maaslog.info("Writing boot image metadata and iSCSI targets.")
452+ maaslog.info("Writing boot image metadata.")
453 write_snapshot_metadata(snapshot_path, meta_file_content)
454- write_targets_conf(snapshot_path)
455
456 maaslog.info("Linking boot images snapshot %s" % snapshot_path)
457 link_bootloaders(snapshot_path)
458@@ -342,8 +215,6 @@
459 # If we got here, all went well. This is now truly the "current" snapshot.
460 update_current_symlink(storage, snapshot_path)
461
462- update_iscsi_targets(snapshot_path)
463-
464 # Now cleanup the old snapshots and cache.
465 maaslog.info('Cleaning up old snapshots and cache.')
466 cleanup_snapshots_and_cache(storage)
467
468=== modified file 'src/provisioningserver/import_images/download_resources.py'
469--- src/provisioningserver/import_images/download_resources.py 2017-03-31 09:35:11 +0000
470+++ src/provisioningserver/import_images/download_resources.py 2017-06-21 22:15:36 +0000
471@@ -8,19 +8,15 @@
472 ]
473
474 from datetime import datetime
475-from gzip import GzipFile
476 import os.path
477 import tarfile
478
479-from provisioningserver.config import is_dev_environment
480 from provisioningserver.import_images.helpers import (
481 get_os_from_product,
482 get_signing_policy,
483 maaslog,
484 )
485 from provisioningserver.logger import LegacyLogger
486-from provisioningserver.utils.shell import call_and_check
487-from simplestreams.contentsource import FdContentSource
488 from simplestreams.mirrors import (
489 BasicMirrorWriter,
490 UrlMirrorReader,
491@@ -65,68 +61,6 @@
492 return [(store._fullpath(tag), name)]
493
494
495-def call_uec2roottar(root_image_path, root_tgz_path):
496- """Invoke `uec2roottar` with the given arguments.
497-
498- Here only so tests can stub it out.
499-
500- :param root_image_path: Input file.
501- :param root_tgz_path: Output file.
502- """
503- if is_dev_environment():
504- # In debug mode this is skipped as it requires the uec2roottar
505- # script to have sudo abilities. The root-tgz is created as an
506- # empty file so the correct links can be made.
507- log.msg(
508- "Conversion of root-image to root-tgz is skipped in DEVELOP mode.")
509- open(root_tgz_path, "wb").close()
510- else:
511- call_and_check([
512- 'sudo', '/usr/bin/uec2roottar',
513- '--user=maas',
514- root_image_path,
515- root_tgz_path,
516- ])
517-
518-
519-def insert_root_image(store, tag, checksums, size, content_source):
520- """Insert a root image into `store`.
521-
522- This may involve converting a UEC boot image into a root tarball.
523-
524- :param store: A simplestreams `ObjectStore`.
525- :param tag: UUID, or "tag," for the file root image file. The root image
526- and root tarball will both be stored in the cache directory under
527- names derived from this tag.
528- :param checksums: A Simplestreams checksums dict, mapping hash algorihm
529- names (such as `sha256`) to the file's respective checksums as
530- computed by those hash algorithms.
531- :param size: Optional size for the file, so Simplestreams knows what size
532- to expect.
533- :param content_source: A Simplestreams `ContentSource` for reading the
534- file.
535- :return: A list of inserted files (root image and root tarball) described
536- as tuples of (path, logical name). The path lies in the directory
537- managed by `store` and has a filename based on `tag`, not logical name.
538- """
539- maaslog.debug("Inserting root image (tag=%s, size=%s).", tag, size)
540- root_image_tag = 'root-image-%s' % tag
541- # XXX jtv 2014-04-24 bug=1313580: Isn't _fullpath meant to be private?
542- root_image_path = store._fullpath(root_image_tag)
543- root_tgz_tag = 'root-tgz-%s' % tag
544- root_tgz_path = store._fullpath(root_tgz_tag)
545- if not os.path.isfile(root_image_path):
546- maaslog.debug("New root image: %s.", root_image_path)
547- store.insert(tag, content_source, checksums, mutable=False, size=size)
548- uncompressed = FdContentSource(GzipFile(store._fullpath(tag)))
549- store.insert(root_image_tag, uncompressed, mutable=False)
550- store.remove(tag)
551- if not os.path.isfile(root_tgz_path):
552- maaslog.debug("Converting root tarball: %s.", root_tgz_path)
553- call_uec2roottar(root_image_path, root_tgz_path)
554- return [(root_image_path, 'root-image'), (root_tgz_path, 'root-tgz')]
555-
556-
557 def extract_archive_tar(store, name, tag, checksums, size, content_source):
558 """Extract an archive.tar.xz into `store`.
559
560@@ -279,9 +213,6 @@
561 if ftype == 'archive.tar.xz':
562 links = extract_archive_tar(
563 self.store, filename, tag, checksums, size, contentsource)
564- elif ftype == 'root-image.gz':
565- links = insert_root_image(
566- self.store, tag, checksums, size, contentsource)
567 else:
568 links = insert_file(
569 self.store, filename, tag, checksums, size, contentsource)
570
571=== modified file 'src/provisioningserver/import_images/tests/test_boot_resources.py'
572--- src/provisioningserver/import_images/tests/test_boot_resources.py 2017-05-30 18:36:16 +0000
573+++ src/provisioningserver/import_images/tests/test_boot_resources.py 2017-06-21 22:15:36 +0000
574@@ -14,10 +14,6 @@
575 import json
576 import os
577 from random import randint
578-from subprocess import (
579- PIPE,
580- Popen,
581-)
582 from unittest import mock
583 from unittest.mock import (
584 call,
585@@ -25,14 +21,11 @@
586 )
587
588 from maastesting.factory import factory
589-from maastesting.fixtures import TempDirectory
590 from maastesting.matchers import (
591 MockAnyCall,
592 MockCalledOnce,
593- MockCalledOnceWith,
594 MockCalledWith,
595 MockCallsMatch,
596- MockNotCalled,
597 )
598 from maastesting.testcase import MAASTestCase
599 from maastesting.utils import age_file
600@@ -50,13 +43,7 @@
601 BootSourcesFixture,
602 ClusterConfigurationFixture,
603 )
604-from provisioningserver.utils.fs import (
605- tempdir,
606- write_text_file,
607-)
608-from provisioningserver.utils.shell import ExternalProcessError
609-from testtools.content import Content
610-from testtools.content_type import UTF8_TEXT
611+from provisioningserver.utils.fs import write_text_file
612 from testtools.matchers import (
613 DirExists,
614 FileExists,
615@@ -65,103 +52,6 @@
616 import yaml
617
618
619-class TestTgtEntry(MAASTestCase):
620- """Tests for `tgt_entry`."""
621-
622- def test_generates_one_target(self):
623- spec = make_image_spec()
624- osystem = factory.make_name('osystem')
625- image = self.make_file()
626- entry = boot_resources.tgt_entry(
627- osystem, spec.arch, spec.subarch, spec.release, spec.label, image)
628- # The entry looks a bit like XML, but isn't well-formed. So don't try
629- # to parse it as such!
630- self.assertIn('<target iqn.2004-05.com.ubuntu:maas:', entry)
631- self.assertIn('backing-store "%s"' % image, entry)
632- self.assertEqual(1, entry.count('</target>'))
633-
634- def test_produces_suitable_output_for_tgt_admin(self):
635- spec = make_image_spec()
636- image = self.make_file()
637- osystem = factory.make_name('osystem')
638- entry = boot_resources.tgt_entry(
639- osystem, spec.arch, spec.subarch, spec.release, spec.label, image)
640- config = self.make_file(contents=entry)
641- # Pretend to be root, but without requiring the actual privileges and
642- # without prompting for a password. In that state, run tgt-admin.
643- # It has to think it's root, even for a "pretend" run.
644- # Make it read the config we just produced, and pretend to update its
645- # iSCSI targets based on what it finds in the config.
646- #
647- # The only real test is that this succeed.
648- cmd = Popen(
649- [
650- 'fakeroot', 'tgt-admin',
651- '--conf', config,
652- '--pretend',
653- '--update', 'ALL',
654- ],
655- stdout=PIPE, stderr=PIPE)
656- stdout, stderr = cmd.communicate()
657- self.addDetail('tgt-stderr', Content(UTF8_TEXT, lambda: [stderr]))
658- self.addDetail('tgt-stdout', Content(UTF8_TEXT, lambda: [stdout]))
659- self.assertEqual(0, cmd.returncode)
660-
661-
662-class TestComposeTargetsConf(MAASTestCase):
663- """Tests for `compose_targets_conf`."""
664-
665- def make_fake_boot_resource(
666- self, boot_resource_path, image, boot_images=None):
667- if boot_images is None:
668- boot_images = []
669- osystem = factory.make_name('osystem')
670- arch = factory.make_name('arch')
671- subarch = factory.make_name('subarch')
672- release = factory.make_name('release')
673- label = factory.make_name('label')
674- boot_images.append({
675- 'osystem': osystem,
676- 'architecture': arch,
677- 'subarchitecture': subarch,
678- 'release': release,
679- 'label': label,
680- })
681- path = os.path.join(
682- boot_resource_path, osystem, arch, subarch, release, label)
683- os.makedirs(path)
684- path = os.path.join(path, image)
685- open(path, 'a').close()
686- return boot_images, path
687-
688- def test__creates_root_image_entry(self):
689- with tempdir('boot_resource_path') as boot_resource_path:
690- boot_images, path = self.make_fake_boot_resource(
691- boot_resource_path, 'root-image')
692- self.patch(
693- boot_resources, 'list_boot_images').return_value = boot_images
694- output = boot_resources.compose_targets_conf(boot_resource_path)
695- self.assertIn(path, output.decode('utf-8'))
696-
697- def test__creates_squashfs_entry(self):
698- with tempdir('boot_resource_path') as boot_resource_path:
699- boot_images, path = self.make_fake_boot_resource(
700- boot_resource_path, 'squashfs')
701- self.patch(
702- boot_resources, 'list_boot_images').return_value = boot_images
703- output = boot_resources.compose_targets_conf(boot_resource_path)
704- self.assertIn(path, output.decode('utf-8'))
705-
706- def test__returns_empty_for_unknown_image(self):
707- with tempdir('boot_resource_path') as boot_resource_path:
708- boot_images, _ = self.make_fake_boot_resource(
709- boot_resource_path, factory.make_name('unknown_image'))
710- self.patch(
711- boot_resources, 'list_boot_images').return_value = boot_images
712- output = boot_resources.compose_targets_conf(boot_resource_path)
713- self.assertEquals(b'', output)
714-
715-
716 class TestUpdateCurrentSymlink(MAASTestCase):
717
718 def make_test_dirs(self):
719@@ -468,7 +358,6 @@
720 self.assertThat(current, DirExists())
721 self.assertThat(os.path.join(current, 'pxelinux.0'), FileExists())
722 self.assertThat(os.path.join(current, 'maas.meta'), FileExists())
723- self.assertThat(os.path.join(current, 'maas.tgt'), FileExists())
724 self.assertThat(
725 os.path.join(
726 current, osystem, arch, subarch, self.release, self.label),
727@@ -549,7 +438,6 @@
728 BootImageMapping())
729 self.patch_maaslog()
730 self.patch(boot_resources, 'RepoWriter')
731- self.patch(boot_resources, 'update_iscsi_targets')
732 args = self.make_args(sources_file=sources_fixture.filename)
733
734 boot_resources.main(args)
735@@ -589,49 +477,6 @@
736 boot_resources.NoConfigFile,
737 boot_resources.main, self.make_args(sources="", sources_file=""))
738
739- def test_update_targets_conf_ensures_tgt_service(self):
740- mock_ensureService = self.patch(
741- boot_resources.service_monitor, "ensureService")
742- self.patch(boot_resources, "call_and_check")
743- boot_resources.update_targets_conf(factory.make_name("snapshot"))
744- self.assertThat(mock_ensureService, MockCalledOnceWith("tgt"))
745-
746- def test_update_targets_conf_logs_error(self):
747- self.patch(boot_resources.service_monitor, "ensureService")
748- mock_try_send_rack_event = self.patch(
749- boot_resources, 'try_send_rack_event')
750- mock_maaslog = self.patch(boot_resources.maaslog, 'warning')
751- self.patch(boot_resources.os.path, 'exists').return_value = True
752- self.patch(boot_resources, 'call_and_check').side_effect = (
753- ExternalProcessError(
754- returncode=2, cmd=('tgt-admin',), output='error'))
755- snapshot = factory.make_name("snapshot")
756- boot_resources.update_targets_conf(snapshot)
757- self.assertThat(mock_try_send_rack_event, MockCalledOnce())
758- self.assertThat(mock_maaslog, MockCalledOnce())
759- self.assertThat(
760- boot_resources.call_and_check,
761- MockCalledOnceWith([
762- 'sudo', '-n', '/usr/sbin/tgt-admin',
763- '--conf', os.path.join(snapshot, 'maas.tgt'),
764- '--update', 'ALL']))
765-
766- def test_update_targets_only_runs_when_conf_exists(self):
767- # Regression test for LP:1655721
768- temp_dir = self.useFixture(TempDirectory()).path
769- self.useFixture(ClusterConfigurationFixture(tftp_root=temp_dir))
770- mock_ensureService = self.patch(
771- boot_resources.service_monitor, "ensureService")
772- mock_call_and_check = self.patch(boot_resources, "call_and_check")
773- mock_path_exists = self.patch(boot_resources.os.path, 'exists')
774- mock_path_exists.return_value = False
775- boot_resources.update_targets_conf(temp_dir)
776- self.assertThat(mock_ensureService, MockCalledOnceWith("tgt"))
777- self.assertThat(
778- mock_path_exists,
779- MockCalledOnceWith(os.path.join(temp_dir, 'maas.tgt')))
780- self.assertThat(mock_call_and_check, MockNotCalled())
781-
782
783 class TestMetaContains(MAASTestCase):
784 """Tests for the `meta_contains` function."""
785@@ -738,8 +583,6 @@
786 fake_download_all_image_descriptions = self.patch(
787 boot_resources, 'download_all_image_descriptions')
788 fake_download_all_image_descriptions.return_value = MagicMock()
789- fake_update_iscsi_targets = self.patch(
790- boot_resources, 'update_iscsi_targets')
791
792 self.patch(boot_resources, 'write_all_keyrings')
793 sources = [
794@@ -758,7 +601,6 @@
795 },
796 ],
797 self.assertFalse(boot_resources.import_images(sources))
798- self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
799
800 def test__returns_false_when_no_new_images(self):
801 # Stop import_images() from actually doing anything.
802@@ -770,8 +612,6 @@
803 fake_download_all_image_descriptions.return_value = (
804 fake_image_descriptions)
805 self.patch(boot_resources, 'meta_contains').return_value = True
806- fake_update_iscsi_targets = self.patch(
807- boot_resources, 'update_iscsi_targets')
808
809 self.patch(boot_resources, 'write_all_keyrings')
810 sources = [
811@@ -790,7 +630,6 @@
812 },
813 ],
814 self.assertFalse(boot_resources.import_images(sources))
815- self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
816
817 def test__cleans_up_on_failure(self):
818 # Stop import_images() from actually doing anything.
819@@ -806,8 +645,6 @@
820 self.patch(
821 boot_resources, 'download_all_boot_resources'
822 ).side_effect = Exception
823- fake_update_iscsi_targets = self.patch(
824- boot_resources, 'update_iscsi_targets')
825 fake_cleanup_snapshots_and_cache = self.patch(
826 boot_resources, 'cleanup_snapshots_and_cache')
827
828@@ -829,7 +666,6 @@
829 ],
830 self.assertRaises(
831 Exception, boot_resources.import_images, sources)
832- self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
833 self.assertThat(fake_cleanup_snapshots_and_cache, MockCalledOnce())
834
835 def test__runs_import_and_returns_true(self):
836@@ -846,13 +682,9 @@
837 self.patch(boot_resources, 'download_all_boot_resources')
838 fake_write_snapshot_metadata = self.patch(
839 boot_resources, 'write_snapshot_metadata')
840- fake_targets_conf = self.patch(
841- boot_resources, 'write_targets_conf')
842 fake_link_bootloaders = self.patch(boot_resources, 'link_bootloaders')
843 fake_update_current_symlink = self.patch(
844 boot_resources, 'update_current_symlink')
845- fake_update_iscsi_targets = self.patch(
846- boot_resources, 'update_iscsi_targets')
847 fake_cleanup_snapshots_and_cache = self.patch(
848 boot_resources, 'cleanup_snapshots_and_cache')
849
850@@ -874,8 +706,6 @@
851 ],
852 self.assertTrue(boot_resources.import_images(sources))
853 self.assertThat(fake_write_snapshot_metadata, MockCalledOnce())
854- self.assertThat(fake_targets_conf, MockCalledOnce())
855 self.assertThat(fake_link_bootloaders, MockCalledOnce())
856 self.assertThat(fake_update_current_symlink, MockCalledOnce())
857- self.assertThat(fake_update_iscsi_targets, MockCalledOnce())
858 self.assertThat(fake_cleanup_snapshots_and_cache, MockCalledOnce())
859
860=== modified file 'src/provisioningserver/import_images/tests/test_download_resources.py'
861--- src/provisioningserver/import_images/tests/test_download_resources.py 2017-01-28 00:51:47 +0000
862+++ src/provisioningserver/import_images/tests/test_download_resources.py 2017-06-21 22:15:36 +0000
863@@ -230,38 +230,6 @@
864 label=product['label'], subarches={subarch},
865 bootloader_type=None))
866
867- def test_inserts_root_image(self):
868- product_mapping = ProductMapping()
869- subarch = factory.make_name('subarch')
870- product = self.make_product(ftype='root-image.gz', subarch=subarch)
871- product_mapping.add(product, subarch)
872- repo_writer = download_resources.RepoWriter(
873- None, None, product_mapping)
874- self.patch(
875- download_resources, 'products_exdata').return_value = product
876- # Prevent MAAS from trying to actually write the file.
877- mock_insert_root_image = self.patch(
878- download_resources, 'insert_root_image')
879- mock_link_resources = self.patch(download_resources, 'link_resources')
880- # We only need to provide the product as the other fields are only used
881- # when writing the actual files to disk.
882- repo_writer.insert_item(product, None, None, None, None)
883- # None is used for the store and the content source as we're not
884- # writing anything to disk.
885- self.assertThat(
886- mock_insert_root_image,
887- MockCalledOnceWith(
888- None, product['sha256'], {'sha256': product['sha256']},
889- product['size'], None))
890- # links are mocked out by the mock_insert_file above.
891- self.assertThat(
892- mock_link_resources,
893- MockCalledOnceWith(
894- snapshot_path=None, links=mock.ANY, osystem=product['os'],
895- arch=product['arch'], release=product['release'],
896- label=product['label'], subarches={subarch},
897- bootloader_type=None))
898-
899 def test_inserts_file(self):
900 product_mapping = ProductMapping()
901 subarch = factory.make_name('subarch')
902
903=== removed file 'src/provisioningserver/import_images/tests/test_uec2roottar.py'
904--- src/provisioningserver/import_images/tests/test_uec2roottar.py 2016-06-22 17:03:02 +0000
905+++ src/provisioningserver/import_images/tests/test_uec2roottar.py 1970-01-01 00:00:00 +0000
906@@ -1,370 +0,0 @@
907-# Copyright 2014-2016 Canonical Ltd. This software is licensed under the
908-# GNU Affero General Public License version 3 (see the file LICENSE).
909-
910-"""Tests for the `uec2roottar` script and its supporting module.."""
911-
912-__all__ = []
913-
914-import os
915-import os.path
916-from subprocess import CalledProcessError
917-from unittest import mock
918-
919-from maastesting.factory import factory
920-from maastesting.matchers import (
921- MockAnyCall,
922- MockCalledOnceWith,
923- MockNotCalled,
924-)
925-from maastesting.testcase import MAASTestCase
926-from provisioningserver.import_images import uec2roottar
927-from testtools.matchers import HasLength
928-from testtools.testcase import ExpectedException
929-
930-
931-def make_image_name(suffix='.img'):
932- """Create an image file name (but not the actual file)."""
933- return factory.make_name('root') + suffix
934-
935-
936-def make_image(testcase, contents=None, suffix='.img'):
937- """Create an image file."""
938- name = make_image_name(suffix)
939- return testcase.make_file(name=name, contents=contents)
940-
941-
942-def make_tarball_name(prefix='tarball'):
943- """Create an arbitrary name for a tarball."""
944- return factory.make_name(prefix) + '.tar.gz'
945-
946-
947-def make_roottar_location(testcase):
948- """Create a name for an output root tarball, in an empty directory."""
949- name = make_tarball_name('root')
950- return os.path.join(testcase.make_dir(), name)
951-
952-
953-def patch_is_filesystem_file(testcase, answer):
954- """Patch `is_filesystem_file` to return the given answer."""
955- testcase.patch(uec2roottar, 'is_filesystem_file').return_value = answer
956-
957-
958-class TestMakeArgParser(MAASTestCase):
959- """Tests for `make_argparser`."""
960-
961- def test__defines_expected_options(self):
962- image = make_image(self)
963- output = make_roottar_location(self)
964- user = factory.make_name('user')
965-
966- parser = uec2roottar.make_argparser(factory.make_string())
967- args = parser.parse_args([image, output, '--user', user])
968-
969- self.assertEqual(
970- (
971- image,
972- output,
973- user,
974- ),
975- (
976- args.image,
977- args.output,
978- args.user,
979- ))
980-
981- def test__user_defaults_to_None(self):
982- parser = uec2roottar.make_argparser(factory.make_string())
983- args = parser.parse_args(
984- [make_image(self), make_roottar_location(self)])
985- self.assertIsNone(args.user)
986-
987-
988-class TestIsFilesystemFile(MAASTestCase):
989- """Tests for `is_filesystem_file`."""
990-
991- def test__returns_True_if_file_looks_like_filesystem(self):
992- image = make_image(self, suffix='.img')
993- self.patch(uec2roottar, 'check_output').return_value = (
994- ("%s: filesystem data" % image).encode('utf-8'))
995- self.assertTrue(uec2roottar.is_filesystem_file(image))
996-
997- def test__returns_False_for_tarball(self):
998- image = make_image(self, suffix='.tar.gz')
999- self.patch(uec2roottar, 'check_output').return_value = (
1000- ("%s: gzip compressed data, was ..." % image).encode('utf-8'))
1001- self.assertFalse(uec2roottar.is_filesystem_file(image))
1002-
1003- def test__calls_file_with_C_language_setting(self):
1004- env_during_invocation = {}
1005-
1006- def fake_check_output(*args, **kwargs):
1007- env_during_invocation.update(os.environ)
1008- return b''
1009-
1010- self.patch(uec2roottar, 'check_output', fake_check_output)
1011-
1012- uec2roottar.is_filesystem_file(make_image(self))
1013-
1014- self.assertEqual('C', env_during_invocation.get('LANG'))
1015-
1016-
1017-class TestExtractImageFromTarball(MAASTestCase):
1018- """Tests for `extract_image_from_tarball`."""
1019-
1020- def test__extracts_image(self):
1021- tarball = make_tarball_name()
1022- self.patch(uec2roottar, 'check_call')
1023- self.patch(uec2roottar, 'check_output')
1024- # Cheat: patch away extraction of the tarball, but pass a temporary
1025- # directory with an image already in it. The function will think it
1026- # just extracted the image from the tarball.
1027- image = make_image(self)
1028- working_dir = os.path.dirname(image)
1029-
1030- result = uec2roottar.extract_image_from_tarball(tarball, working_dir)
1031-
1032- self.assertThat(
1033- uec2roottar.check_call,
1034- MockCalledOnceWith([
1035- 'tar',
1036- '-C', working_dir,
1037- '--wildcards', '*.img',
1038- '-Sxvzf',
1039- tarball,
1040- ]))
1041- self.assertEqual(image, result)
1042-
1043- def test__ignores_other_files(self):
1044- tarball = make_tarball_name()
1045- self.patch(uec2roottar, 'check_call')
1046- self.patch(uec2roottar, 'check_output')
1047- # Make the function think that it found two files in the tarball: an
1048- # image and some other file.
1049- image = make_image(self)
1050- working_dir = os.path.dirname(image)
1051- # This other file doesn't upset things, because it doesn't look like
1052- # an image file.
1053- factory.make_file(working_dir)
1054-
1055- self.assertEqual(
1056- image,
1057- uec2roottar.extract_image_from_tarball(tarball, working_dir))
1058-
1059- def test__fails_if_no_image_found(self):
1060- tarball = make_tarball_name()
1061- self.patch(uec2roottar, 'check_call')
1062- self.patch(uec2roottar, 'check_output')
1063- empty_dir = self.make_dir()
1064- error = self.assertRaises(
1065- uec2roottar.ImageFileError,
1066- uec2roottar.extract_image_from_tarball, tarball, empty_dir)
1067- self.assertEqual(
1068- "Tarball %s does not contain any *.img." % tarball,
1069- str(error))
1070-
1071- def test__fails_if_multiple_images_found(self):
1072- tarball = make_tarball_name()
1073- self.patch(uec2roottar, 'check_call')
1074- self.patch(uec2roottar, 'check_output')
1075- working_dir = self.make_dir()
1076- files = sorted(
1077- factory.make_file(working_dir, name=make_image_name())
1078- for _ in range(2))
1079- error = self.assertRaises(
1080- uec2roottar.ImageFileError,
1081- uec2roottar.extract_image_from_tarball, tarball, working_dir)
1082- self.assertEqual(
1083- "Tarball %s contains multiple image files: %s."
1084- % (tarball, ', '.join(files)),
1085- str(error))
1086-
1087-
1088-class TestGetImageFile(MAASTestCase):
1089- """Tests for `get_image_file`."""
1090-
1091- def test__returns_actual_image_file_unchanged(self):
1092- patch_is_filesystem_file(self, True)
1093- image = make_image(self)
1094- self.assertEqual(
1095- image,
1096- uec2roottar.get_image_file(image, factory.make_name('dir')))
1097-
1098- def test__extracts_tarball_into_temp_dir(self):
1099- patch_is_filesystem_file(self, False)
1100- tarball = make_tarball_name()
1101- temp_dir = self.make_dir()
1102- image = make_image_name()
1103- patch = self.patch(uec2roottar, 'extract_image_from_tarball')
1104- patch.return_value = image
1105- result = uec2roottar.get_image_file(tarball, temp_dir)
1106- self.assertEqual(image, result)
1107- self.assertThat(patch, MockCalledOnceWith(tarball, temp_dir))
1108-
1109- def test__rejects_other_files(self):
1110- patch_is_filesystem_file(self, False)
1111- filename = factory.make_name('weird-file')
1112- error = self.assertRaises(
1113- uec2roottar.ImageFileError,
1114- uec2roottar.get_image_file, filename, factory.make_name('dir'))
1115- self.assertEqual(
1116- "Expected '%s' to be either a filesystem file, or a "
1117- "gzipped tarball containing one." % filename,
1118- str(error))
1119-
1120-
1121-class TestUnmount(MAASTestCase):
1122- """Tests for `unmount`."""
1123-
1124- def test__calls_umount(self):
1125- self.patch(uec2roottar, 'check_call')
1126- mountpoint = factory.make_name('mount')
1127- uec2roottar.unmount(mountpoint)
1128- self.assertThat(
1129- uec2roottar.check_call,
1130- MockCalledOnceWith(['umount', mountpoint]))
1131-
1132- def test__propagates_failure(self):
1133- failure = CalledProcessError(9, factory.make_name('delibfail'))
1134- self.patch(uec2roottar, 'check_call').side_effect = failure
1135- self.patch(uec2roottar, 'maaslog')
1136- mountpoint = factory.make_name('mount')
1137- self.assertRaises(CalledProcessError, uec2roottar.unmount, mountpoint)
1138- self.assertThat(
1139- uec2roottar.maaslog.error,
1140- MockCalledOnceWith(
1141- "Could not unmount %s: %s", mountpoint, failure))
1142-
1143-
1144-class TestLoopMount(MAASTestCase):
1145- """Tests for `loop_mount`."""
1146-
1147- def test__mounts_and_unmounts_image(self):
1148- image = make_image_name()
1149- self.patch(uec2roottar, 'check_call')
1150- mountpoint = factory.make_name('mount')
1151-
1152- calls_before = len(uec2roottar.check_call.mock_calls)
1153- with uec2roottar.loop_mount(image, mountpoint):
1154- calls_during = len(uec2roottar.check_call.mock_calls)
1155- calls_after = len(uec2roottar.check_call.mock_calls)
1156-
1157- self.assertEqual(
1158- (0, 1, 2),
1159- (calls_before, calls_during, calls_after))
1160- self.assertThat(
1161- uec2roottar.check_call,
1162- MockAnyCall(['mount', '-o', 'ro', image, mountpoint]))
1163- self.assertThat(
1164- uec2roottar.check_call,
1165- MockAnyCall(['umount', mountpoint]))
1166-
1167- def test__cleans_up_after_failure(self):
1168- class DeliberateException(Exception):
1169- pass
1170-
1171- self.patch(uec2roottar, 'check_call')
1172- image = make_image_name()
1173- mountpoint = factory.make_name('mount')
1174- with ExpectedException(DeliberateException):
1175- with uec2roottar.loop_mount(image, mountpoint):
1176- raise DeliberateException()
1177-
1178- self.assertThat(
1179- uec2roottar.check_call, MockAnyCall(['umount', mountpoint]))
1180-
1181-
1182-class TestTarSupportsXattrOpts(MAASTestCase):
1183- """Tests for `tar_supports_xattr_opts`."""
1184-
1185- def test__returns_True_if_help_contains_ref_to_xattr(self):
1186- mock_check_call = self.patch(uec2roottar, 'check_output')
1187- mock_check_call.return_value = b'xattr'
1188- self.assertTrue(uec2roottar.tar_supports_xattr_opts())
1189- self.assertThat(mock_check_call, MockCalledOnceWith(['tar', '--help']))
1190-
1191- def test__returns_False_if_help_doesnt_contain_ref_to_xattr(self):
1192- mock_check_call = self.patch(uec2roottar, 'check_output')
1193- mock_check_call.return_value = b'nothing'
1194- self.assertFalse(uec2roottar.tar_supports_xattr_opts())
1195- self.assertThat(mock_check_call, MockCalledOnceWith(['tar', '--help']))
1196-
1197-
1198-class TestExtractImage(MAASTestCase):
1199- """Tests for `extract_image`."""
1200-
1201- def extract_command_line(self, call):
1202- """Extract the command line from a `mock.call` for `check_call`."""
1203- _, args, _ = call
1204- [command] = args
1205- return command
1206-
1207- def test__extracts_image_if_tar_supports_xattr(self):
1208- image = make_image_name()
1209- output = make_tarball_name()
1210- self.patch(uec2roottar, 'check_call')
1211- self.patch(uec2roottar, 'tar_supports_xattr_opts').return_value = False
1212- uec2roottar.extract_image(image, output)
1213- self.assertThat(uec2roottar.check_call.mock_calls, HasLength(3))
1214- [mount_call, tar_call, umount_call] = uec2roottar.check_call.mock_calls
1215- self.assertEqual('mount', self.extract_command_line(mount_call)[0])
1216- tar_command = self.extract_command_line(tar_call)
1217- self.assertEqual(['tar', '-C'], tar_command[:2])
1218- self.assertEqual('umount', self.extract_command_line(umount_call)[0])
1219-
1220- def test__extracts_image_if_tar_doesnt_supports_xattr(self):
1221- image = make_image_name()
1222- output = make_tarball_name()
1223- self.patch(uec2roottar, 'check_call')
1224- self.patch(uec2roottar, 'tar_supports_xattr_opts').return_value = True
1225- uec2roottar.extract_image(image, output)
1226- self.assertThat(uec2roottar.check_call.mock_calls, HasLength(3))
1227- [mount_call, tar_call, umount_call] = uec2roottar.check_call.mock_calls
1228- self.assertEqual('mount', self.extract_command_line(mount_call)[0])
1229- tar_command = self.extract_command_line(tar_call)
1230- self.assertEqual(
1231- ['tar', '--xattrs', '--xattrs-include=*', '-C'], tar_command[:4])
1232- self.assertEqual('umount', self.extract_command_line(umount_call)[0])
1233-
1234-
1235-class TestSetOwnership(MAASTestCase):
1236- """Tests for `set_ownership`."""
1237-
1238- def test__does_nothing_if_no_user_specified(self):
1239- self.patch(uec2roottar, 'check_call')
1240- uec2roottar.set_ownership(make_tarball_name(), user=None)
1241- self.assertThat(uec2roottar.check_call, MockNotCalled())
1242-
1243- def test__calls_chown_if_user_specified(self):
1244- self.patch(uec2roottar, 'check_call')
1245- user = factory.make_name('user')
1246- tarball = make_tarball_name()
1247- uec2roottar.set_ownership(tarball, user=user)
1248- self.assertThat(
1249- uec2roottar.check_call,
1250- MockCalledOnceWith(['/bin/chown', user, tarball]))
1251-
1252-
1253-class TestUEC2RootTar(MAASTestCase):
1254- """Integration tests for `uec2roottar`."""
1255-
1256- def make_args(self, **kwargs):
1257- """Fake an `argparser` arguments object."""
1258- args = mock.Mock()
1259- for key, value in kwargs.items():
1260- setattr(args, key, value)
1261- return args
1262-
1263- def test__integrates(self):
1264- image_name = factory.make_name('root-image') + '.img'
1265- image = self.make_file(name=image_name)
1266- output_name = factory.make_name('root-tar') + '.tar.gz'
1267- output = os.path.join(self.make_dir(), output_name)
1268- args = self.make_args(image=image, output=output)
1269- self.patch(uec2roottar, 'check_call')
1270- self.patch(uec2roottar, 'check_output')
1271- patch_is_filesystem_file(self, True)
1272-
1273- uec2roottar.main(args)
1274-
1275- self.assertThat(
1276- uec2roottar.is_filesystem_file, MockCalledOnceWith(image))
1277
1278=== removed file 'src/provisioningserver/import_images/uec2roottar.py'
1279--- src/provisioningserver/import_images/uec2roottar.py 2015-12-01 18:12:59 +0000
1280+++ src/provisioningserver/import_images/uec2roottar.py 1970-01-01 00:00:00 +0000
1281@@ -1,187 +0,0 @@
1282-# Copyright 2014-2015 Canonical Ltd. This software is licensed under the
1283-# GNU Affero General Public License version 3 (see the file LICENSE).
1284-
1285-"""Code for the `uec2roottar` script."""
1286-
1287-__all__ = [
1288- 'main',
1289- 'make_argparser',
1290- ]
1291-
1292-import argparse
1293-from contextlib import contextmanager
1294-from glob import glob
1295-import os.path
1296-from subprocess import (
1297- check_call,
1298- check_output,
1299-)
1300-
1301-from provisioningserver.logger import get_maas_logger
1302-from provisioningserver.utils.env import environment_variables
1303-from provisioningserver.utils.fs import tempdir
1304-
1305-
1306-maaslog = get_maas_logger("uec2roottar")
1307-
1308-
1309-def make_argparser(description):
1310- """Create an `ArgumentParser` for this script."""
1311- parser = argparse.ArgumentParser(description=description)
1312- parser.add_argument(
1313- 'image', metavar='IMAGE-FILE', help="Input file: UEC root image.")
1314- parser.add_argument(
1315- 'output', metavar='TARBALL', help="Output file: root tarball.")
1316- parser.add_argument(
1317- '--user', '-u', help="Set output file ownership to USER.")
1318- return parser
1319-
1320-
1321-def is_filesystem_file(path):
1322- """Does the file at `path` look like a filesystem-in-a-file?"""
1323- # Identify filesystems using the "file" utility. We'll be parsing the
1324- # output, so suppress any translation.
1325- with environment_variables({'LANG': 'C'}):
1326- output = check_output(['file', path])
1327- return b"filesystem data" in output
1328-
1329-
1330-class ImageFileError(Exception):
1331- """Problem with the given image file."""
1332-
1333-
1334-def extract_image_from_tarball(tarball, working_dir):
1335- """Extract image file from `tarball` into `working_dir`, return its path.
1336-
1337- This may extract multiple files into `working_dir`; it looks for files with
1338- names like `*.img`. The function only succeeds, however, if there is
1339- exactly one of those, in the tarball's root directory.
1340- """
1341- glob_pattern = '*.img'
1342- maaslog.debug(
1343- "Extracting %s from %s into %s.", glob_pattern, tarball, working_dir)
1344- check_call([
1345- 'tar',
1346- '-C', working_dir,
1347- '--wildcards', glob_pattern,
1348- '-Sxvzf', tarball,
1349- ])
1350- # Look for .img files. Sort just so that if there is more than one image
1351- # file, we'll produce a consistent error message.
1352- candidates = sorted(glob(os.path.join(working_dir, glob_pattern)))
1353- if len(candidates) == 0:
1354- raise ImageFileError(
1355- "Tarball %s does not contain any %s." % (tarball, glob_pattern))
1356- if len(candidates) > 1:
1357- raise ImageFileError(
1358- "Tarball %s contains multiple image files: %s."
1359- % (tarball, ', '.join(candidates)))
1360- [image] = candidates
1361- return image
1362-
1363-
1364-def get_image_file(path, temp_dir):
1365- """Return image file at, or contained in tarball at, `path`.
1366-
1367- :param path: Path to the image file. Must point to either a file
1368- containing a filesystem, or a tarball containing one, of the same
1369- base name.
1370- :param temp_dir: A temporary working directory. If the image needs to be
1371- extracted from a tarball, the tarball will be extracted here.
1372- """
1373- if is_filesystem_file(path):
1374- # Easy. This is the actual image file.
1375- return path
1376- elif path.endswith('.tar.gz'):
1377- # Tarball. Extract image file.
1378- return extract_image_from_tarball(path, temp_dir)
1379- else:
1380- raise ImageFileError(
1381- "Expected '%s' to be either a filesystem file, or "
1382- "a gzipped tarball containing one." % path)
1383-
1384-
1385-def unmount(mountpoint):
1386- """Unmount filesystem at given mount point.
1387-
1388- If this fails, it logs the error as well as raising it. This means that
1389- error code paths can suppress the exception without depriving the user of
1390- the information.
1391- """
1392- try:
1393- check_call(['umount', mountpoint])
1394- except BaseException as e:
1395- maaslog.error("Could not unmount %s: %s", mountpoint, e)
1396- raise
1397-
1398-
1399-@contextmanager
1400-def loop_mount(image, mountpoint):
1401- """Context manager: temporarily loop-mount `image` at `mountpoint`."""
1402- check_call(['mount', '-o', 'ro', image, mountpoint])
1403- try:
1404- yield
1405- except:
1406- try:
1407- unmount(mountpoint)
1408- except Exception:
1409- # This is probably a secondary error resulting from the original
1410- # problem. Stick with the original exception.
1411- pass
1412- raise
1413- else:
1414- # Unmount after successful run. If this fails, let the exception
1415- # propagate.
1416- unmount(mountpoint)
1417-
1418-
1419-def tar_supports_xattr_opts():
1420- """Returns True if the system's tar supports the 'xattrs' options."""
1421- out = check_output(['tar', '--help'])
1422- return b"xattr" in out
1423-
1424-
1425-def extract_image(image, output):
1426- """Loop-mount `image`, and tar its contents into `output`."""
1427-
1428- xattr_opts = []
1429- if tar_supports_xattr_opts():
1430- # Only add the xattrs options if tar supports it.
1431- # For insance tar on 12.04 does *not* support xattrs.
1432- xattr_opts = ['--xattrs', '--xattrs-include=*']
1433- with tempdir() as mountpoint:
1434- cmd = ['tar'] + xattr_opts + [
1435- # Work from mountpoint as the current directory.
1436- '-C', mountpoint,
1437- # Options:
1438- # -c: Create tarfile.
1439- # -p: Preserve permissions.
1440- # -S: Handle sparse files efficiently (images have those).
1441- # -z: Compress using gzip.
1442- # -f: Work on given tar file.
1443- '-cpSzf', output,
1444- '--numeric-owner',
1445- # Tar up the "current directory": the mountpoint.
1446- '.',
1447- ]
1448-
1449- with loop_mount(image, mountpoint):
1450- check_call(cmd)
1451-
1452-
1453-def set_ownership(path, user=None):
1454- """Set file ownership to `user` if specified."""
1455- if user is not None:
1456- maaslog.debug("Setting file owner to %s.", user)
1457- check_call(['/bin/chown', user, path])
1458-
1459-
1460-def main(args):
1461- """Do the work: loop-mount image, write contents to output file."""
1462- output = args.output
1463- maaslog.debug("Converting %s to %s.", args.image, output)
1464- with tempdir() as working_dir:
1465- image = get_image_file(args.image, working_dir)
1466- extract_image(image, output)
1467- set_ownership(output, args.user)
1468- maaslog.debug("Finished. Wrote to %s.", output)
1469
1470=== modified file 'src/provisioningserver/kernel_opts.py'
1471--- src/provisioningserver/kernel_opts.py 2017-06-21 22:15:36 +0000
1472+++ src/provisioningserver/kernel_opts.py 2017-06-21 22:15:36 +0000
1473@@ -42,7 +42,6 @@
1474 "fs_host", # Host/IP on which ephemeral filesystems are hosted.
1475 "extra_opts", # String of extra options to supply, will be appended
1476 # verbatim to the kernel command line
1477- "http_boot", # Whether or not to boot over HTTP.
1478 ))
1479
1480
1481@@ -72,56 +71,18 @@
1482 return max(dirs)
1483
1484
1485-ISCSI_TARGET_NAME_PREFIX = "iqn.2004-05.com.ubuntu:maas"
1486-
1487-
1488-def get_ephemeral_name(osystem, arch, subarch, release, label):
1489- """Return the name of the most recent ephemeral image."""
1490- return "ephemeral-%s-%s-%s-%s-%s" % (
1491- osystem,
1492- arch,
1493- subarch,
1494- release,
1495- label
1496- )
1497-
1498-
1499-def prefix_target_name(name):
1500- """Prefix an ISCSI target name with the standard target-name prefix."""
1501- return "%s:%s" % (ISCSI_TARGET_NAME_PREFIX, name)
1502-
1503-
1504 def compose_purpose_opts(params):
1505 """Return the list of the purpose-specific kernel options."""
1506- if params.http_boot:
1507- kernel_params = [
1508- "root=squash:http://%s:5248/images/%s/%s/%s/%s/%s/squashfs" % (
1509- (
1510- '[%s]' % params.fs_host
1511- if IPAddress(params.fs_host).version == 6
1512- else params.fs_host
1513- ),
1514- params.osystem, params.arch, params.subarch, params.release,
1515- params.label),
1516- ]
1517- else:
1518- tname = prefix_target_name(
1519- get_ephemeral_name(
1520- params.osystem, params.arch, params.subarch,
1521- params.release, params.label))
1522- kernel_params = [
1523- # Read by the open-iscsi initramfs code.
1524- "iscsi_target_name=%s" % tname,
1525- "iscsi_target_ip=%s" % params.fs_host,
1526- "iscsi_target_port=3260",
1527- "iscsi_initiator=%s" % params.hostname,
1528- # kernel / udev name iscsi devices with this path
1529- "root=/dev/disk/by-path/ip-%s:%s-iscsi-%s-lun-1" % (
1530- params.fs_host, "3260", tname),
1531- ]
1532-
1533- kernel_params += [
1534+ kernel_params = [
1535 "ro",
1536+ "root=squash:http://%s:5248/images/%s/%s/%s/%s/%s/squashfs" % (
1537+ (
1538+ '[%s]' % params.fs_host
1539+ if IPAddress(params.fs_host).version == 6
1540+ else params.fs_host
1541+ ),
1542+ params.osystem, params.arch, params.subarch, params.release,
1543+ params.label),
1544 # Read by cloud-initramfs-dyn-netconf initramfs-tools networking
1545 # configuration in the initramfs. Choose IPv4 or IPv6 based on the
1546 # family of fs_host. If BOOTIF is set, IPv6 config uses that
1547
1548=== modified file 'src/provisioningserver/rpc/region.py'
1549--- src/provisioningserver/rpc/region.py 2017-06-21 22:15:36 +0000
1550+++ src/provisioningserver/rpc/region.py 2017-01-28 00:51:47 +0000
1551@@ -138,7 +138,6 @@
1552 (b"log_host", amp.Unicode()),
1553 (b"extra_opts", amp.Unicode()),
1554 (b"system_id", amp.Unicode(optional=True)),
1555- (b"http_boot", amp.Boolean(optional=True)),
1556 ]
1557 errors = {
1558 BootConfigNoResponse: b"BootConfigNoResponse",
1559
1560=== modified file 'src/provisioningserver/service_monitor.py'
1561--- src/provisioningserver/service_monitor.py 2016-10-17 19:19:44 +0000
1562+++ src/provisioningserver/service_monitor.py 2017-06-21 22:15:36 +0000
1563@@ -10,7 +10,6 @@
1564 from provisioningserver.rpc.exceptions import NoConnectionsAvailable
1565 from provisioningserver.rpc.region import GetControllerType
1566 from provisioningserver.utils.service_monitor import (
1567- AlwaysOnService,
1568 Service,
1569 SERVICE_STATE,
1570 ServiceMonitor,
1571@@ -59,14 +58,6 @@
1572 snap_service_name = "dhcpd6"
1573
1574
1575-class TGTService(AlwaysOnService):
1576- """Monitored tgt service."""
1577-
1578- name = "tgt"
1579- service_name = "tgt"
1580- snap_service_name = "tgt"
1581-
1582-
1583 class NTPServiceOnRack(Service):
1584 """Monitored NTP service on a rack controller host."""
1585
1586@@ -100,5 +91,4 @@
1587 DHCPv4Service(),
1588 DHCPv6Service(),
1589 NTPServiceOnRack(),
1590- TGTService(),
1591 )
1592
1593=== modified file 'src/provisioningserver/support_dump.py'
1594--- src/provisioningserver/support_dump.py 2016-06-06 09:01:44 +0000
1595+++ src/provisioningserver/support_dump.py 2017-06-21 22:15:36 +0000
1596@@ -91,9 +91,6 @@
1597 "title": "get_image_metadata()",
1598 "function": lambda *args: json.loads(get_image_metadata(*args))
1599 },
1600- {
1601- "command": "/usr/sbin/tgt-admin --show"
1602- }
1603 ]
1604
1605
1606
1607=== modified file 'src/provisioningserver/tests/test_kernel_opts.py'
1608--- src/provisioningserver/tests/test_kernel_opts.py 2017-06-21 22:15:36 +0000
1609+++ src/provisioningserver/tests/test_kernel_opts.py 2017-06-21 22:15:36 +0000
1610@@ -47,8 +47,6 @@
1611 {field: factory.make_name(field)
1612 for field in KernelParameters._fields
1613 if field not in parms})
1614- if not isinstance(parms['http_boot'], bool):
1615- parms['http_boot'] = True
1616 params = KernelParameters(**parms)
1617
1618 if testcase is not None:
1619@@ -353,12 +351,3 @@
1620 (
1621 params.fs_host, params.osystem, params.arch,
1622 params.subarch, params.release, params.label)]))
1623-
1624- def test_compose_rootfs_over_iscsi(self):
1625- params = make_kernel_parameters(
1626- fs_host=factory.make_ipv6_address(), http_boot=False)
1627- self.assertThat(
1628- compose_kernel_command_line(params),
1629- ContainsAll([
1630- "root=/dev/disk/by-path/ip-",
1631- "iscsi_initiator="]))
1632
1633=== modified file 'src/provisioningserver/tests/test_service_monitor.py'
1634--- src/provisioningserver/tests/test_service_monitor.py 2017-01-28 00:51:47 +0000
1635+++ src/provisioningserver/tests/test_service_monitor.py 2017-06-21 22:15:36 +0000
1636@@ -21,7 +21,6 @@
1637 DHCPv6Service,
1638 NTPServiceOnRack,
1639 service_monitor,
1640- TGTService,
1641 )
1642 from provisioningserver.utils.service_monitor import SERVICE_STATE
1643 from testtools.matchers import Equals
1644@@ -98,17 +97,6 @@
1645 self.assertEqual("maas-dhcpd6", service.service_name)
1646
1647
1648-class TestTGTService(MAASTestCase):
1649-
1650- def test_service_name(self):
1651- tgt = TGTService()
1652- self.assertEqual("tgt", tgt.service_name)
1653-
1654- def test_getExpectedState(self):
1655- tgt = TGTService()
1656- self.assertEqual((SERVICE_STATE.ON, None), tgt.getExpectedState())
1657-
1658-
1659 class TestNTPServiceOnRack(MAASTestCase):
1660
1661 def test_name_and_service_name(self):
1662@@ -161,5 +149,5 @@
1663
1664 def test__includes_all_services(self):
1665 self.assertItemsEqual(
1666- ["dhcpd", "dhcpd6", "ntp_rack", "tgt"],
1667+ ["dhcpd", "dhcpd6", "ntp_rack"],
1668 service_monitor._services.keys())
1669
1670=== modified file 'src/provisioningserver/tests/test_upgrade_cluster.py'
1671--- src/provisioningserver/tests/test_upgrade_cluster.py 2017-03-30 15:48:28 +0000
1672+++ src/provisioningserver/tests/test_upgrade_cluster.py 2017-06-21 22:15:36 +0000
1673@@ -344,20 +344,3 @@
1674 os.path.join(
1675 storage_dir, 'current', arch, subarch, release, label))
1676 return storage_dir
1677-
1678- def test__calls_write_targets_conf_with_current_dir(self):
1679- storage_dir = self.setup_working_migration_scenario()
1680- mock_write = self.patch(upgrade_cluster, 'write_targets_conf')
1681- self.patch(upgrade_cluster, 'update_targets_conf')
1682- upgrade_cluster.migrate_architectures_into_ubuntu_directory()
1683- self.assertThat(
1684- mock_write,
1685- MockCalledOnceWith(os.path.join(storage_dir, 'current')))
1686-
1687- def test__calls_update_targets_conf_with_current_dir(self):
1688- storage_dir = self.setup_working_migration_scenario()
1689- mock_update = self.patch(upgrade_cluster, 'update_targets_conf')
1690- upgrade_cluster.migrate_architectures_into_ubuntu_directory()
1691- self.assertThat(
1692- mock_update,
1693- MockCalledOnceWith(os.path.join(storage_dir, 'current')))
1694
1695=== modified file 'src/provisioningserver/upgrade_cluster.py'
1696--- src/provisioningserver/upgrade_cluster.py 2016-12-28 19:06:18 +0000
1697+++ src/provisioningserver/upgrade_cluster.py 2017-06-21 22:15:36 +0000
1698@@ -35,10 +35,6 @@
1699 list_subdirs,
1700 )
1701 from provisioningserver.config import ClusterConfiguration
1702-from provisioningserver.import_images.boot_resources import (
1703- update_targets_conf,
1704- write_targets_conf,
1705-)
1706 from provisioningserver.logger import get_maas_logger
1707 from provisioningserver.utils import snappy
1708
1709@@ -192,11 +188,6 @@
1710 for arch in arches:
1711 shutil.move(os.path.join(current_dir, arch), ubuntu_dir)
1712
1713- # Re-write the maas.tgt to point to the new location for the ubuntu boot
1714- # resources.
1715- write_targets_conf(current_dir)
1716- update_targets_conf(current_dir)
1717-
1718
1719 # Upgrade hooks, from oldest to newest. The hooks are callables, taking no
1720 # arguments. They are called in order.