Merge lp:~blake-rouse/maas/fix-pod-delete into lp:~maas-committers/maas/trunk

Proposed by Blake Rouse
Status: Merged
Approved by: Blake Rouse
Approved revision: no longer in the source branch.
Merged at revision: 5750
Proposed branch: lp:~blake-rouse/maas/fix-pod-delete
Merge into: lp:~maas-committers/maas/trunk
Diff against target: 684 lines (+324/-24)
14 files modified
src/maasserver/api/pods.py (+9/-1)
src/maasserver/api/tests/test_pods.py (+17/-2)
src/maasserver/enum.py (+11/-0)
src/maasserver/forms/pods.py (+6/-2)
src/maasserver/migrations/builtin/maasserver/0114_node_dynamic_to_creation_type.py (+26/-0)
src/maasserver/models/bmc.py (+115/-3)
src/maasserver/models/node.py (+6/-5)
src/maasserver/models/signals/nodes.py (+3/-1)
src/maasserver/models/signals/tests/test_nodes.py (+15/-3)
src/maasserver/models/tests/test_bmc.py (+109/-2)
src/maasserver/models/tests/test_node.py (+3/-1)
src/maasserver/testing/factory.py (+2/-2)
src/maasserver/websockets/handlers/device.py (+1/-1)
src/maasserver/websockets/handlers/machine.py (+1/-1)
To merge this branch: bzr merge lp:~blake-rouse/maas/fix-pod-delete
Reviewer Review Type Date Requested Status
Newell Jensen (community) Approve
Review via email: mp+317653@code.launchpad.net

Commit message

Fix pod deletion to only decompose machines that MAAS has composed. Convert pod deletion into an async operation.

MAAS know records the creation type of a machine. This type allows MAAS to know what to do upon deletion of that machine. If the machine existed in the pod or was added to the pod out of band from MAAS then it will not be decomposed when the machine or pod is deleted. If the machine was created by MAAS then it will be decomposed in the pod when the machine or pod is deleted. The pod delete operation is not an asynchronous operation where it tries its best to decompose all machines. If decomposing fails then the deletion process will be stopped and the reason for the failure will be raised. If all the machines that required decomposition are successfully decomposed then it will also delete the pre-existing machines and the pod.

To post a comment you must log in.
Revision history for this message
Newell Jensen (newell-jensen) wrote :

I am getting this error when doing a `make run` with your branch (I initially saw this when trying to install packages for RSD testing).

django.db.utils.ProgrammingError: column maasserver_node.creation_type does not exist
LINE 1: ...node"."netboot", "maasserver_node"."license_key", "maasserve...
                                                             ^

review: Needs Fixing
Revision history for this message
Newell Jensen (newell-jensen) wrote :

I tested this in the RSD lab. Deleting the Pod work with or without pre-composed nodes present.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'src/maasserver/api/pods.py'
2--- src/maasserver/api/pods.py 2017-02-17 14:23:04 +0000
3+++ src/maasserver/api/pods.py 2017-02-18 00:02:02 +0000
4@@ -13,12 +13,14 @@
5 operation,
6 OperationsHandler,
7 )
8+from maasserver.enum import NODE_CREATION_TYPE
9 from maasserver.exceptions import MAASAPIValidationError
10 from maasserver.forms.pods import (
11 ComposeMachineForm,
12 PodForm,
13 )
14 from maasserver.models.bmc import Pod
15+from maasserver.models.node import Machine
16 from piston3.utils import rc
17 from provisioningserver.drivers.pod import Capabilities
18
19@@ -108,7 +110,13 @@
20 Returns 204 if the pod is successfully deleted.
21 """
22 pod = get_object_or_404(Pod, id=id)
23- pod.delete()
24+ # Calculate the wait time based on the number of none pre-existing
25+ # machines. We allow maximum of 60 seconds per machine plus 60 seconds
26+ # for the pod.
27+ num_machines = Machine.objects.filter(bmc=pod)
28+ num_machines = num_machines.exclude(
29+ creation_type=NODE_CREATION_TYPE.PRE_EXISTING)
30+ pod.async_delete().wait((num_machines.count() * 60) + 60)
31 return rc.DELETED
32
33 @admin_method
34
35=== modified file 'src/maasserver/api/tests/test_pods.py'
36--- src/maasserver/api/tests/test_pods.py 2017-02-17 14:23:04 +0000
37+++ src/maasserver/api/tests/test_pods.py 2017-02-18 00:02:02 +0000
38@@ -10,12 +10,15 @@
39 from unittest.mock import MagicMock
40
41 from django.core.urlresolvers import reverse
42+from maasserver.enum import NODE_CREATION_TYPE
43 from maasserver.forms import pods
44+from maasserver.models.bmc import Pod
45 from maasserver.models.node import Machine
46 from maasserver.testing.api import APITestCase
47 from maasserver.testing.factory import factory
48 from maasserver.utils.converters import json_load_bytes
49 from maasserver.utils.orm import reload_object
50+from maastesting.matchers import MockCalledOnceWith
51 from provisioningserver.drivers.pod import (
52 Capabilities,
53 DiscoveredMachine,
54@@ -335,13 +338,25 @@
55 self.assertEqual(
56 http.client.BAD_REQUEST, response.status_code, response.content)
57
58- def test_DELETE_removes_pod(self):
59+ def test_DELETE_calls_async_delete(self):
60 self.become_admin()
61 pod = factory.make_Pod()
62+ for _ in range(3):
63+ factory.make_Machine(
64+ bmc=pod, creation_type=NODE_CREATION_TYPE.PRE_EXISTING)
65+ for _ in range(3):
66+ factory.make_Machine(
67+ bmc=pod, creation_type=NODE_CREATION_TYPE.MANUAL)
68+ for _ in range(3):
69+ factory.make_Machine(
70+ bmc=pod, creation_type=NODE_CREATION_TYPE.DYNAMIC)
71+ mock_eventual = MagicMock()
72+ mock_async_delete = self.patch(Pod, "async_delete")
73+ mock_async_delete.return_value = mock_eventual
74 response = self.client.delete(get_pod_uri(pod))
75 self.assertEqual(
76 http.client.NO_CONTENT, response.status_code, response.content)
77- self.assertIsNone(reload_object(pod))
78+ self.assertThat(mock_eventual.wait, MockCalledOnceWith(60 * 7))
79
80 def test_DELETE_rejects_deletion_if_not_permitted(self):
81 pod = factory.make_Pod()
82
83=== modified file 'src/maasserver/enum.py'
84--- src/maasserver/enum.py 2017-01-28 00:51:47 +0000
85+++ src/maasserver/enum.py 2017-02-18 00:02:02 +0000
86@@ -188,6 +188,17 @@
87 )
88
89
90+class NODE_CREATION_TYPE:
91+ """Creation types a machine can have in a node."""
92+ #: Pre-existing, machine already exists in Pod when MAAS discovered it.
93+ PRE_EXISTING = 1
94+ #: Manual, machine was manually composed through MAAS.
95+ MANUAL = 2
96+ #: Dynamic, machine was composed during allocation and should be decomposed
97+ # upon release of the machine.
98+ DYNAMIC = 3
99+
100+
101 class NODE_PERMISSION:
102 """Permissions relating to nodes."""
103 VIEW = 'view_node'
104
105=== modified file 'src/maasserver/forms/pods.py'
106--- src/maasserver/forms/pods.py 2017-02-10 17:51:00 +0000
107+++ src/maasserver/forms/pods.py 2017-02-18 00:02:02 +0000
108@@ -23,7 +23,10 @@
109 discover_pod,
110 get_best_discovered_result,
111 )
112-from maasserver.enum import BMC_TYPE
113+from maasserver.enum import (
114+ BMC_TYPE,
115+ NODE_CREATION_TYPE,
116+)
117 from maasserver.exceptions import PodProblem
118 from maasserver.forms import MAASModelForm
119 from maasserver.models import (
120@@ -278,6 +281,7 @@
121
122 created_machine = self.pod.create_machine(
123 discovered_machine, self.request.user,
124- skip_commissioning=skip_commissioning)
125+ skip_commissioning=skip_commissioning,
126+ creation_type=NODE_CREATION_TYPE.MANUAL)
127 self.pod.sync_hints(pod_hints)
128 return created_machine
129
130=== added file 'src/maasserver/migrations/builtin/maasserver/0114_node_dynamic_to_creation_type.py'
131--- src/maasserver/migrations/builtin/maasserver/0114_node_dynamic_to_creation_type.py 1970-01-01 00:00:00 +0000
132+++ src/maasserver/migrations/builtin/maasserver/0114_node_dynamic_to_creation_type.py 2017-02-18 00:02:02 +0000
133@@ -0,0 +1,26 @@
134+# -*- coding: utf-8 -*-
135+from __future__ import unicode_literals
136+
137+from django.db import (
138+ migrations,
139+ models,
140+)
141+
142+
143+class Migration(migrations.Migration):
144+
145+ dependencies = [
146+ ('maasserver', '0113_set_filepath_limit_to_linux_max'),
147+ ]
148+
149+ operations = [
150+ migrations.RemoveField(
151+ model_name='node',
152+ name='dynamic',
153+ ),
154+ migrations.AddField(
155+ model_name='node',
156+ name='creation_type',
157+ field=models.IntegerField(default=1),
158+ ),
159+ ]
160
161=== modified file 'src/maasserver/models/bmc.py'
162--- src/maasserver/models/bmc.py 2017-02-17 14:23:04 +0000
163+++ src/maasserver/models/bmc.py 2017-02-18 00:02:02 +0000
164@@ -7,6 +7,7 @@
165 "BMC",
166 ]
167
168+from functools import partial
169 import re
170
171 from django.contrib.postgres.fields import ArrayField
172@@ -25,13 +26,16 @@
173 )
174 from django.db.models.query import QuerySet
175 from maasserver import DefaultMeta
176+from maasserver.clusterrpc.pods import decompose_machine
177 from maasserver.enum import (
178 BMC_TYPE,
179 BMC_TYPE_CHOICES,
180 INTERFACE_TYPE,
181 IPADDRESS_TYPE,
182+ NODE_CREATION_TYPE,
183 NODE_STATUS,
184 )
185+from maasserver.exceptions import PodProblem
186 from maasserver.fields import JSONObjectField
187 from maasserver.models.blockdevice import BlockDevice
188 from maasserver.models.cleansave import CleanSave
189@@ -44,11 +48,18 @@
190 from maasserver.models.subnet import Subnet
191 from maasserver.models.tag import Tag
192 from maasserver.models.timestampedmodel import TimestampedModel
193-from maasserver.rpc import getAllClients
194+from maasserver.rpc import (
195+ getAllClients,
196+ getClientFromIdentifiers,
197+)
198+from maasserver.utils.orm import transactional
199+from maasserver.utils.threads import deferToDatabase
200 import petname
201 from provisioningserver.drivers import SETTING_SCOPE
202 from provisioningserver.drivers.power.registry import PowerDriverRegistry
203 from provisioningserver.logger import get_maas_logger
204+from provisioningserver.utils.twisted import asynchronous
205+from twisted.internet.defer import inlineCallbacks
206
207
208 maaslog = get_maas_logger("node")
209@@ -490,7 +501,8 @@
210
211 def create_machine(
212 self, discovered_machine, commissioning_user,
213- skip_commissioning=False):
214+ skip_commissioning=False,
215+ creation_type=NODE_CREATION_TYPE.PRE_EXISTING):
216 """Create's a `Machine` from `discovered_machines` for this pod."""
217 # Create the machine.
218 machine = Machine(
219@@ -499,7 +511,8 @@
220 cpu_count=discovered_machine.cores,
221 cpu_speed=discovered_machine.cpu_speed,
222 memory=discovered_machine.memory,
223- power_state=discovered_machine.power_state)
224+ power_state=discovered_machine.power_state,
225+ creation_type=creation_type)
226 machine.bmc = self
227 machine.instance_power_parameters = discovered_machine.power_parameters
228 machine.set_random_hostname()
229@@ -796,6 +809,105 @@
230 if isinstance(blockdevice.actual_instance, PhysicalBlockDevice)
231 ])
232
233+ def delete(self, *args, **kwargs):
234+ raise AttributeError(
235+ "Use `async_delete` instead. Deleting a Pod takes "
236+ "an asynchronous action.")
237+
238+ @asynchronous
239+ def async_delete(self):
240+ """Delete a pod asynchronously.
241+
242+ Any machine in the pod that needs to be decomposed will be decomposed
243+ before it is removed from the database. If a failure occurs during the
244+ decomposition process then only the machines that where successfully
245+ decomposed will be deleted in the database and the pod will not
246+ be deleted. If all machines are successfully decomposed then the
247+ machines that should not be decomposed and the pod will finally be
248+ removed from the database.
249+ """
250+
251+ @transactional
252+ def gather_clients_and_machines(pod):
253+ decompose, pre_existing = [], []
254+ for machine in Machine.objects.filter(
255+ bmc__id=pod.id).order_by('id').select_related('bmc'):
256+ if machine.creation_type == NODE_CREATION_TYPE.PRE_EXISTING:
257+ pre_existing.append(machine.id)
258+ else:
259+ decompose.append((
260+ machine.id,
261+ machine.power_parameters))
262+ return (
263+ pod.id, pod.name, pod.power_type, pod.get_client_identifiers(),
264+ decompose, pre_existing)
265+
266+ @inlineCallbacks
267+ def decompose(result):
268+ (pod_id, pod_name, pod_type, client_idents,
269+ decompose, pre_existing) = result
270+ decomposed, updated_hints, error = [], None, None
271+ for machine_id, parameters in decompose:
272+ # Get a new client for every decompose because we might lose
273+ # a connection to a rack during this operation.
274+ client = yield getClientFromIdentifiers(client_idents)
275+ try:
276+ updated_hints = yield decompose_machine(
277+ client, pod_type, parameters,
278+ pod_id=pod_id, name=pod_name)
279+ except PodProblem as exc:
280+ error = exc
281+ break
282+ else:
283+ decomposed.append(machine_id)
284+ return pod_id, decomposed, pre_existing, error, updated_hints
285+
286+ @transactional
287+ def perform_deletion(result):
288+ (pod_id, decomposed_ids, pre_existing_ids,
289+ error, updated_hints) = result
290+ # No matter the error for decompose, we ensure that the machines
291+ # that where decomposed before the error are actually deleted.
292+ pod = Pod.objects.get(id=pod_id)
293+ machines = Machine.objects.filter(id__in=decomposed_ids)
294+ for machine in machines:
295+ # Clear BMC (aka. this pod) so the signal handler does not
296+ # try to decompose of it. Its already been decomposed.
297+ machine.bmc = None
298+ machine.delete()
299+
300+ if error is not None:
301+ # Error occurred so we update the pod hints as the pod and
302+ # pre-existing machines will not be deleted. The error is
303+ # returned not raised so that the transaction is not rolled
304+ # back.
305+ if updated_hints is not None:
306+ pod.sync_hints(updated_hints)
307+ return error
308+ else:
309+ # Error did not occur. Delete the pre-existing machines
310+ # and finally the pod.
311+ for machine in Machine.objects.filter(id__in=pre_existing_ids):
312+ # We loop and call delete to ensure the `delete` method
313+ # on the machine object is actually called.
314+ machine.delete()
315+ # Call delete by bypassing the override that prevents its call.
316+ super(BMC, pod).delete()
317+
318+ def raise_error(error):
319+ # Error gets returned from the previous callback if it should
320+ # be raised. This way allows the transaction that was running
321+ # in the other callback to be committed.
322+ if error is not None:
323+ raise error
324+
325+ # Don't catch any errors here they are raised to the caller.
326+ d = deferToDatabase(gather_clients_and_machines, self)
327+ d.addCallback(decompose)
328+ d.addCallback(partial(deferToDatabase, perform_deletion))
329+ d.addCallback(raise_error)
330+ return d
331+
332
333 class BMCRoutableRackControllerRelationship(CleanSave, TimestampedModel):
334 """Records the link routable status of a BMC from a RackController.
335
336=== modified file 'src/maasserver/models/node.py'
337--- src/maasserver/models/node.py 2017-02-17 14:23:04 +0000
338+++ src/maasserver/models/node.py 2017-02-18 00:02:02 +0000
339@@ -72,6 +72,7 @@
340 INTERFACE_TYPE,
341 IPADDRESS_FAMILY,
342 IPADDRESS_TYPE,
343+ NODE_CREATION_TYPE,
344 NODE_PERMISSION,
345 NODE_STATUS,
346 NODE_STATUS_CHOICES,
347@@ -938,10 +939,10 @@
348
349 license_key = CharField(max_length=30, null=True, blank=True)
350
351- # Only used by Machine. Set to True when the machine was composed
352- # dynamically from a pod during allocation. When the machine is
353- # released it will be deleted.
354- dynamic = BooleanField(default=False)
355+ # Only used by Machine. Set to the creation type based on how the machine
356+ # ended up in the Pod.
357+ creation_type = IntegerField(
358+ null=False, blank=False, default=NODE_CREATION_TYPE.PRE_EXISTING)
359
360 tags = ManyToManyField(Tag)
361
362@@ -2704,7 +2705,7 @@
363 final power-down. This method should be the absolute last method
364 called.
365 """
366- if self.dynamic:
367+ if self.creation_type == NODE_CREATION_TYPE.DYNAMIC:
368 self.delete()
369 else:
370 self.release_interface_config()
371
372=== modified file 'src/maasserver/models/signals/nodes.py'
373--- src/maasserver/models/signals/nodes.py 2017-02-17 14:23:04 +0000
374+++ src/maasserver/models/signals/nodes.py 2017-02-18 00:02:02 +0000
375@@ -17,6 +17,7 @@
376 from maasserver.clusterrpc.pods import decompose_machine
377 from maasserver.enum import (
378 BMC_TYPE,
379+ NODE_CREATION_TYPE,
380 NODE_STATUS,
381 NODE_TYPE,
382 )
383@@ -126,7 +127,8 @@
384 if (instance.node_type == NODE_TYPE.MACHINE and
385 bmc is not None and
386 bmc.bmc_type == BMC_TYPE.POD and
387- Capabilities.COMPOSABLE in bmc.capabilities):
388+ Capabilities.COMPOSABLE in bmc.capabilities and
389+ instance.creation_type != NODE_CREATION_TYPE.PRE_EXISTING):
390 pod = bmc.as_pod()
391
392 @asynchronous
393
394=== modified file 'src/maasserver/models/signals/tests/test_nodes.py'
395--- src/maasserver/models/signals/tests/test_nodes.py 2017-02-17 14:23:04 +0000
396+++ src/maasserver/models/signals/tests/test_nodes.py 2017-02-18 00:02:02 +0000
397@@ -10,6 +10,7 @@
398
399 import crochet
400 from maasserver.enum import (
401+ NODE_CREATION_TYPE,
402 NODE_STATUS,
403 NODE_TYPE,
404 NODE_TYPE_CHOICES,
405@@ -214,6 +215,14 @@
406 machine.delete()
407 self.assertThat(client, MockNotCalled())
408
409+ def test_does_nothing_if_pre_existing_machine(self):
410+ client = self.fake_rpc_client()
411+ machine = factory.make_Node()
412+ machine.bmc = self.make_composable_pod()
413+ machine.save()
414+ machine.delete()
415+ self.assertThat(client, MockNotCalled())
416+
417 def test_performs_decompose_machine(self):
418 hints = DiscoveredPodHints(
419 cores=random.randint(1, 8),
420@@ -224,7 +233,8 @@
421 client.return_value = succeed({
422 'hints': hints,
423 })
424- machine = factory.make_Node()
425+ machine = factory.make_Node(
426+ creation_type=NODE_CREATION_TYPE.MANUAL)
427 machine.bmc = pod
428 machine.instance_power_parameters = {
429 'power_id': factory.make_name('power_id'),
430@@ -246,7 +256,8 @@
431 pod = self.make_composable_pod()
432 client = self.fake_rpc_client()
433 client.side_effect = crochet.TimeoutError()
434- machine = factory.make_Node()
435+ machine = factory.make_Node(
436+ creation_type=NODE_CREATION_TYPE.MANUAL)
437 machine.bmc = pod
438 machine.instance_power_parameters = {
439 'power_id': factory.make_name('power_id'),
440@@ -261,7 +272,8 @@
441 pod = self.make_composable_pod()
442 client = self.fake_rpc_client()
443 client.return_value = fail(PodActionFail())
444- machine = factory.make_Node()
445+ machine = factory.make_Node(
446+ creation_type=NODE_CREATION_TYPE.MANUAL)
447 machine.bmc = pod
448 machine.instance_power_parameters = {
449 'power_id': factory.make_name('power_id'),
450
451=== modified file 'src/maasserver/models/tests/test_bmc.py'
452--- src/maasserver/models/tests/test_bmc.py 2017-02-17 14:23:04 +0000
453+++ src/maasserver/models/tests/test_bmc.py 2017-02-18 00:02:02 +0000
454@@ -6,18 +6,25 @@
455 __all__ = []
456
457 import random
458-from unittest.mock import Mock
459+from unittest.mock import (
460+ Mock,
461+ sentinel,
462+)
463
464+from crochet import wait_for
465 from maasserver.enum import (
466 INTERFACE_TYPE,
467 IPADDRESS_TYPE,
468+ NODE_CREATION_TYPE,
469 POWER_STATE,
470 )
471+from maasserver.exceptions import PodProblem
472 from maasserver.models import bmc as bmc_module
473 from maasserver.models.blockdevice import BlockDevice
474 from maasserver.models.bmc import (
475 BMC,
476 BMCRoutableRackControllerRelationship,
477+ Pod,
478 )
479 from maasserver.models.fabric import Fabric
480 from maasserver.models.node import Machine
481@@ -25,8 +32,12 @@
482 from maasserver.models.staticipaddress import StaticIPAddress
483 from maasserver.testing.factory import factory
484 from maasserver.testing.matchers import MatchesSetwiseWithAll
485-from maasserver.testing.testcase import MAASServerTestCase
486+from maasserver.testing.testcase import (
487+ MAASServerTestCase,
488+ MAASTransactionServerTestCase,
489+)
490 from maasserver.utils.orm import reload_object
491+from maasserver.utils.threads import deferToDatabase
492 from maastesting.matchers import MockCalledOnceWith
493 from provisioningserver.drivers.pod import (
494 DiscoveredMachine,
495@@ -35,12 +46,21 @@
496 DiscoveredPod,
497 DiscoveredPodHints,
498 )
499+from provisioningserver.rpc.cluster import DecomposeMachine
500 from testtools.matchers import (
501 Equals,
502 HasLength,
503 MatchesSetwise,
504 MatchesStructure,
505 )
506+from twisted.internet.defer import (
507+ fail,
508+ inlineCallbacks,
509+ succeed,
510+)
511+
512+
513+wait_for_reactor = wait_for(30) # 30 seconds.
514
515
516 class TestBMC(MAASServerTestCase):
517@@ -665,6 +685,7 @@
518 memory=Equals(machine.memory),
519 power_state=Equals(machine.power_state),
520 instance_power_parameters=Equals(machine.power_parameters),
521+ creation_type=Equals(NODE_CREATION_TYPE.PRE_EXISTING),
522 tags=MatchesSetwiseWithAll(*[
523 MatchesStructure(name=Equals(tag))
524 for tag in machine.tags
525@@ -919,3 +940,89 @@
526 Equals(tag)
527 for tag in dnew_interface.tags
528 ])))
529+
530+
531+class TestPodDelete(MAASTransactionServerTestCase):
532+
533+ def test_delete_is_not_allowed(self):
534+ pod = factory.make_Pod()
535+ self.assertRaises(AttributeError, pod.delete)
536+
537+ @wait_for_reactor
538+ @inlineCallbacks
539+ def test_delete_async_simply_deletes_empty_pod(self):
540+ pod = yield deferToDatabase(factory.make_Pod)
541+ yield pod.async_delete()
542+ pod = yield deferToDatabase(reload_object, pod)
543+ self.assertIsNone(pod)
544+
545+ @wait_for_reactor
546+ @inlineCallbacks
547+ def test_decomposes_and_deletes_machines_and_pod(self):
548+ pod = yield deferToDatabase(factory.make_Pod)
549+ decomposable_machine = yield deferToDatabase(
550+ factory.make_Machine, bmc=pod,
551+ creation_type=NODE_CREATION_TYPE.MANUAL)
552+ delete_machine = yield deferToDatabase(
553+ factory.make_Machine, bmc=pod)
554+ client = Mock()
555+ client.return_value = succeed({'hints': None})
556+ self.patch(
557+ bmc_module, "getClientFromIdentifiers").return_value = client
558+ yield pod.async_delete()
559+ self.assertThat(
560+ client, MockCalledOnceWith(
561+ DecomposeMachine, type=pod.power_type, context={},
562+ pod_id=pod.id, name=pod.name))
563+ decomposable_machine = yield deferToDatabase(
564+ reload_object, decomposable_machine)
565+ delete_machine = yield deferToDatabase(reload_object, delete_machine)
566+ pod = yield deferToDatabase(reload_object, pod)
567+ self.assertIsNone(decomposable_machine)
568+ self.assertIsNone(delete_machine)
569+ self.assertIsNone(pod)
570+
571+ @wait_for_reactor
572+ @inlineCallbacks
573+ def test_decomposes_handles_failure_after_one_successful(self):
574+ pod = yield deferToDatabase(factory.make_Pod)
575+ decomposable_machine_one = yield deferToDatabase(
576+ factory.make_Machine, bmc=pod,
577+ creation_type=NODE_CREATION_TYPE.MANUAL)
578+ decomposable_machine_two = yield deferToDatabase(
579+ factory.make_Machine, bmc=pod,
580+ creation_type=NODE_CREATION_TYPE.MANUAL)
581+ delete_machine = yield deferToDatabase(
582+ factory.make_Machine, bmc=pod)
583+ client = Mock()
584+ client.side_effect = [
585+ succeed({'hints': sentinel.hints}),
586+ fail(PodProblem()),
587+ ]
588+ mock_sync_hints = self.patch(Pod, 'sync_hints')
589+ self.patch(
590+ bmc_module, "getClientFromIdentifiers").return_value = client
591+
592+ # Ensure that the exeception is raised.
593+ try:
594+ yield pod.async_delete()
595+ except PodProblem as exc:
596+ pass
597+ else:
598+ self.fail("PodProblem exception was not raised.")
599+
600+ # Ensure that the pod hints where updated.
601+ self.assertThat(mock_sync_hints, MockCalledOnceWith(sentinel.hints))
602+
603+ # Only the first machine should have been deleted, the others should
604+ # all remain.
605+ decomposable_machine_one = yield deferToDatabase(
606+ reload_object, decomposable_machine_one)
607+ decomposable_machine_two = yield deferToDatabase(
608+ reload_object, decomposable_machine_two)
609+ delete_machine = yield deferToDatabase(reload_object, delete_machine)
610+ pod = yield deferToDatabase(reload_object, pod)
611+ self.assertIsNone(decomposable_machine_one)
612+ self.assertIsNotNone(decomposable_machine_two)
613+ self.assertIsNotNone(delete_machine)
614+ self.assertIsNotNone(pod)
615
616=== modified file 'src/maasserver/models/tests/test_node.py'
617--- src/maasserver/models/tests/test_node.py 2017-02-17 14:23:04 +0000
618+++ src/maasserver/models/tests/test_node.py 2017-02-18 00:02:02 +0000
619@@ -43,6 +43,7 @@
620 INTERFACE_LINK_TYPE,
621 INTERFACE_TYPE,
622 IPADDRESS_TYPE,
623+ NODE_CREATION_TYPE,
624 NODE_PERMISSION,
625 NODE_STATUS,
626 NODE_STATUS_CHOICES,
627@@ -1995,7 +1996,8 @@
628 owner = factory.make_User()
629 node = factory.make_Node(
630 status=NODE_STATUS.ALLOCATED, owner=owner, agent_name=agent_name,
631- dynamic=True, power_state=POWER_STATE.OFF)
632+ creation_type=NODE_CREATION_TYPE.DYNAMIC,
633+ power_state=POWER_STATE.OFF)
634 with post_commit_hooks:
635 node.release()
636 self.assertIsNone(reload_object(node))
637
638=== modified file 'src/maasserver/testing/factory.py'
639--- src/maasserver/testing/factory.py 2017-02-17 14:23:04 +0000
640+++ src/maasserver/testing/factory.py 2017-02-18 00:02:02 +0000
641@@ -372,7 +372,7 @@
642 created=None, zone=None, networks=None, sortable_name=False,
643 power_type=None, power_parameters=None, power_state=None,
644 power_state_updated=undefined, with_boot_disk=True, vlan=None,
645- fabric=None, bmc_connected_to=None, owner_data={}, dynamic=False,
646+ fabric=None, bmc_connected_to=None, owner_data={},
647 with_empty_script_sets=False, **kwargs):
648 """Make a :class:`Node`.
649
650@@ -408,7 +408,7 @@
651 min_hwe_kernel=min_hwe_kernel, hwe_kernel=hwe_kernel,
652 node_type=node_type, zone=zone,
653 power_state=power_state, power_state_updated=power_state_updated,
654- domain=domain, dynamic=dynamic, **kwargs)
655+ domain=domain, **kwargs)
656 node.power_type = power_type
657 node.power_parameters = power_parameters
658 self._save_node_unchecked(node)
659
660=== modified file 'src/maasserver/websockets/handlers/device.py'
661--- src/maasserver/websockets/handlers/device.py 2017-02-17 14:23:04 +0000
662+++ src/maasserver/websockets/handlers/device.py 2017-02-18 00:02:02 +0000
663@@ -79,7 +79,7 @@
664 'create_interface',
665 'action']
666 exclude = [
667- "dynamic",
668+ "creation_type",
669 "type",
670 "boot_interface",
671 "boot_cluster_ip",
672
673=== modified file 'src/maasserver/websockets/handlers/machine.py'
674--- src/maasserver/websockets/handlers/machine.py 2017-02-17 14:23:04 +0000
675+++ src/maasserver/websockets/handlers/machine.py 2017-02-18 00:02:02 +0000
676@@ -128,7 +128,7 @@
677 ]
678 form = AdminMachineWithMACAddressesForm
679 exclude = [
680- "dynamic",
681+ "creation_type",
682 "status_expires",
683 "previous_status",
684 "parent",