Merge lp:~mpontillo/maas/networking-constraints-refactor-part2 into lp:~maas-committers/maas/trunk

Proposed by Mike Pontillo
Status: Merged
Approved by: Mike Pontillo
Approved revision: no longer in the source branch.
Merged at revision: 4454
Proposed branch: lp:~mpontillo/maas/networking-constraints-refactor-part2
Merge into: lp:~maas-committers/maas/trunk
Prerequisite: lp:~mpontillo/maas/network-constraint-filters-minor-restructuring
Diff against target: 1015 lines (+609/-105)
9 files modified
.idea/inspectionProfiles/Project_Default.xml (+2/-0)
src/maasserver/models/interface.py (+82/-12)
src/maasserver/models/subnet.py (+46/-70)
src/maasserver/models/tests/test_interface.py (+208/-11)
src/maasserver/models/tests/test_subnet.py (+56/-1)
src/maasserver/node_constraint_filter_forms.py (+43/-0)
src/maasserver/testing/factory.py (+3/-3)
src/maasserver/tests/test_node_constraint_filter_forms.py (+55/-1)
src/maasserver/utils/orm.py (+114/-7)
To merge this branch: bzr merge lp:~mpontillo/maas/networking-constraints-refactor-part2
Reviewer Review Type Date Requested Status
Blake Rouse (community) Approve
Review via email: mp+276329@code.launchpad.net

Commit message

Network interfaces constraint refactoring, part 2.

 * Enable basic "interfaces" constraints for Node.acquire() API.
 * Does not yet support matching each constraint back to its interface.
 * Extend framework for specifier-based queries into the Interface manager.
 * Extend the generic query-by-specifiers functionality to support specifiers other than functions. (specifier aliases, Django query strings, and references to a different model object's query-by-specifiers are now supported.)
 * Make the superclass more robust to handle collaboration between filtering entities.
 * Support for (the positive form of) the following specifiers for interfaces: subnet, space, subnet_cidr, fabric, fabric_class, id, vid, and name.

Description of the change

Note: This branch does not yet generate the data to match the constraint labels back to a node's interfaces.

Parts 3 and 4 will be for negative constraints and interface match reporting.

To post a comment you must log in.
Revision history for this message
Blake Rouse (blake-rouse) wrote :

This is really good code. Well done! Just a couple of comments, but nothing major. Very clean and very well commented.

review: Approve
Revision history for this message
Mike Pontillo (mpontillo) wrote :

Thanks for reviewing.

I fixed what you pointed out, and realized I needed to add a test case that checks if multiple labels are parsed okay in the form.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file '.idea/inspectionProfiles/Project_Default.xml'
2--- .idea/inspectionProfiles/Project_Default.xml 2015-10-30 10:48:25 +0000
3+++ .idea/inspectionProfiles/Project_Default.xml 2015-11-02 23:13:19 +0000
4@@ -9,6 +9,8 @@
5 <option value="maasserver.models.node.NodeQueriesMixin.exclude" />
6 <option value="maasserver.models.subnet.SubnetQueriesMixin.filter" />
7 <option value="maasserver.models.subnet.SubnetQueriesMixin.exclude" />
8+ <option value="maasserver.utils.orm.MAASQueriesMixin.filter" />
9+ <option value="maasserver.utils.orm.MAASQueriesMixin.exclude" />
10 </list>
11 </option>
12 </inspection_tool>
13
14=== modified file 'src/maasserver/models/interface.py'
15--- src/maasserver/models/interface.py 2015-10-30 20:50:01 +0000
16+++ src/maasserver/models/interface.py 2015-11-02 23:13:19 +0000
17@@ -37,6 +37,7 @@
18 PROTECT,
19 Q,
20 )
21+from django.db.models.query import QuerySet
22 from django.http import Http404
23 from django.shortcuts import get_object_or_404
24 from djorm_pgarray.fields import ArrayField
25@@ -63,12 +64,15 @@
26 )
27 from maasserver.models.cleansave import CleanSave
28 from maasserver.models.nodegroupinterface import NodeGroupInterface
29-from maasserver.models.space import Space
30 from maasserver.models.staticipaddress import StaticIPAddress
31 from maasserver.models.timestampedmodel import TimestampedModel
32-from maasserver.utils.orm import get_one
33+from maasserver.utils.orm import (
34+ get_one,
35+ MAASQueriesMixin,
36+)
37 from maasserver.utils.signals import connect_to_field_change
38 from netaddr import (
39+ AddrFormatError,
40 IPAddress,
41 IPNetwork,
42 )
43@@ -76,6 +80,7 @@
44 from provisioningserver.utils.ipaddr import (
45 get_first_and_last_usable_host_in_network,
46 )
47+from provisioningserver.utils.network import parse_integer
48
49
50 maaslog = get_maas_logger("interface")
51@@ -113,7 +118,76 @@
52 return None
53
54
55-class InterfaceManager(Manager):
56+class InterfaceQueriesMixin(MAASQueriesMixin):
57+ def get_specifiers_q(self, specifiers, separator=':'):
58+ """Returns a Q object for objects matching the given specifiers.
59+
60+ :return:django.db.models.Q
61+ """
62+ # Circular imports.
63+ from maasserver.models import Subnet
64+
65+ # This dict is used by the constraints code in order to identify
66+ # interfaces with particular properties. Please note that changing
67+ # the keys here can impact backward compatibility, so use caution.
68+ interface_specifier_types = {
69+ None: self._add_default_query,
70+ 'id': self._add_interface_id_query,
71+ # XXX mpontillo 2015-10-31 the fabric matcher should probably
72+ # become a tuple, after we have specifier-based fabric queries.
73+ 'fabric': 'vlan__fabric__name',
74+ 'fabric_class': 'vlan__fabric__class_type',
75+ 'ip': 'ip_addresses__ip',
76+ 'mode': self._add_mode_query,
77+ 'name': '__name',
78+ 'subnet': (Subnet.objects, 'staticipaddress__interface'),
79+ 'space': 'subnet{s}space'.format(s=separator),
80+ 'subnet_cidr': 'subnet{s}cidr'.format(s=separator),
81+ 'type': '__type',
82+ 'vid': self._add_vlan_vid_query,
83+ }
84+ return super(InterfaceQueriesMixin, self).get_specifiers_q(
85+ specifiers, specifier_types=interface_specifier_types,
86+ separator=separator)
87+
88+ def _add_interface_id_query(self, current_q, op, item):
89+ try:
90+ item = parse_integer(item)
91+ except ValueError:
92+ raise ValidationError("Interface ID must be numeric.")
93+ else:
94+ current_q = op(current_q, Q(id=item))
95+ return current_q
96+
97+ def _add_default_query(self, current_q, op, item):
98+ # By default, search for a specific CIDR first, then
99+ # fall back to the name.
100+ try:
101+ ip = IPNetwork(item)
102+ except AddrFormatError:
103+ current_q = op(current_q, Q(name=item))
104+ else:
105+ cidr = unicode(ip.cidr)
106+ current_q = op(current_q, Q(ip_addresses__subnet__cidr=cidr))
107+ return current_q
108+
109+ def _add_mode_query(self, current_q, op, item):
110+ if item.strip().lower() != 'unconfigured':
111+ raise ValidationError(
112+ "The only valid value for 'mode' is 'unconfigured'.")
113+ return op(
114+ current_q,
115+ Q(ip_addresses__ip__isnull=True) | Q(ip_addresses__ip=''))
116+
117+
118+class InterfaceQuerySet(InterfaceQueriesMixin, QuerySet):
119+ """Custom QuerySet which mixes in some additional queries specific to
120+ subnets. This needs to be a mixin because an identical method is needed on
121+ both the Manager and all QuerySets which result from calling the manager.
122+ """
123+
124+
125+class InterfaceManager(Manager, InterfaceQueriesMixin):
126 """A Django manager managing one type of interface."""
127
128 def get_queryset(self):
129@@ -123,7 +197,7 @@
130 or `VLANInterface` it will filter only allow returning those
131 interfaces.
132 """
133- qs = super(InterfaceManager, self).get_query_set()
134+ qs = InterfaceQuerySet(self.model, using=self._db)
135 interface_type = self.model.get_type()
136 if interface_type is None:
137 # Not a specific interface type so we don't filter by the type.
138@@ -321,15 +395,13 @@
139 interface is connected.
140 """
141 # Circular imports.
142- from maasserver.models import StaticIPAddress, Subnet, Fabric
143+ from maasserver.models import StaticIPAddress, Subnet
144
145 # Delete all current DISCOVERED IP address on this interface. As new
146 # ones are about to be added.
147 StaticIPAddress.objects.filter(
148 interface=self, alloc_type=IPADDRESS_TYPE.DISCOVERED).delete()
149
150- # XXX:fabric - Using the default Fabric for now.
151- fabric = Fabric.objects.get_default_fabric()
152 for ip in cidr_list:
153 network = IPNetwork(ip)
154 cidr = unicode(network.cidr)
155@@ -338,12 +410,10 @@
156 # Find the Subnet for each IP address seen (will be used later
157 # to create or update the Subnet)
158 try:
159- subnet = Subnet.objects.get(cidr=cidr, vlan__fabric=fabric)
160+ subnet = Subnet.objects.get(cidr=cidr)
161 except Subnet.DoesNotExist:
162- vlan = fabric.get_default_vlan()
163- space = Space.objects.get_default_space()
164- subnet = Subnet.objects.create_from_cidr(
165- cidr=cidr, vlan=vlan, space=space)
166+ # XXX mpontillo 2015-11-01 configuration != state
167+ subnet = Subnet.objects.create_from_cidr(cidr)
168 maaslog.info(
169 "Creating subnet %s connected to interface %s "
170 "of node %s.", cidr, self, self.get_node())
171
172=== modified file 'src/maasserver/models/subnet.py'
173--- src/maasserver/models/subnet.py 2015-10-31 06:47:39 +0000
174+++ src/maasserver/models/subnet.py 2015-11-02 23:13:19 +0000
175@@ -99,9 +99,6 @@
176
177 class SubnetQueriesMixin(MAASQueriesMixin):
178
179- def __init__(self, *args, **kwargs):
180- super(SubnetQueriesMixin, self).__init__(*args, **kwargs)
181-
182 find_subnets_with_ip_query = """
183 SELECT DISTINCT subnet.*, masklen(subnet.cidr) "prefixlen"
184 FROM
185@@ -175,24 +172,47 @@
186 except (ValueError, AddrFormatError) as e:
187 raise ValidationError(e.message)
188
189- def _get_specifiers_query(self, specifiers):
190+ def get_specifiers_q(self, specifiers, separator=':'):
191 """Returns a Q object for objects matching the given specifiers.
192
193- See documentation for `filter_by_specifiers()`.
194+ Allows a number of types to be prefixed in front of each specifier:
195+ * 'ip:' Matches the subnet that best matches the given IP address.
196+ * 'cidr:' Matches a subnet with the exact given CIDR.
197+ * 'name': Matches a subnet with the given name.
198+ * 'vid:' Matches a subnet whose VLAN has the given VID.
199+ Can be used with a hexadecimal or binary string by prefixing
200+ it with '0x' or '0b'.
201+ ' 'vlan:' Synonym for 'vid' for compatibility with older MAAS
202+ versions.
203+ * 'space:' Matches the name of this subnet's space.
204+
205+ If no specifier is given, the input will be treated as a CIDR. If
206+ the input is not a valid CIDR, it will be treated as subnet name.
207+
208+ :raise:AddrFormatError:If a specific IP address or CIDR is requested,
209+ but the address could not be parsed.
210
211 :return:django.db.models.Q
212 """
213+ # Circular imports.
214+ from maasserver.models import Interface
215+
216+ # This dict is used by the constraints code in order to identify
217+ # subnets with particular properties. Please note that changing
218+ # the keys here can impact backward compatibility, so use caution.
219 subnet_specifier_types = {
220+ None: self._add_default_query,
221+ 'cidr': self._add_unvalidated_cidr_query,
222+ 'id': self._add_subnet_id_query,
223+ 'interface': (Interface.objects, 'ip_addresses__subnet'),
224 'ip': self._add_ip_in_subnet_query,
225- 'cidr': self._add_unvalidated_cidr_query,
226- 'name': self._add_subnet_name_query,
227- 'vlan': self._add_vlan_vid_query,
228+ 'name': "__name",
229 'vid': self._add_vlan_vid_query,
230- 'space': self._add_space_name_query,
231- None: self._add_default_query,
232+ 'vlan': 'vid',
233+ 'space': "space__name",
234 }
235- return super(SubnetQueriesMixin, self)._get_specifiers_query(
236- specifiers, specifier_types=subnet_specifier_types)
237+ return super(SubnetQueriesMixin, self).get_specifiers_q(
238+ specifiers, specifier_types=subnet_specifier_types, separator=':')
239
240 def _add_default_query(self, current_q, op, item):
241 # By default, search for a specific CIDR first, then
242@@ -200,7 +220,7 @@
243 try:
244 ip = IPNetwork(item)
245 except AddrFormatError:
246- current_q = self._add_subnet_name_query(current_q, op, item)
247+ current_q = op(current_q, Q(name=item))
248 else:
249 cidr = unicode(ip.cidr)
250 current_q = op(current_q, Q(cidr=cidr))
251@@ -223,63 +243,14 @@
252 current_q = op(current_q, Q(id__in=ids))
253 return current_q
254
255- def _add_space_name_query(self, current_q, op, item):
256- current_q = op(current_q, Q(space__name=item))
257- return current_q
258-
259- def _add_subnet_name_query(self, current_q, op, item):
260- current_q = op(current_q, Q(name=item))
261- return current_q
262-
263- def _add_vlan_vid_query(self, current_q, op, item):
264- if item.lower() == 'untagged':
265- vid = 0
266+ def _add_subnet_id_query(self, current_q, op, item):
267+ try:
268+ item = parse_integer(item)
269+ except ValueError:
270+ raise ValidationError("Subnet ID must be numeric.")
271 else:
272- vid = parse_integer(item)
273- if vid < 0 or vid >= 0xfff:
274- raise ValidationError(
275- "VLAN tag (VID) out of range "
276- "(0-4094; 0 for untagged.)")
277- current_q = op(current_q, Q(vlan__vid=vid))
278- return current_q
279-
280- def filter_by_specifiers(self, specifiers):
281- """Filters subnets by the given list of specifiers (or single
282- specifier).
283-
284- Allows a number of types to be prefixed in front of each specifier:
285- * 'ip:' Matches the subnet that best matches the given IP address.
286- * 'cidr:' Matches a subnet with the exact given CIDR.
287- * 'name': Matches a subnet with the given name.
288- * 'vid:' Matches a subnet whose VLAN has the given VID.
289- Can be used with a hexadecimal or binary string by prefixing
290- it with '0x' or '0b'.
291- ' 'vlan:' Synonym for 'vid' for compatibility with older MAAS
292- versions.
293- * 'space:' Matches the name of this subnet's space.
294-
295- If no specifier is given, the input will be treated as a CIDR. If
296- the input is not a valid CIDR, it will be treated as subnet name.
297-
298- :raise:AddrFormatError:If a specific IP address or CIDR is requested,
299- but the address could not be parsed.
300- :return:QuerySet
301- """
302- query = self._get_specifiers_query(specifiers)
303- return self.filter(query)
304-
305- def exclude_by_specifiers(self, specifiers):
306- """Excludes subnets by the given list of specifiers (or single
307- specifier).
308-
309- See documentation for `filter_by_specifiers()`.
310-
311- :raise:AddrFormatError:If a specific IP address or CIDR is requested,
312- but the address could not be parsed.
313- :return:QuerySet
314- """
315- query = self._get_specifiers_query(specifiers)
316- return self.exclude(query)
317+ current_q = op(current_q, Q(id=item))
318+ return current_q
319
320
321 class SubnetQuerySet(QuerySet, SubnetQueriesMixin):
322@@ -296,9 +267,14 @@
323 queryset = SubnetQuerySet(self.model, using=self._db)
324 return queryset
325
326- def create_from_cidr(self, cidr, vlan, space):
327+ def create_from_cidr(self, cidr, vlan=None, space=None):
328 """Create a subnet from the given CIDR."""
329 name = "subnet-" + unicode(cidr)
330+ from maasserver.models import (Space, VLAN)
331+ if space is None:
332+ space = Space.objects.get_default_space()
333+ if vlan is None:
334+ vlan = VLAN.objects.get_default_vlan()
335 return self.create(name=name, cidr=cidr, vlan=vlan, space=space)
336
337 def _find_fabric(self, fabric):
338
339=== modified file 'src/maasserver/models/tests/test_interface.py'
340--- src/maasserver/models/tests/test_interface.py 2015-10-30 20:50:01 +0000
341+++ src/maasserver/models/tests/test_interface.py 2015-11-02 23:13:19 +0000
342@@ -135,6 +135,205 @@
343 user, NODE_PERMISSION.ADMIN)
344
345
346+class TestInterfaceQueriesMixin(MAASServerTestCase):
347+
348+ def test__filter_by_specifiers_default_matches_cidr_or_name(self):
349+ subnet1 = factory.make_Subnet()
350+ subnet2 = factory.make_Subnet()
351+ node1 = factory.make_Node_with_Interface_on_Subnet(
352+ subnet=subnet1)
353+ node2 = factory.make_Node_with_Interface_on_Subnet(
354+ subnet=subnet2)
355+ iface1 = node1.get_boot_interface()
356+ iface2 = node2.get_boot_interface()
357+ iface3 = factory.make_Interface(
358+ iftype=INTERFACE_TYPE.BOND, parents=[iface2], name='bond0')
359+ self.assertItemsEqual(
360+ Interface.objects.filter_by_specifiers(
361+ "%s" % subnet1.cidr), [iface1])
362+ self.assertItemsEqual(
363+ Interface.objects.filter_by_specifiers(
364+ "%s" % subnet2.cidr), [iface2])
365+ self.assertItemsEqual(
366+ Interface.objects.filter_by_specifiers(
367+ ["%s" % subnet1.cidr,
368+ "%s" % subnet2.cidr], ), [iface1, iface2])
369+ self.assertItemsEqual(
370+ Interface.objects.filter_by_specifiers(iface1.name), [iface1])
371+ self.assertItemsEqual(
372+ Interface.objects.filter_by_specifiers(iface3.name), [iface3])
373+
374+ def test__filter_by_specifiers_matches_fabric_class(self):
375+ fabric1 = factory.make_Fabric(class_type='10g')
376+ fabric2 = factory.make_Fabric(class_type='1g')
377+ vlan1 = factory.make_VLAN(vid=1, fabric=fabric1)
378+ vlan2 = factory.make_VLAN(vid=2, fabric=fabric2)
379+ iface1 = factory.make_Interface(vlan=vlan1)
380+ iface2 = factory.make_Interface(vlan=vlan2)
381+ self.assertItemsEqual(
382+ Interface.objects.filter_by_specifiers(
383+ "fabric_class:10g"), [iface1])
384+ self.assertItemsEqual(
385+ Interface.objects.filter_by_specifiers(
386+ "fabric_class:1g"), [iface2])
387+ self.assertItemsEqual(
388+ Interface.objects.filter_by_specifiers(
389+ ["fabric_class:1g", "fabric_class:10g"]), [iface1, iface2])
390+
391+ def test__filter_by_specifiers_matches_fabric(self):
392+ fabric1 = factory.make_Fabric(name="fabric1")
393+ fabric2 = factory.make_Fabric(name="fabric2")
394+ vlan1 = factory.make_VLAN(vid=1, fabric=fabric1)
395+ vlan2 = factory.make_VLAN(vid=2, fabric=fabric2)
396+ iface1 = factory.make_Interface(vlan=vlan1)
397+ iface2 = factory.make_Interface(vlan=vlan2)
398+ self.assertItemsEqual(
399+ Interface.objects.filter_by_specifiers(
400+ "fabric:fabric1"), [iface1])
401+ self.assertItemsEqual(
402+ Interface.objects.filter_by_specifiers(
403+ "fabric:fabric2"), [iface2])
404+ self.assertItemsEqual(
405+ Interface.objects.filter_by_specifiers(
406+ ["fabric:fabric1", "fabric:fabric2"]), [iface1, iface2])
407+
408+ def test__filter_by_specifiers_matches_interface_id(self):
409+ iface1 = factory.make_Interface()
410+ iface2 = factory.make_Interface()
411+ self.assertItemsEqual(
412+ Interface.objects.filter_by_specifiers(
413+ "id:%s" % iface1.id), [iface1])
414+ self.assertItemsEqual(
415+ Interface.objects.filter_by_specifiers(
416+ "id:%s" % iface2.id), [iface2])
417+ self.assertItemsEqual(
418+ Interface.objects.filter_by_specifiers(
419+ ["id:%s" % iface1.id, "id:%s" % iface2.id]), [iface1, iface2])
420+
421+ def test__filter_by_specifiers_matches_vid(self):
422+ iface1 = factory.make_Interface()
423+ iface2 = factory.make_Interface()
424+ self.assertItemsEqual(
425+ Interface.objects.filter_by_specifiers(
426+ "vid:%s" % iface1.vlan.vid), [iface1])
427+ self.assertItemsEqual(
428+ Interface.objects.filter_by_specifiers(
429+ "vid:%s" % iface2.vlan.vid), [iface2])
430+ self.assertItemsEqual(
431+ Interface.objects.filter_by_specifiers(
432+ ["vid:%s" % iface1.vlan.vid, "vid:%s" % iface2.vlan.vid]),
433+ [iface1, iface2])
434+
435+ def test__filter_by_specifiers_matches_subnet_specifier(self):
436+ subnet1 = factory.make_Subnet()
437+ subnet2 = factory.make_Subnet()
438+ node1 = factory.make_Node_with_Interface_on_Subnet(subnet=subnet1)
439+ node2 = factory.make_Node_with_Interface_on_Subnet(subnet=subnet2)
440+ iface1 = node1.get_boot_interface()
441+ iface2 = node2.get_boot_interface()
442+ self.assertItemsEqual(
443+ Interface.objects.filter_by_specifiers(
444+ "subnet:cidr:%s" % subnet1.cidr), [iface1])
445+ self.assertItemsEqual(
446+ Interface.objects.filter_by_specifiers(
447+ "subnet:cidr:%s" % subnet2.cidr), [iface2])
448+ self.assertItemsEqual(
449+ Interface.objects.filter_by_specifiers(
450+ ["subnet:cidr:%s" % subnet1.cidr,
451+ "subnet:cidr:%s" % subnet2.cidr], ), [iface1, iface2])
452+
453+ def test__filter_by_specifiers_matches_subnet_cidr_alias(self):
454+ subnet1 = factory.make_Subnet()
455+ subnet2 = factory.make_Subnet()
456+ node1 = factory.make_Node_with_Interface_on_Subnet(subnet=subnet1)
457+ node2 = factory.make_Node_with_Interface_on_Subnet(subnet=subnet2)
458+ iface1 = node1.get_boot_interface()
459+ iface2 = node2.get_boot_interface()
460+ self.assertItemsEqual(
461+ Interface.objects.filter_by_specifiers(
462+ "subnet_cidr:%s" % subnet1.cidr), [iface1])
463+ self.assertItemsEqual(
464+ Interface.objects.filter_by_specifiers(
465+ "subnet_cidr:%s" % subnet2.cidr), [iface2])
466+ self.assertItemsEqual(
467+ Interface.objects.filter_by_specifiers(
468+ ["subnet_cidr:%s" % subnet1.cidr,
469+ "subnet_cidr:%s" % subnet2.cidr], ), [iface1, iface2])
470+
471+ def test__filter_by_specifiers_matches_space(self):
472+ space1 = factory.make_Space()
473+ space2 = factory.make_Space()
474+ subnet1 = factory.make_Subnet(space=space1)
475+ subnet2 = factory.make_Subnet(space=space2)
476+ node1 = factory.make_Node_with_Interface_on_Subnet(subnet=subnet1)
477+ node2 = factory.make_Node_with_Interface_on_Subnet(subnet=subnet2)
478+ iface1 = node1.get_boot_interface()
479+ iface2 = node2.get_boot_interface()
480+ self.assertItemsEqual(
481+ Interface.objects.filter_by_specifiers(
482+ "space:%s" % space1.name), [iface1])
483+ self.assertItemsEqual(
484+ Interface.objects.filter_by_specifiers(
485+ "space:%s" % space2.name), [iface2])
486+ self.assertItemsEqual(
487+ Interface.objects.filter_by_specifiers(
488+ ["space:%s" % space1.name,
489+ "space:%s" % space2.name], ), [iface1, iface2])
490+
491+ def test__filter_by_specifiers_matches_type(self):
492+ physical = factory.make_Interface()
493+ bond = factory.make_Interface(
494+ iftype=INTERFACE_TYPE.BOND, parents=[physical])
495+ vlan = factory.make_Interface(
496+ iftype=INTERFACE_TYPE.VLAN, parents=[physical])
497+ unknown = factory.make_Interface(iftype=INTERFACE_TYPE.UNKNOWN)
498+ self.assertItemsEqual(Interface.objects.filter_by_specifiers(
499+ "type:physical"), [physical])
500+ self.assertItemsEqual(Interface.objects.filter_by_specifiers(
501+ "type:vlan"), [vlan])
502+ self.assertItemsEqual(Interface.objects.filter_by_specifiers(
503+ "type:bond"), [bond])
504+ self.assertItemsEqual(Interface.objects.filter_by_specifiers(
505+ "type:unknown"), [unknown])
506+
507+ def test__filter_by_specifiers_matches_ip(self):
508+ subnet1 = factory.make_Subnet(cidr='10.0.0.0/24')
509+ subnet2 = factory.make_Subnet(cidr='10.0.1.0/24')
510+ iface1 = factory.make_Interface()
511+ iface2 = factory.make_Interface()
512+ factory.make_StaticIPAddress(
513+ ip='10.0.0.1', alloc_type=IPADDRESS_TYPE.AUTO, subnet=subnet1,
514+ interface=iface1)
515+ factory.make_StaticIPAddress(
516+ ip='10.0.1.1', alloc_type=IPADDRESS_TYPE.AUTO, subnet=subnet2,
517+ interface=iface2)
518+ self.assertItemsEqual(Interface.objects.filter_by_specifiers(
519+ "ip:10.0.0.1"), [iface1])
520+ self.assertItemsEqual(Interface.objects.filter_by_specifiers(
521+ "ip:10.0.1.1"), [iface2])
522+
523+ def test__filter_by_specifiers_matches_unconfigured_mode(self):
524+ subnet1 = factory.make_Subnet(cidr='10.0.0.0/24')
525+ subnet2 = factory.make_Subnet(cidr='10.0.1.0/24')
526+ subnet3 = factory.make_Subnet(cidr='10.0.2.0/24')
527+ iface1 = factory.make_Interface()
528+ iface2 = factory.make_Interface()
529+ iface3 = factory.make_Interface()
530+ factory.make_StaticIPAddress(
531+ ip='', alloc_type=IPADDRESS_TYPE.STICKY, subnet=subnet1,
532+ interface=iface1)
533+ factory.make_StaticIPAddress(
534+ ip=None, alloc_type=IPADDRESS_TYPE.AUTO, subnet=subnet2,
535+ interface=iface2)
536+ factory.make_StaticIPAddress(
537+ ip='10.0.2.1', alloc_type=IPADDRESS_TYPE.AUTO, subnet=subnet3,
538+ interface=iface3)
539+ self.assertItemsEqual(
540+ [iface1, iface2],
541+ Interface.objects.filter_by_specifiers(
542+ "mode:unconfigured"))
543+
544+
545 class InterfaceTest(MAASServerTestCase):
546
547 def test_rejects_invalid_name(self):
548@@ -742,7 +941,7 @@
549
550 self.assertEqual(num_connections, interface.ip_addresses.count())
551 for i in range(num_connections):
552- ip = interface.ip_addresses.all()[i]
553+ ip = interface.ip_addresses.order_by('id')[i]
554 self.assertThat(ip, MatchesStructure.byEquality(
555 alloc_type=IPADDRESS_TYPE.DISCOVERED, subnet=subnet_list[i],
556 ip=unicode(IPNetwork(cidr_list[i]).ip)))
557@@ -792,7 +991,7 @@
558 "Discovered IP address should have been deleted.")
559 self.assertEqual(num_connections, interface.ip_addresses.count())
560 for i in range(num_connections):
561- ip = interface.ip_addresses.all()[i]
562+ ip = interface.ip_addresses.order_by('id')[i]
563 self.assertThat(ip, MatchesStructure.byEquality(
564 alloc_type=IPADDRESS_TYPE.DISCOVERED, subnet=subnet_list[i],
565 ip=unicode(IPNetwork(cidr_list[i]).ip)))
566@@ -874,7 +1073,7 @@
567 "Sticky IP address should have been deleted.")
568 self.assertEqual(num_connections, interface.ip_addresses.count())
569 for i in range(num_connections):
570- ip = interface.ip_addresses.all()[i]
571+ ip = interface.ip_addresses.order_by('id')[i]
572 self.assertThat(ip, MatchesStructure.byEquality(
573 alloc_type=IPADDRESS_TYPE.DISCOVERED, subnet=subnet_list[i],
574 ip=unicode(IPNetwork(cidr_list[i]).ip)))
575@@ -2126,14 +2325,12 @@
576 factory.make_NodeGroupInterface(
577 nodegroup, management=NODEGROUPINTERFACE_MANAGEMENT.DHCP,
578 subnet=subnet)
579- ip = factory.pick_ip_in_network(subnet.get_ipnetwork())
580- factory.make_StaticIPAddress(
581- alloc_type=IPADDRESS_TYPE.AUTO, ip=ip,
582- subnet=subnet, interface=interface)
583- sip = factory.pick_ip_in_network(subnet.get_ipnetwork())
584- factory.make_StaticIPAddress(
585- alloc_type=IPADDRESS_TYPE.STICKY, ip=sip,
586- subnet=subnet, interface=interface)
587+ factory.make_StaticIPAddress(
588+ alloc_type=IPADDRESS_TYPE.AUTO,
589+ subnet=subnet, interface=interface)
590+ sip = factory.make_StaticIPAddress(
591+ alloc_type=IPADDRESS_TYPE.STICKY,
592+ subnet=subnet, interface=interface).ip
593 interface.release_auto_ips()
594 self.assertThat(
595 mock_update_host_maps,
596
597=== modified file 'src/maasserver/models/tests/test_subnet.py'
598--- src/maasserver/models/tests/test_subnet.py 2015-10-30 10:48:25 +0000
599+++ src/maasserver/models/tests/test_subnet.py 2015-11-02 23:13:19 +0000
600@@ -98,7 +98,7 @@
601 self.assertEqual("169.254.0.0/16", cidr)
602
603
604-class TestSubnetManager(MAASServerTestCase):
605+class TestSubnetQueriesMixin(MAASServerTestCase):
606
607 def test__filter_by_specifiers_takes_single_item(self):
608 subnet1 = factory.make_Subnet(name="subnet1")
609@@ -254,6 +254,61 @@
610 Subnet.objects.filter_by_specifiers("ip:1.1.1.1"),
611 [])
612
613+ def test__matches_interfaces(self):
614+ node1 = factory.make_Node_with_Interface_on_Subnet()
615+ node2 = factory.make_Node_with_Interface_on_Subnet()
616+ iface1 = node1.get_boot_interface()
617+ iface2 = node2.get_boot_interface()
618+ subnet1 = iface1.ip_addresses.first().subnet
619+ subnet2 = iface2.ip_addresses.first().subnet
620+ self.assertItemsEqual(
621+ Subnet.objects.filter_by_specifiers("interface:id:%s" % iface1.id),
622+ [subnet1])
623+ self.assertItemsEqual(
624+ Subnet.objects.filter_by_specifiers("interface:id:%s" % iface2.id),
625+ [subnet2])
626+ self.assertItemsEqual(
627+ Subnet.objects.filter_by_specifiers(
628+ ["interface:id:%s" % iface1.id,
629+ "interface:id:%s" % iface2.id]),
630+ [subnet1, subnet2])
631+
632+ def test__craziness_works(self):
633+ # This test validates that filters can be "chained" to each other
634+ # in an arbitrary way.
635+ node1 = factory.make_Node_with_Interface_on_Subnet()
636+ node2 = factory.make_Node_with_Interface_on_Subnet()
637+ iface1 = node1.get_boot_interface()
638+ iface2 = node2.get_boot_interface()
639+ subnet1 = iface1.ip_addresses.first().subnet
640+ subnet2 = iface2.ip_addresses.first().subnet
641+ self.assertItemsEqual(
642+ Subnet.objects.filter_by_specifiers(
643+ "interface:subnet:id:%s" % subnet1.id),
644+ [subnet1])
645+ self.assertItemsEqual(
646+ Subnet.objects.filter_by_specifiers(
647+ "interface:subnet:id:%s" % subnet2.id),
648+ [subnet2])
649+ self.assertItemsEqual(
650+ Subnet.objects.filter_by_specifiers(
651+ ["interface:subnet:id:%s" % subnet1.id,
652+ "interface:subnet:id:%s" % subnet2.id]),
653+ [subnet1, subnet2])
654+ self.assertItemsEqual(
655+ Subnet.objects.filter_by_specifiers(
656+ "interface:subnet:interface:subnet:id:%s" % subnet1.id),
657+ [subnet1])
658+ self.assertItemsEqual(
659+ Subnet.objects.filter_by_specifiers(
660+ "interface:subnet:interface:subnet:id:%s" % subnet2.id),
661+ [subnet2])
662+ self.assertItemsEqual(
663+ Subnet.objects.filter_by_specifiers(
664+ ["interface:subnet:interface:subnet:id:%s" % subnet1.id,
665+ "interface:subnet:interface:subnet:id:%s" % subnet2.id]),
666+ [subnet1, subnet2])
667+
668
669 class TestSubnetManagerGetSubnetOr404(MAASServerTestCase):
670
671
672=== modified file 'src/maasserver/node_constraint_filter_forms.py'
673--- src/maasserver/node_constraint_filter_forms.py 2015-10-30 19:44:05 +0000
674+++ src/maasserver/node_constraint_filter_forms.py 2015-11-02 23:13:19 +0000
675@@ -32,6 +32,7 @@
676 )
677 import maasserver.forms as maasserver_forms
678 from maasserver.models import (
679+ Interface,
680 PhysicalBlockDevice,
681 Subnet,
682 Tag,
683@@ -377,6 +378,40 @@
684 return nodes
685
686
687+def nodes_by_interface(interfaces_label_map):
688+ """Determines the set of nodes that match the specified
689+ LabeledConstraintMap (which must be a map of storage constraints.)
690+
691+ :param interfaces_label_map: LabeledConstraintMap
692+ :return: set
693+ """
694+ node_ids = None
695+ for label in interfaces_label_map:
696+ specifiers = []
697+ constraints = interfaces_label_map[label]
698+ for specifier_type in constraints.iterkeys():
699+ # Build constraints string.
700+ specifiers.extend([
701+ specifier_type + ":" + item
702+ for item in constraints[specifier_type]
703+ ])
704+ if node_ids is None:
705+ # The first time through the filter, build the list
706+ # of candidate nodes.
707+ nodes = Interface.objects.filter_by_specifiers(
708+ specifiers)
709+ node_ids = set(nodes.values_list('node__id', flat=True))
710+ else:
711+ # For subsequent labels, only match nodes that already matched a
712+ # preceding label.
713+ current_matching_nodes = Interface.objects.filter(
714+ node__id__in=node_ids)
715+ filtered_nodes = current_matching_nodes.filter_by_specifiers(
716+ specifiers)
717+ node_ids = filtered_nodes.values_list('node__id', flat=True)
718+ return node_ids
719+
720+
721 class LabeledConstraintMapField(Field):
722
723 def __init__(self, *args, **kwargs):
724@@ -765,6 +800,14 @@
725 if node_ids is not None:
726 filtered_nodes = filtered_nodes.filter(id__in=node_ids)
727
728+ # Filter by interfaces (networking)
729+ interfaces_label_map = self.cleaned_data.get(
730+ self.get_field_name('interfaces'))
731+ if interfaces_label_map is not None:
732+ node_ids = nodes_by_interface(interfaces_label_map)
733+ if node_ids is not None:
734+ filtered_nodes = filtered_nodes.filter(id__in=node_ids)
735+
736 # This uses a very simple procedure to compute a machine's
737 # cost. This procedure is loosely based on how ec2 computes
738 # the costs of machines. This is here to give a hint to let
739
740=== modified file 'src/maasserver/testing/factory.py'
741--- src/maasserver/testing/factory.py 2015-10-30 13:20:12 +0000
742+++ src/maasserver/testing/factory.py 2015-11-02 23:13:19 +0000
743@@ -561,7 +561,7 @@
744 def make_Node_with_Interface_on_Subnet(
745 self, management=NODEGROUPINTERFACE_MANAGEMENT.DHCP,
746 interface_count=1, nodegroup=None, vlan=None, subnet=None,
747- cidr=None, fabric=None, **kwargs):
748+ cidr=None, fabric=None, ifname=None, **kwargs):
749 """Create a Node that has a Interface which is on a Subnet that has a
750 NodeGroupInterface.
751
752@@ -592,7 +592,7 @@
753 self.make_NodeGroupInterface(
754 nodegroup, vlan=vlan, management=management, subnet=subnet)
755 boot_interface = self.make_Interface(
756- iftype, node=node, vlan=vlan,
757+ iftype, name=ifname, node=node, vlan=vlan,
758 mac_address=mac_address)
759 node.boot_interface = boot_interface
760 node.save()
761@@ -688,7 +688,7 @@
762
763 def make_Space(self, name=None):
764 if name is None:
765- name = self.make_name('space')
766+ name = self.make_name('space--')
767 space = Space(name=name)
768 space.save()
769 return space
770
771=== modified file 'src/maasserver/tests/test_node_constraint_filter_forms.py'
772--- src/maasserver/tests/test_node_constraint_filter_forms.py 2015-10-30 19:51:23 +0000
773+++ src/maasserver/tests/test_node_constraint_filter_forms.py 2015-11-02 23:13:19 +0000
774@@ -977,7 +977,15 @@
775 u'interfaces': u'label:fabric=fabric-0'})
776 self.assertTrue(form.is_valid(), dict(form.errors))
777
778- @skip("XXX mpontillo 2015-10-30 need to use upcoming interfaces filter")
779+ def test_interfaces_constraint_with_multiple_labels_and_values_validated(
780+ self):
781+ factory.make_Node_with_Interface_on_Subnet()
782+ form = AcquireNodeForm({
783+ u'interfaces':
784+ u'label:fabric=fabric-0,fabric=fabric-1,space=default;'
785+ u'label2:fabric=fabric-3,fabric=fabric-4,space=foo'})
786+ self.assertTrue(form.is_valid(), dict(form.errors))
787+
788 def test_interfaces_filters_by_fabric_class(self):
789 fabric1 = factory.make_Fabric(class_type="1g")
790 fabric2 = factory.make_Fabric(class_type="10g")
791@@ -996,6 +1004,52 @@
792 filtered_nodes, _ = form.filter_nodes(Node.nodes)
793 self.assertItemsEqual([node1], filtered_nodes)
794
795+ def test_interfaces_filters_work_with_multiple_labels(self):
796+ fabric1 = factory.make_Fabric(class_type="1g")
797+ fabric2 = factory.make_Fabric(class_type="10g")
798+ vlan1 = factory.make_VLAN(vid=1, fabric=fabric1)
799+ vlan2 = factory.make_VLAN(vid=2, fabric=fabric2)
800+ node1 = factory.make_Node_with_Interface_on_Subnet(
801+ fabric=fabric1, vlan=vlan1)
802+ node2 = factory.make_Node_with_Interface_on_Subnet(
803+ fabric=fabric2, vlan=vlan2)
804+
805+ form = AcquireNodeForm({
806+ u'interfaces': u'fabric:fabric_class=1g;vlan:vid=1'})
807+ self.assertTrue(form.is_valid(), dict(form.errors))
808+ filtered_nodes, _ = form.filter_nodes(Node.nodes)
809+ self.assertItemsEqual([node1], filtered_nodes)
810+
811+ form = AcquireNodeForm({
812+ u'interfaces': u'label:fabric_class=10g;vlan:vid=2'})
813+ self.assertTrue(form.is_valid(), dict(form.errors))
814+ filtered_nodes, _ = form.filter_nodes(Node.nodes)
815+ self.assertItemsEqual([node2], filtered_nodes)
816+
817+ def test_interfaces_filters_multiples_treated_as_OR_operation(self):
818+ fabric1 = factory.make_Fabric(class_type="1g")
819+ fabric2 = factory.make_Fabric(class_type="10g")
820+ vlan1 = factory.make_VLAN(vid=1, fabric=fabric1)
821+ vlan2 = factory.make_VLAN(vid=2, fabric=fabric2)
822+ node1 = factory.make_Node_with_Interface_on_Subnet(
823+ fabric=fabric1, vlan=vlan1)
824+ node2 = factory.make_Node_with_Interface_on_Subnet(
825+ fabric=fabric2, vlan=vlan2)
826+
827+ form = AcquireNodeForm({
828+ u'interfaces':
829+ u'fabric:fabric_class=1g,fabric_class=10g;vlan:vid=1'})
830+ self.assertTrue(form.is_valid(), dict(form.errors))
831+ filtered_nodes, _ = form.filter_nodes(Node.nodes)
832+ self.assertItemsEqual([node1], filtered_nodes)
833+
834+ form = AcquireNodeForm({
835+ u'interfaces':
836+ u'label:fabric_class=10g,fabric_class=1g;vlan:vid=2'})
837+ self.assertTrue(form.is_valid(), dict(form.errors))
838+ filtered_nodes, _ = form.filter_nodes(Node.nodes)
839+ self.assertItemsEqual([node2], filtered_nodes)
840+
841 def test_combined_constraints(self):
842 tag_big = factory.make_Tag(name='big')
843 arch = '%s/generic' % factory.make_name('arch')
844
845=== modified file 'src/maasserver/utils/orm.py'
846--- src/maasserver/utils/orm.py 2015-10-31 06:47:39 +0000
847+++ src/maasserver/utils/orm.py 2015-11-02 23:13:19 +0000
848@@ -48,8 +48,12 @@
849 )
850 import threading
851 from time import sleep
852+import types
853
854-from django.core.exceptions import MultipleObjectsReturned
855+from django.core.exceptions import (
856+ MultipleObjectsReturned,
857+ ValidationError,
858+)
859 from django.db import (
860 connection,
861 connections,
862@@ -64,6 +68,7 @@
863 exponential_growth,
864 full_jitter,
865 )
866+from provisioningserver.utils.network import parse_integer
867 from provisioningserver.utils.twisted import callOut
868 import psycopg2
869 from psycopg2.errorcodes import SERIALIZATION_FAILURE
870@@ -703,7 +708,7 @@
871 return specifier, op
872
873
874-def parse_item_specifier_type(specifier, types={}, separator=':'):
875+def parse_item_specifier_type(specifier, spec_types={}, separator=':'):
876 """
877 Returns a tuple that splits the string int a specifier, and its specifier
878 type.
879@@ -712,13 +717,14 @@
880 be found in the set, returns None in place of the specifier_type.
881
882 :param specifier: The specifier string, such as "ip:10.0.0.1".
883- :param types: A set of strings that will be recognized as specifier types.
884+ :param spec_types: A dict whose keys are strings that will be recognized
885+ as specifier types.
886 :param separator: Optional specifier. Defaults to ':'.
887 :return: tuple
888 """
889 if separator in specifier:
890 tokens = specifier.split(separator, 1)
891- if tokens[0] in types:
892+ if tokens[0] in spec_types:
893 specifier_type = tokens[0]
894 specifier = tokens[1].strip()
895 else:
896@@ -751,7 +757,78 @@
897 ids = self.get_id_list(raw_query)
898 return self.filter(id__in=ids)
899
900- def _get_specifiers_query(self, specifiers, specifier_types=None):
901+ def get_filter_function(
902+ self, specifier_type, spec_types, item, separator=':'):
903+ """Returns a function that must return a Q() based on some pervious
904+ Q(), an operation function (which will manipulate it), and the data
905+ that will be used as an argument to the filter operation function.
906+
907+ :param:specifier_type: a string which will be used as a key to get
908+ the specifier from the spec_types dictionary.
909+ :param:spec_types: the dictionary of valid specifier types.
910+ :param:item: the string that will be used to filter by
911+ :param:separator: a string that must separate specifiers from their
912+ values. (for example, the default of ':' would be used if you
913+ wanted specifiers to look like "id:42".)
914+
915+ :return: types.FunctionType or types.MethodType
916+ """
917+ query = spec_types.get(specifier_type, None)
918+ while True:
919+ if isinstance(query, (types.FunctionType, types.MethodType)):
920+ # Found a function or method that will appending the filter
921+ # string for us. Parameters must be in the format:
922+ # (<current_Q()>, <operation_function>, <item>), where
923+ # the operation_function must be a function that takes action
924+ # on the current_Q() to append a new query object (Q()).
925+ return query
926+ elif isinstance(query, types.TupleType):
927+ # Specifies a query to a subordinate specifier function.
928+ # This will be a tuple in the format:
929+ # (manager_object, filter_from_object)
930+ # That is, filter_from_object defines how to relate the object
931+ # we're querying back to the object that we care about, and
932+ # manager_object is a Django Manager instance.
933+ (manager_object, filter_from_object) = query
934+ sub_ids = manager_object.filter_by_specifiers(
935+ item).values_list(filter_from_object + '__id', flat=True)
936+ # Return a function to filter the current object based on
937+ # its IDs (as gathered from the query above to the related
938+ # object).
939+ kwargs = {"id__in": sub_ids}
940+ return lambda cq, op, i: op(cq, Q(**kwargs))
941+ elif isinstance(query, unicode):
942+ if separator in query:
943+ # We got a string like "subnet:space". This means we want
944+ # to actually use the query specifier at the 'subnet' key,
945+ # but we want to convert the item from (for example)
946+ # "space1" to "space:space1". When we loop back around,
947+ # "subnet" will resolve to a tuple, and we'll query the
948+ # specifier-based filter for Subnet.
949+ query, subordinate = query.split(separator, 1)
950+ item = subordinate + separator + item
951+ elif '__' in query:
952+ # If the value for this query specifier contains the string
953+ # '__', assume it's a Django filter expression, and return
954+ # the appropriate query. Disambiguate what could be an
955+ # 'alias expression' by allowing the __ to appear before
956+ # the filter. (that is, prefix the filter string with __
957+ # to query the current object.)
958+ if query.startswith('__'):
959+ query = query[2:]
960+ kwargs = {query: item}
961+ return lambda cq, op, i: op(cq, Q(**kwargs))
962+ else:
963+ query = spec_types.get(query, None)
964+ elif query is None:
965+ # The None key is for the default query for this specifier.
966+ query = spec_types[None]
967+ else:
968+ break
969+ return None
970+
971+ def get_specifiers_q(
972+ self, specifiers, specifier_types=None, separator=':'):
973 """Returns a Q object for objects matching the given specifiers.
974
975 See documentation for `filter_by_specifiers()`.
976@@ -766,7 +843,37 @@
977 for item in specifiers:
978 item, op = parse_item_operation(item)
979 item, specifier_type = parse_item_specifier_type(
980- item, types=specifier_types)
981- query = specifier_types.get(specifier_type)
982+ item, spec_types=specifier_types, separator=separator)
983+ query = self.get_filter_function(
984+ specifier_type, specifier_types, item, separator=separator)
985 current_q = query(current_q, op, item)
986 return current_q
987+
988+ def filter_by_specifiers(self, specifiers, separator=':'):
989+ query = self.get_specifiers_q(specifiers, separator=separator)
990+ return self.filter(query)
991+
992+ def exclude_by_specifiers(self, specifiers):
993+ """Excludes subnets by the given list of specifiers (or single
994+ specifier).
995+
996+ See documentation for `filter_by_specifiers()`.
997+
998+ :raise:AddrFormatError:If a specific IP address or CIDR is requested,
999+ but the address could not be parsed.
1000+ :return:QuerySet
1001+ """
1002+ query = self.get_specifiers_q(specifiers)
1003+ return self.exclude(query)
1004+
1005+ def _add_vlan_vid_query(self, current_q, op, item):
1006+ if item.lower() == 'untagged':
1007+ vid = 0
1008+ else:
1009+ vid = parse_integer(item)
1010+ if vid < 0 or vid >= 0xfff:
1011+ raise ValidationError(
1012+ "VLAN tag (VID) out of range "
1013+ "(0-4094; 0 for untagged.)")
1014+ current_q = op(current_q, Q(vlan__vid=vid))
1015+ return current_q