MAAS

Merge ~newell-jensen/maas:lxd-commissioning-hooks-cpu into maas:master

  1. Git
  2. lp:~newell-jensen/maas
  3. lxd-commissioning-hooks-cpu
  4. Merge into master
Proposed by Newell Jensen
Status: Merged
Approved by: Newell Jensen
Approved revision: 5148db745e19d3bec6a338716395685294f5ef66
Merge reported by: MAAS Lander
Merged at revision: not available
Proposed branch: ~newell-jensen/maas:lxd-commissioning-hooks-cpu
Merge into: maas:master
Diff against target: 501 lines (+159/-189)
3 files modified
src/metadataserver/builtin_scripts/hooks.py (+73/-43)
src/metadataserver/builtin_scripts/tests/test_hooks.py (+86/-130)
src/provisioningserver/refresh/node_info_scripts.py (+0/-16)
Reviewer Review Type Date Requested Status
MAAS Lander Needs Fixing
Alberto Donato (community) Approve
Review via email: mp+372380@code.launchpad.net

Commit message

Add hooks for LXD cpu information. Remove lscpu and related tests.

To post a comment you must log in.
Revision history for this message
MAAS Lander (maas-lander) wrote :

UNIT TESTS
-b lxd-commissioning-hooks-cpu lp:~newell-jensen/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: d075ac0c49218777f6b188fd3643bfe2d6320284

review: Approve
Revision history for this message
Alberto Donato (ack) wrote :

LGTM, +1

A few minor comments inline.

review: Approve
Revision history for this message
MAAS Lander (maas-lander) wrote :

LANDING
-b lxd-commissioning-hooks-cpu lp:~newell-jensen/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/6358/consoleText

Revision history for this message
MAAS Lander (maas-lander) wrote :

UNIT TESTS
-b lxd-commissioning-hooks-cpu lp:~newell-jensen/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/6359/console
COMMIT: 6080a600c8e3afc9a651756903889d5a8fa5035e

review: Needs Fixing
Revision history for this message
MAAS Lander (maas-lander) wrote :

LANDING
-b lxd-commissioning-hooks-cpu lp:~newell-jensen/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED BUILD
LOG: http://maas-ci-jenkins.internal:8080/job/maas/job/branch-tester/6360/consoleText

~newell-jensen/maas:lxd-commissioning-hooks-cpu updated
5148db7... by Newell Jensen

Add raw string literals for regexs.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/src/metadataserver/builtin_scripts/hooks.py b/src/metadataserver/builtin_scripts/hooks.py
2index 5745ec5..179691c 100644
3--- a/src/metadataserver/builtin_scripts/hooks.py
4+++ b/src/metadataserver/builtin_scripts/hooks.py
5@@ -7,6 +7,7 @@ __all__ = [
6 'NODE_INFO_SCRIPTS',
7 'parse_lshw_nic_info',
8 'update_node_network_information',
9+ 'lxd_update_cpu_details',
10 ]
11
12 import fnmatch
13@@ -34,11 +35,11 @@ from maasserver.utils.orm import get_one
14 from metadataserver.enum import SCRIPT_STATUS
15 from provisioningserver.refresh.node_info_scripts import (
16 BLOCK_DEVICES_OUTPUT_NAME,
17- CPUINFO_OUTPUT_NAME,
18 GET_FRUID_DATA_OUTPUT_NAME,
19 IPADDR_OUTPUT_NAME,
20 LIST_MODALIASES_OUTPUT_NAME,
21 LSHW_OUTPUT_NAME,
22+ LXD_OUTPUT_NAME,
23 NODE_INFO_SCRIPTS,
24 SRIOV_OUTPUT_NAME,
25 VIRTUALITY_OUTPUT_NAME,
26@@ -296,9 +297,8 @@ def get_xml_field_value(evaluator, expression):
27 def update_hardware_details(node, output, exit_status):
28 """Process the results of `LSHW_SCRIPT`.
29
30- Updates `node.cpu_count`, `node.memory`, and `node.storage`
31- fields, and also evaluates all tag expressions against the given
32- ``lshw`` XML.
33+ Updates `node.memory`, and `node.storage` fields, and also
34+ evaluates all tag expressions against the given ``lshw`` XML.
35
36 If `exit_status` is non-zero, this function returns without doing
37 anything.
38@@ -370,55 +370,85 @@ def update_hardware_details(node, output, exit_status):
39 defaults={'value': value})
40
41
42-def parse_cpuinfo(node, output, exit_status):
43- """Parse the output of /proc/cpuinfo."""
44+def process_lxd_results(node, output, exit_status):
45+ """Process the results of `LXD_SCRIPT`.
46+
47+ If `exit_status` is non-zero, this function returns without doing
48+ anything.
49+ """
50 if exit_status != 0:
51 logger.error(
52- "%s: cpuinfo script failed with status: %s." % (
53- node.hostname, exit_status))
54+ "%s: lxd script failed with status: "
55+ "%s." % (node.hostname, exit_status))
56 return
57 assert isinstance(output, bytes)
58- output = output.decode('ascii')
59-
60- cpu_count = len(
61- re.findall(
62- r'^(?P<CPU>\d+),(?P<CORE>\d+),(?P<SOCKET>\d+)$',
63- output, re.MULTILINE))
64- node.cpu_count = cpu_count
65-
66- # Some CPU vendors(Intel) include the speed in the model. If so use that
67- # for the CPU speed as the speeds from lscpu are effected by CPU scaling.
68- m = re.search(
69- r'^Model name:\s+(?P<model_name>.+)(\s@\s(?P<ghz>\d+\.\d+)GHz)$',
70- output, re.MULTILINE)
71- if m is not None:
72- cpu_model = m.group('model_name')
73- node.cpu_speed = int(float(m.group('ghz')) * 1000)
74- else:
75+ try:
76+ data = json.loads(output.decode('utf-8'))
77+ except ValueError as e:
78+ raise ValueError(e.message + ': ' + output)
79+
80+ # Update CPU details.
81+ lxd_update_cpu_details(node, data)
82+
83+
84+def lxd_update_cpu_details(node, data):
85+ """Updates `node.cpu_count`, `node.cpu_speed`, `node.cpu_model`"""
86+ # cpu_count, cpu_speed, cpu_model.
87+ node.cpu_count, node.cpu_speed, cpu_model = _parse_lxd_cpuinfo(data)
88+ # memory.
89+ node.memory = data.get('memory', {}).get('total', 0)
90+
91+ if cpu_model:
92+ NodeMetadata.objects.update_or_create(
93+ node=node, key='cpu_model', defaults={'value': cpu_model})
94+
95+ node.save(update_fields=['cpu_count', 'cpu_speed', 'memory'])
96+
97+
98+def _parse_lxd_cpuinfo(data):
99+ """Retrieve cpu_count, cpu_speed, and cpu_model."""
100+ cpu_speed = 0
101+ cpu_model = None
102+ cpu_count = data.get('cpu', {}).get('total', 0)
103+ # Only update the cpu_model if all the socket names match.
104+ sockets = data.get('cpu', {}).get('sockets', {})
105+ names = []
106+ for socket in sockets:
107+ name = socket.get('name')
108+ if name is not None:
109+ names.append(name)
110+ if len(names) > 0 and all(name == names[0] for name in names):
111+ cpu = names[0]
112 m = re.search(
113- r'^Model name:\s+(?P<model_name>.+)$', output, re.MULTILINE)
114+ r'(?P<model_name>.+)', cpu, re.MULTILINE)
115 if m is not None:
116 cpu_model = m.group('model_name')
117- else:
118- cpu_model = None
119- # Try the max MHz if available.
120+ if '@' in cpu_model:
121+ cpu_model = cpu_model.split(' @')[0]
122+
123+ # Some CPU vendors include the speed in the model. If so use
124+ # that for the CPU speed as the other speeds are effected by
125+ # CPU scaling.
126 m = re.search(
127- r'^CPU max MHz:\s+(?P<mhz>\d+)(\.\d+)?$', output, re.MULTILINE)
128+ r'(\s@\s(?P<ghz>\d+\.\d+)GHz)$', cpu, re.MULTILINE)
129 if m is not None:
130- node.cpu_speed = int(m.group('mhz'))
131+ cpu_speed = int(float(m.group('ghz')) * 1000)
132+ # When socket names don't match or cpu_speed couldn't be retrieved,
133+ # use the max frequency if available before resulting to current
134+ # frequency average.
135+ if not cpu_speed:
136+ frequency_turbo = socket.get('frequency_turbo')
137+ if frequency_turbo is not None:
138+ cpu_speed = frequency_turbo
139 else:
140- # Fall back on the current speed, round it to the nearest hundredth
141- # as the number may be effected by CPU scaling.
142- m = re.search(
143- r'^CPU MHz:\s+(?P<mhz>\d+)(\.\d+)?$', output, re.MULTILINE)
144- if m is not None:
145- node.cpu_speed = round(int(m.group('mhz')) / 100) * 100
146-
147- if cpu_model:
148- NodeMetadata.objects.update_or_create(
149- node=node, key='cpu_model', defaults={'value': cpu_model})
150+ frequency_current_average = socket.get('frequency')
151+ if frequency_current_average is not None:
152+ # Fall back on the current speed, round it to
153+ # the nearest hundredth as the number may be
154+ # effected by CPU scaling.
155+ cpu_speed = round(frequency_current_average / 100) * 100
156
157- node.save(update_fields=['cpu_count', 'cpu_speed'])
158+ return cpu_count, cpu_speed, cpu_model
159
160
161 def set_virtual_tag(node, output, exit_status):
162@@ -945,7 +975,6 @@ def retag_node_for_hardware_by_modalias(
163
164 # Register the post processing hooks.
165 NODE_INFO_SCRIPTS[LSHW_OUTPUT_NAME]['hook'] = update_hardware_details
166-NODE_INFO_SCRIPTS[CPUINFO_OUTPUT_NAME]['hook'] = parse_cpuinfo
167 NODE_INFO_SCRIPTS[VIRTUALITY_OUTPUT_NAME]['hook'] = set_virtual_tag
168 NODE_INFO_SCRIPTS[GET_FRUID_DATA_OUTPUT_NAME]['hook'] = (
169 update_node_fruid_metadata)
170@@ -957,3 +986,4 @@ NODE_INFO_SCRIPTS[SRIOV_OUTPUT_NAME]['hook'] = (
171 update_node_network_interface_tags)
172 NODE_INFO_SCRIPTS[LIST_MODALIASES_OUTPUT_NAME]['hook'] = (
173 create_metadata_by_modalias)
174+NODE_INFO_SCRIPTS[LXD_OUTPUT_NAME]['hook'] = process_lxd_results
175diff --git a/src/metadataserver/builtin_scripts/tests/test_hooks.py b/src/metadataserver/builtin_scripts/tests/test_hooks.py
176index cd4e251..4966736 100644
177--- a/src/metadataserver/builtin_scripts/tests/test_hooks.py
178+++ b/src/metadataserver/builtin_scripts/tests/test_hooks.py
179@@ -5,6 +5,7 @@
180
181 __all__ = []
182
183+from copy import deepcopy
184 import doctest
185 import json
186 import os.path
187@@ -40,7 +41,7 @@ from metadataserver.builtin_scripts.hooks import (
188 extract_router_mac_addresses,
189 filter_modaliases,
190 get_dmi_data,
191- parse_cpuinfo,
192+ lxd_update_cpu_details,
193 retag_node_for_hardware_by_modalias,
194 set_virtual_tag,
195 SWITCH_OPENBMC_MAC,
196@@ -72,6 +73,7 @@ lldp_output_template = """
197 </lldp>
198 """
199
200+
201 lldp_output_interface_template = """
202 <interface label="Interface" name="eth1" via="LLDP">
203 <chassis label="Chassis">
204@@ -86,6 +88,65 @@ lldp_output_interface_template = """
205 """
206
207
208+SAMPLE_JSON_CPUINFO = {
209+ "cpu": {
210+ "architecture": "x86_64",
211+ "sockets": [
212+ {
213+ "name": "Intel(R) Core(TM) i7-4700MQ CPU @ 2.40GHz",
214+ "vendor": "GenuineIntel",
215+ "socket": 0,
216+ "cache": [
217+ {
218+ "level": 1,
219+ "type": "Data",
220+ "size": 32768
221+ },
222+ {
223+ "level": 1,
224+ "type": "Instruction",
225+ "size": 32768
226+ },
227+ {
228+ "level": 2,
229+ "type": "Unified",
230+ "size": 262144
231+ },
232+ {
233+ "level": 3,
234+ "type": "Unified",
235+ "size": 6291456
236+ }
237+ ],
238+ "cores": [
239+ {
240+ "core": 0,
241+ "numa_node": 0,
242+ "threads": [
243+ {
244+ "id": 0,
245+ "thread": 0,
246+ "online": True
247+ },
248+ {
249+ "id": 1,
250+ "thread": 1,
251+ "online": True
252+ }
253+ ],
254+ "frequency": 3247
255+ }
256+ ],
257+ "frequency": 3247,
258+ "frequency_minimum": 800,
259+ "frequency_turbo": 3400
260+ }
261+ ],
262+ "total": 8
263+ }
264+}
265+
266+
267 def make_lldp_output(macs):
268 """Return an example raw lldp output containing the given MACs."""
269 interfaces = '\n'.join(
270@@ -821,159 +882,54 @@ class TestUpdateHardwareDetails(MAASServerTestCase):
271 self.assertIsNone(NodeMetadata.objects.get(node=node, key=key))
272
273
274-class TestParseCPUInfo(MAASServerTestCase):
275+class TestLXDUpdateCPUDetails(MAASServerTestCase):
276
277 doctest_flags = doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE
278
279- def test_parse_cpuinfo_speed_in_model(self):
280+ def test__speed_in_name(self):
281 node = factory.make_Node()
282 node.cpu_count = 2
283 node.cpu_speed = 9999
284 node.save()
285- # Sample lscpu output from a single socket, quad core with
286- # hyperthreading CPU. Flags have been ommitted to avoid lint errors.
287- cpuinfo = dedent("""\
288- Architecture: x86_64
289- CPU op-mode(s): 32-bit, 64-bit
290- Byte Order: Little Endian
291- CPU(s): 8
292- On-line CPU(s) list: 0-7
293- Thread(s) per core: 2
294- Core(s) per socket: 4
295- Socket(s): 1
296- NUMA node(s): 1
297- Vendor ID: GenuineIntel
298- CPU family: 6
299- Model: 60
300- Model name: Intel(R) Core(TM) i7-4910MQ CPU @ 2.90GHz
301- Stepping: 3
302- CPU MHz: 2893.284
303- CPU max MHz: 3900.0000
304- CPU min MHz: 800.0000
305- BogoMIPS: 5786.32
306- Virtualization: VT-x
307- L1d cache: 32K
308- L1i cache: 32K
309- L2 cache: 256K
310- L3 cache: 8192K
311- NUMA node0 CPU(s): 0-7
312- # The following is the parsable format, which can be fed to other
313- # programs. Each different item in every column has an unique ID
314- # starting from zero.
315- # CPU,Core,Socket
316- 0,0,0
317- 1,0,0
318- 2,1,0
319- 3,1,0
320- 4,2,0
321- 5,2,0
322- 6,3,0
323- 7,3,0
324- """).encode('utf-8')
325- parse_cpuinfo(node, cpuinfo, 0)
326+
327+ lxd_update_cpu_details(node, SAMPLE_JSON_CPUINFO)
328 node = reload_object(node)
329 self.assertEqual(8, node.cpu_count)
330- self.assertEqual(2900, node.cpu_speed)
331+ self.assertEqual(2400, node.cpu_speed)
332 nmd = NodeMetadata.objects.get(node=node, key='cpu_model')
333- self.assertEqual('Intel(R) Core(TM) i7-4910MQ CPU', nmd.value)
334+ self.assertEqual('Intel(R) Core(TM) i7-4700MQ CPU', nmd.value)
335
336- def test_parse_cpuinfo_max_speed(self):
337+ def test__max_speed(self):
338 node = factory.make_Node()
339 node.cpu_count = 2
340 node.cpu_speed = 9999
341 node.save()
342- # Sample lscpu output from a single socket, quad core with
343- # hyperthreading CPU. Flags have been ommitted to avoid lint errors.
344- cpuinfo = dedent("""\
345- Architecture: x86_64
346- CPU op-mode(s): 32-bit, 64-bit
347- Byte Order: Little Endian
348- CPU(s): 4
349- On-line CPU(s) list: 0-3
350- Thread(s) per core: 1
351- Core(s) per socket: 4
352- Socket(s): 1
353- NUMA node(s): 1
354- Vendor ID: AuthenticAMD
355- CPU family: 22
356- Model: 0
357- Model name: AMD Athlon(tm) 5350 APU with Radeon(tm) R3
358- Stepping: 1
359- CPU MHz: 800.000
360- CPU max MHz: 2050.0000
361- CPU min MHz: 800.0000
362- BogoMIPS: 4092.20
363- Virtualization: AMD-V
364- L1d cache: 32K
365- L1i cache: 32K
366- L2 cache: 2048K
367- NUMA node0 CPU(s): 0-3
368- # The following is the parsable format, which can be fed to other
369- # programs. Each different item in every column has an unique ID
370- # starting from zero.
371- # CPU,Core,Socket
372- 0,0,0
373- 1,1,0
374- 2,2,0
375- 3,3,0
376- """).encode('utf-8')
377- parse_cpuinfo(node, cpuinfo, 0)
378+
379+ SAMPLE_JSON_CPUINFO_NO_SPEED_IN_NAME = deepcopy(SAMPLE_JSON_CPUINFO)
380+ SAMPLE_JSON_CPUINFO_NO_SPEED_IN_NAME['cpu']['sockets'][0]['name'] = (
381+ 'Intel(R) Core(TM) i7-4700MQ CPU')
382+ lxd_update_cpu_details(node, SAMPLE_JSON_CPUINFO_NO_SPEED_IN_NAME)
383 node = reload_object(node)
384- self.assertEqual(4, node.cpu_count)
385- self.assertEqual(2050, node.cpu_speed)
386+ self.assertEqual(8, node.cpu_count)
387+ self.assertEqual(3400, node.cpu_speed)
388 nmd = NodeMetadata.objects.get(node=node, key='cpu_model')
389- self.assertEqual(
390- 'AMD Athlon(tm) 5350 APU with Radeon(tm) R3', nmd.value)
391+ self.assertEqual('Intel(R) Core(TM) i7-4700MQ CPU', nmd.value)
392
393- def test_parse_cpuinfo_speed_current(self):
394+ def test__current_speed(self):
395 node = factory.make_Node()
396 node.cpu_count = 2
397 node.cpu_speed = 9999
398 node.save()
399- # Sample lscpu output from a single socket, quad core with
400- # hyperthreading CPU. Flags have been ommitted to avoid lint errors.
401- cpuinfo = dedent("""\
402- Architecture: x86_64
403- CPU op-mode(s): 32-bit, 64-bit
404- Byte Order: Little Endian
405- CPU(s): 8
406- On-line CPU(s) list: 0-7
407- Thread(s) per core: 2
408- Core(s) per socket: 4
409- Socket(s): 1
410- NUMA node(s): 1
411- Vendor ID: GenuineIntel
412- CPU family: 6
413- Model: 60
414- Model name: Intel(R) Core(TM) i7-4910MQ CPU
415- Stepping: 3
416- CPU MHz: 2893.284
417- BogoMIPS: 5786.32
418- Virtualization: VT-x
419- L1d cache: 32K
420- L1i cache: 32K
421- L2 cache: 256K
422- L3 cache: 8192K
423- NUMA node0 CPU(s): 0-7
424- # The following is the parsable format, which can be fed to other
425- # programs. Each different item in every column has an unique ID
426- # starting from zero.
427- # CPU,Core,Socket
428- 0,0,0
429- 1,0,0
430- 2,1,0
431- 3,1,0
432- 4,2,0
433- 5,2,0
434- 6,3,0
435- 7,3,0
436- """).encode('utf-8')
437- parse_cpuinfo(node, cpuinfo, 0)
438+
439+ SAMPLE_JSON_CPUINFO_NO_NAME_OR_MAX_FREQ = deepcopy(SAMPLE_JSON_CPUINFO)
440+ del SAMPLE_JSON_CPUINFO_NO_NAME_OR_MAX_FREQ[
441+ 'cpu']['sockets'][0]['name']
442+ del SAMPLE_JSON_CPUINFO_NO_NAME_OR_MAX_FREQ[
443+ 'cpu']['sockets'][0]['frequency_turbo']
444+ lxd_update_cpu_details(node, SAMPLE_JSON_CPUINFO_NO_NAME_OR_MAX_FREQ)
445 node = reload_object(node)
446 self.assertEqual(8, node.cpu_count)
447- self.assertEqual(2900, node.cpu_speed)
448- nmd = NodeMetadata.objects.get(node=node, key='cpu_model')
449- self.assertEqual('Intel(R) Core(TM) i7-4910MQ CPU', nmd.value)
450+ self.assertEqual(3200, node.cpu_speed)
451
452
453 class TestUpdateNodePhysicalBlockDevices(MAASServerTestCase):
454diff --git a/src/provisioningserver/refresh/node_info_scripts.py b/src/provisioningserver/refresh/node_info_scripts.py
455index e9a24f2..beeef29 100644
456--- a/src/provisioningserver/refresh/node_info_scripts.py
457+++ b/src/provisioningserver/refresh/node_info_scripts.py
458@@ -5,7 +5,6 @@
459
460 __all__ = [
461 'BLOCK_DEVICES_OUTPUT_NAME',
462- 'CPUINFO_OUTPUT_NAME',
463 'DHCP_EXPLORE_OUTPUT_NAME',
464 'GET_FRUID_DATA_OUTPUT_NAME',
465 'IPADDR_OUTPUT_NAME',
466@@ -35,7 +34,6 @@ from textwrap import dedent
467 # 0014_rename_dhcp_unconfigured_ifaces.py
468 SUPPORT_INFO_OUTPUT_NAME = '00-maas-00-support-info'
469 LSHW_OUTPUT_NAME = '00-maas-01-lshw'
470-CPUINFO_OUTPUT_NAME = '00-maas-01-cpuinfo'
471 VIRTUALITY_OUTPUT_NAME = '00-maas-02-virtuality'
472 LLDP_INSTALL_OUTPUT_NAME = '00-maas-03-install-lldpd'
473 LIST_MODALIASES_OUTPUT_NAME = '00-maas-04-list-modaliases'
474@@ -156,14 +154,6 @@ VIRTUALITY_SCRIPT = dedent("""\
475 fi
476 """)
477
478-CPUINFO_SCRIPT = dedent("""\
479- #!/bin/bash
480- # Gather the standard output as it has some extra info
481- lscpu
482- # Gather the machine readable output for processing
483- lscpu -p=cpu,core,socket
484- """)
485-
486 SERIAL_PORTS_SCRIPT = dedent("""\
487 #!/bin/bash
488 find /sys/class/tty/ ! -type d -print0 2> /dev/null \
489@@ -690,12 +680,6 @@ NODE_INFO_SCRIPTS = OrderedDict([
490 'timeout': timedelta(minutes=1),
491 'run_on_controller': True,
492 }),
493- (CPUINFO_OUTPUT_NAME, {
494- 'content': CPUINFO_SCRIPT.encode('ascii'),
495- 'hook': null_hook,
496- 'timeout': timedelta(seconds=10),
497- 'run_on_controller': True,
498- }),
499 (VIRTUALITY_OUTPUT_NAME, {
500 'content': VIRTUALITY_SCRIPT.encode('ascii'),
501 'hook': null_hook,

Subscribers

People subscribed via source and target branches