Merge lp:~ack/landscape-client/swift-usage into lp:~landscape/landscape-client/trunk
- swift-usage
- Merge into trunk
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Alberto Donato | ||||
Approved revision: | 784 | ||||
Merged at revision: | 765 | ||||
Proposed branch: | lp:~ack/landscape-client/swift-usage | ||||
Merge into: | lp:~landscape/landscape-client/trunk | ||||
Prerequisite: | lp:~ack/landscape-client/ceph-usage-report-bytes | ||||
Diff against target: |
963 lines (+354/-504) 4 files modified
landscape/message_schemas.py (+5/-1) landscape/monitor/config.py (+1/-1) landscape/monitor/swiftusage.py (+129/-158) landscape/monitor/tests/test_swiftusage.py (+219/-344) |
||||
To merge this branch: | bzr merge lp:~ack/landscape-client/swift-usage | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Chris Glass (community) | Approve | ||
Geoff Teale (community) | Approve | ||
Review via email: mp+219845@code.launchpad.net |
Commit message
This branch replaces the SwiftDeviceInfo plugin with SwiftUsage, which reports free/used/available space for each device in the cluster.
It uses a different message type from the previous plugin.
Description of the change
This branch replaces the SwiftDeviceInfo plugin with SwiftUsage, which reports free/used/available space for each device in the cluster.
It uses a different message type from the previous plugin.
- 778. By Alberto Donato
-
Merged ceph-usage-
report- bytes into swift-usage. - 779. By Alberto Donato
-
Report usages as bytes, with integer type.
- 780. By Alberto Donato
-
Merge from trunk.
Free Ekanayaka (free.ekanayaka) wrote : | # |
Quick comment:
+SWIFT = Message("swift", {
+ "usages": List(Tuple(Int(), Unicode(), Int(), Int(), Int()))})
+
I'd rename this to SWIFT_USAGE/
Please add a comment explaining the format of the message too, e.g.:
# Report Swift node usage in the last time interval.
SWIFT = Message("swift", {
# List of data points of the form (timestamp, device, size, avail, used)
"usages": List(Tuple(Int(), Unicode(), Int(), Int(), Int()))})
Free Ekanayaka (free.ekanayaka) wrote : | # |
Uhm looking at the other branch it seems you're changing "ceph-usage" to "ceph". FWIW I prefer "ceph-usage", you could just bump the client API and change the format (e.g. rename "ceph-usages" to "data-points"). In case we have other Ceph-related messages they should be probably be separate (e.g. "ceph-otherstuff").
- 781. By Alberto Donato
-
Change plugin interval.
- 782. By Alberto Donato
-
Address Geoff's review points.
- 783. By Alberto Donato
-
Rename swift message to swift-usage.
- 784. By Alberto Donato
-
Fix docstring.
Preview Diff
1 | === modified file 'landscape/message_schemas.py' | |||
2 | --- landscape/message_schemas.py 2014-05-19 07:45:24 +0000 | |||
3 | +++ landscape/message_schemas.py 2014-05-21 06:46:43 +0000 | |||
4 | @@ -136,6 +136,10 @@ | |||
5 | 136 | KeyDict({"device": Unicode(), "mounted": Bool()})) | 136 | KeyDict({"device": Unicode(), "mounted": Bool()})) |
6 | 137 | }) | 137 | }) |
7 | 138 | 138 | ||
8 | 139 | SWIFT_USAGE = Message("swift-usage", { | ||
9 | 140 | # Usage data points in the form (timestamp, device, size, avail, used) | ||
10 | 141 | "data-points": List(Tuple(Int(), Unicode(), Int(), Int(), Int()))}) | ||
11 | 142 | |||
12 | 139 | KEYSTONE_TOKEN = Message("keystone-token", { | 143 | KEYSTONE_TOKEN = Message("keystone-token", { |
13 | 140 | "data": Any(Bytes(), Constant(None)) | 144 | "data": Any(Bytes(), Constant(None)) |
14 | 141 | }) | 145 | }) |
15 | @@ -470,5 +474,5 @@ | |||
16 | 470 | EUCALYPTUS_INFO_ERROR, NETWORK_DEVICE, NETWORK_ACTIVITY, | 474 | EUCALYPTUS_INFO_ERROR, NETWORK_DEVICE, NETWORK_ACTIVITY, |
17 | 471 | REBOOT_REQUIRED_INFO, UPDATE_MANAGER_INFO, CPU_USAGE, | 475 | REBOOT_REQUIRED_INFO, UPDATE_MANAGER_INFO, CPU_USAGE, |
18 | 472 | CEPH_USAGE, CEPH, SWIFT_DEVICE_INFO, KEYSTONE_TOKEN, | 476 | CEPH_USAGE, CEPH, SWIFT_DEVICE_INFO, KEYSTONE_TOKEN, |
20 | 473 | CHANGE_HA_SERVICE, JUJU_INFO, CLOUD_METADATA]: | 477 | CHANGE_HA_SERVICE, JUJU_INFO, CLOUD_METADATA, SWIFT_USAGE]: |
21 | 474 | message_schemas[schema.type] = schema | 478 | message_schemas[schema.type] = schema |
22 | 475 | 479 | ||
23 | === modified file 'landscape/monitor/config.py' | |||
24 | --- landscape/monitor/config.py 2013-09-26 16:12:39 +0000 | |||
25 | +++ landscape/monitor/config.py 2014-05-21 06:46:43 +0000 | |||
26 | @@ -5,7 +5,7 @@ | |||
27 | 5 | "LoadAverage", "MemoryInfo", "MountInfo", "ProcessorInfo", | 5 | "LoadAverage", "MemoryInfo", "MountInfo", "ProcessorInfo", |
28 | 6 | "Temperature", "PackageMonitor", "UserMonitor", | 6 | "Temperature", "PackageMonitor", "UserMonitor", |
29 | 7 | "RebootRequired", "AptPreferences", "NetworkActivity", | 7 | "RebootRequired", "AptPreferences", "NetworkActivity", |
31 | 8 | "NetworkDevice", "UpdateManager", "CPUUsage", "SwiftDeviceInfo", | 8 | "NetworkDevice", "UpdateManager", "CPUUsage", "SwiftUsage", |
32 | 9 | "CephUsage", "JujuInfo"] | 9 | "CephUsage", "JujuInfo"] |
33 | 10 | 10 | ||
34 | 11 | 11 | ||
35 | 12 | 12 | ||
36 | === renamed file 'landscape/monitor/swiftdeviceinfo.py' => 'landscape/monitor/swiftusage.py' | |||
37 | --- landscape/monitor/swiftdeviceinfo.py 2013-07-08 14:31:52 +0000 | |||
38 | +++ landscape/monitor/swiftusage.py 2014-05-21 06:46:43 +0000 | |||
39 | @@ -1,183 +1,154 @@ | |||
40 | 1 | import logging | 1 | import logging |
41 | 2 | import time | 2 | import time |
42 | 3 | import os | 3 | import os |
46 | 4 | import json | 4 | |
47 | 5 | 5 | from twisted.internet import threads | |
48 | 6 | from landscape.lib.fetch import fetch, HTTPCodeError, PyCurlError, FetchError | 6 | |
49 | 7 | from landscape.accumulate import Accumulator | ||
50 | 7 | from landscape.lib.monitor import CoverageMonitor | 8 | from landscape.lib.monitor import CoverageMonitor |
51 | 8 | from landscape.lib.network import get_active_device_info | 9 | from landscape.lib.network import get_active_device_info |
52 | 9 | from landscape.monitor.plugin import MonitorPlugin | 10 | from landscape.monitor.plugin import MonitorPlugin |
53 | 10 | 11 | ||
58 | 11 | 12 | try: | |
59 | 12 | class SwiftDeviceInfo(MonitorPlugin): | 13 | from swift.common.ring import Ring |
60 | 13 | 14 | from swift.cli.recon import Scout | |
61 | 14 | persist_name = "swift-device-info" | 15 | has_swift = True |
62 | 16 | except ImportError: | ||
63 | 17 | has_swift = False | ||
64 | 18 | |||
65 | 19 | |||
66 | 20 | class SwiftUsage(MonitorPlugin): | ||
67 | 21 | """Plugin reporting Swift cluster usage. | ||
68 | 22 | |||
69 | 23 | This only works if the client runs on a Swift node. It requires the | ||
70 | 24 | 'python-swift' package to be installed (which is installed on swift nodes). | ||
71 | 25 | |||
72 | 26 | """ | ||
73 | 27 | |||
74 | 28 | persist_name = "swift-usage" | ||
75 | 15 | scope = "storage" | 29 | scope = "storage" |
76 | 16 | 30 | ||
78 | 17 | def __init__(self, interval=300, monitor_interval=60 * 60, | 31 | def __init__(self, interval=30, monitor_interval=60 * 60, |
79 | 18 | create_time=time.time, | 32 | create_time=time.time, |
80 | 19 | swift_config="/etc/swift/object-server.conf", | ||
81 | 20 | swift_ring="/etc/swift/object.ring.gz"): | 33 | swift_ring="/etc/swift/object.ring.gz"): |
83 | 21 | self.run_interval = interval | 34 | self._interval = interval |
84 | 22 | self._monitor_interval = monitor_interval | 35 | self._monitor_interval = monitor_interval |
85 | 23 | self._create_time = create_time | 36 | self._create_time = create_time |
95 | 24 | self._fetch = fetch | 37 | self._swift_ring = swift_ring # To discover Recon host/port |
96 | 25 | self._get_network_devices = get_active_device_info | 38 | |
97 | 26 | self._swift_config = swift_config # If exists, we are a swift node | 39 | self._has_swift = has_swift |
98 | 27 | self._swift_ring = swift_ring # To discover swift recon port | 40 | self._swift_usage_points = [] |
99 | 28 | self._swift_recon_url = None | 41 | self.active = True |
91 | 29 | self._create_time = create_time | ||
92 | 30 | self._swift_device_info = [] | ||
93 | 31 | self._swift_device_info_to_persist = [] | ||
94 | 32 | self.enabled = True | ||
100 | 33 | 42 | ||
101 | 34 | def register(self, registry): | 43 | def register(self, registry): |
108 | 35 | super(SwiftDeviceInfo, self).register(registry) | 44 | super(SwiftUsage, self).register(registry) |
109 | 36 | self._monitor = CoverageMonitor(self.run_interval, 0.8, | 45 | self._accumulate = Accumulator(self._persist, self._interval) |
110 | 37 | "swift device info snapshot", | 46 | self._monitor = CoverageMonitor( |
111 | 38 | create_time=self._create_time) | 47 | self.run_interval, 0.8, "Swift device usage snapshot", |
112 | 39 | self.registry.reactor.call_every(self._monitor_interval, | 48 | create_time=self._create_time) |
113 | 40 | self._monitor.log) | 49 | self.registry.reactor.call_every( |
114 | 50 | self._monitor_interval, self._monitor.log) | ||
115 | 51 | |||
116 | 41 | self.registry.reactor.call_on("stop", self._monitor.log, priority=2000) | 52 | self.registry.reactor.call_on("stop", self._monitor.log, priority=2000) |
130 | 42 | self.call_on_accepted("swift-device-info", self.send_messages, True) | 53 | self.call_on_accepted("swift-usage", self.send_message, True) |
131 | 43 | 54 | ||
132 | 44 | def create_swift_device_info_message(self): | 55 | def create_message(self): |
133 | 45 | if self._swift_device_info: | 56 | usage_points = self._swift_usage_points |
134 | 46 | message = {"type": "swift-device-info", | 57 | self._swift_usage_points = [] |
135 | 47 | "swift-device-info": self._swift_device_info} | 58 | if usage_points: |
136 | 48 | self._swift_device_info_to_persist = self._swift_device_info[:] | 59 | return {"type": "swift-usage", "data-points": usage_points} |
137 | 49 | self._swift_device_info = [] | 60 | |
138 | 50 | return message | 61 | def send_message(self, urgent=False): |
139 | 51 | return None | 62 | message = self.create_message() |
127 | 52 | |||
128 | 53 | def send_messages(self, urgent=False): | ||
129 | 54 | message = self.create_swift_device_info_message() | ||
140 | 55 | if message: | 63 | if message: |
143 | 56 | logging.info("Queueing message with updated swift device info.") | 64 | self.registry.broker.send_message( |
142 | 57 | d = self.registry.broker.send_message( | ||
144 | 58 | message, self._session_id, urgent=urgent) | 65 | message, self._session_id, urgent=urgent) |
161 | 59 | d.addCallback(lambda x: self.persist_swift_info()) | 66 | |
162 | 60 | 67 | def exchange(self, urgent=False): | |
163 | 61 | def exchange(self): | 68 | self.registry.broker.call_if_accepted( |
164 | 62 | self.registry.broker.call_if_accepted("swift-device-info", | 69 | "swift-usage", self.send_message, urgent) |
149 | 63 | self.send_messages) | ||
150 | 64 | |||
151 | 65 | def persist_swift_info(self): | ||
152 | 66 | for swift_device_info in self._swift_device_info_to_persist: | ||
153 | 67 | device_name = swift_device_info["device"] | ||
154 | 68 | key = (self.persist_name, device_name) | ||
155 | 69 | self._persist.set(key, swift_device_info) | ||
156 | 70 | self._swift_device_info_to_persist = None | ||
157 | 71 | # This forces the registry to write the persistent store to disk | ||
158 | 72 | # This means that the persistent data reflects the state of the | ||
159 | 73 | # messages sent. | ||
160 | 74 | self.registry.flush() | ||
165 | 75 | 70 | ||
166 | 76 | def run(self): | 71 | def run(self): |
168 | 77 | if not self.enabled: | 72 | if not self._should_run(): |
169 | 78 | return | 73 | return |
170 | 74 | |||
171 | 79 | self._monitor.ping() | 75 | self._monitor.ping() |
172 | 80 | 76 | ||
198 | 81 | current_swift_devices = self._get_swift_devices() | 77 | host = self._get_recon_host() |
199 | 82 | current_device_names = [] | 78 | deferred = threads.deferToThread(self._perform_recon_call, host) |
200 | 83 | for swift_info in current_swift_devices: | 79 | deferred.addCallback(self._handle_usage) |
201 | 84 | device_name = swift_info["device"] | 80 | return deferred |
202 | 85 | current_device_names.append(device_name) | 81 | |
203 | 86 | key = (self.persist_name, device_name) | 82 | def _should_run(self): |
204 | 87 | prev_swift_info = self._persist.get(key) | 83 | """Return whether the plugin should run.""" |
205 | 88 | if not prev_swift_info or prev_swift_info != swift_info: | 84 | if not self.active: |
206 | 89 | if swift_info not in self._swift_device_info: | 85 | return False |
207 | 90 | self._swift_device_info.append(swift_info) | 86 | |
208 | 91 | 87 | if not self._has_swift: | |
184 | 92 | # Get all persisted devices and remove those that no longer exist | ||
185 | 93 | persisted_devices = self._persist.get(self.persist_name) | ||
186 | 94 | if persisted_devices: | ||
187 | 95 | for device_name in persisted_devices.keys(): | ||
188 | 96 | if device_name not in current_device_names: | ||
189 | 97 | self._persist.remove((self.persist_name, device_name)) | ||
190 | 98 | |||
191 | 99 | def _get_swift_devices(self): | ||
192 | 100 | config_file = self._swift_config | ||
193 | 101 | # Check if a swift storage config file is available. No need to run | ||
194 | 102 | # if we know that we're not on a swift monitor node anyway. | ||
195 | 103 | if not os.path.exists(config_file): | ||
196 | 104 | # There is no config file - it's not a swift storage machine. | ||
197 | 105 | self.enabled = False | ||
209 | 106 | logging.info( | 88 | logging.info( |
274 | 107 | "This does not appear to be a swift storage server. '%s' " | 89 | "This machine does not appear to be a Swift machine. " |
275 | 108 | "plugin has been disabled." % self.persist_name) | 90 | "Deactivating plugin.") |
276 | 109 | return [] | 91 | self.active = False |
277 | 110 | 92 | return False | |
278 | 111 | # Extract the swift service URL from the ringfile and cache it. | 93 | |
279 | 112 | if self._swift_recon_url is None: | 94 | # Check for object ring config file. |
280 | 113 | ring = self._get_ring() | 95 | # If it is not present, it's not a Swift machine or it not yet set up. |
217 | 114 | if ring is None: | ||
218 | 115 | return [] | ||
219 | 116 | |||
220 | 117 | network_devices = self._get_network_devices() | ||
221 | 118 | local_ips = [device["ip_address"] for device in network_devices] | ||
222 | 119 | |||
223 | 120 | # Grab first swift service with an IP on this host | ||
224 | 121 | for dev in ring.devs: | ||
225 | 122 | if dev and dev["ip"] in local_ips: | ||
226 | 123 | self._swift_recon_url = "http://%s:%d/recon/diskusage" % ( | ||
227 | 124 | dev['ip'], dev['port']) | ||
228 | 125 | break | ||
229 | 126 | |||
230 | 127 | if self._swift_recon_url is None: | ||
231 | 128 | self.enabled = False | ||
232 | 129 | logging.error( | ||
233 | 130 | "Local swift service not found. '%s' plugin has " | ||
234 | 131 | "been disabled." % self.persist_name) | ||
235 | 132 | return [] | ||
236 | 133 | |||
237 | 134 | recon_disk_info = self._get_swift_disk_usage() | ||
238 | 135 | # We don't care about avail and free figures because we track | ||
239 | 136 | # free_space for mounted devices in free-space messages | ||
240 | 137 | return [{"device": "/dev/%s" % device["device"], | ||
241 | 138 | "mounted": device["mounted"]} for device in recon_disk_info] | ||
242 | 139 | |||
243 | 140 | def _get_swift_disk_usage(self): | ||
244 | 141 | """ | ||
245 | 142 | Query the swift storage usage data by parsing the curled recon data | ||
246 | 143 | from http://localhost:<_swift_service_port>/recon/diskusage. | ||
247 | 144 | Lots of recon data for the picking described at: | ||
248 | 145 | http://docs.openstack.org/developer/swift/admin_guide.html | ||
249 | 146 | """ | ||
250 | 147 | error_message = None | ||
251 | 148 | try: | ||
252 | 149 | content = self._fetch(self._swift_recon_url) | ||
253 | 150 | except HTTPCodeError, error: | ||
254 | 151 | error_message = ( | ||
255 | 152 | "Swift service is running without swift-recon enabled.") | ||
256 | 153 | except (FetchError, PyCurlError), error: | ||
257 | 154 | error_message = ( | ||
258 | 155 | "Swift service not available at %s. %s." % | ||
259 | 156 | (self._swift_recon_url, str(error))) | ||
260 | 157 | if error_message is not None: | ||
261 | 158 | self.enabled = False | ||
262 | 159 | logging.error("%s '%s' plugin has been disabled." % ( | ||
263 | 160 | error_message, self.persist_name)) | ||
264 | 161 | return None | ||
265 | 162 | |||
266 | 163 | if not content: | ||
267 | 164 | return None | ||
268 | 165 | |||
269 | 166 | swift_disk_usages = json.loads(content) # list of device dicts | ||
270 | 167 | return swift_disk_usages | ||
271 | 168 | |||
272 | 169 | def _get_ring(self): | ||
273 | 170 | """Return ring-file object from self._swift_ring location""" | ||
281 | 171 | if not os.path.exists(self._swift_ring): | 96 | if not os.path.exists(self._swift_ring): |
294 | 172 | logging.warning( | 97 | return False |
295 | 173 | "Swift ring files are not available yet.") | 98 | |
296 | 174 | return None | 99 | return True |
297 | 175 | try: | 100 | |
298 | 176 | from swift.common.ring import Ring | 101 | def _get_recon_host(self): |
299 | 177 | except ImportError: | 102 | """Return a tuple with Recon (host, port).""" |
300 | 178 | self.enabled = False | 103 | local_ips = self._get_local_ips() |
301 | 179 | logging.error( | 104 | ring = Ring(self._swift_ring) |
302 | 180 | "Swift python common libraries not found. '%s' plugin has " | 105 | for dev in ring.devs: |
303 | 181 | "been disabled." % self.persist_name) | 106 | if dev and dev["ip"] in local_ips: |
304 | 182 | return None | 107 | return dev["ip"], dev["port"] |
305 | 183 | return Ring(self._swift_ring) | 108 | |
306 | 109 | def _get_local_ips(self): | ||
307 | 110 | """Return a list of IP addresses for local devices.""" | ||
308 | 111 | return [ | ||
309 | 112 | device["ip_address"] for device in get_active_device_info()] | ||
310 | 113 | |||
311 | 114 | def _perform_recon_call(self, host): | ||
312 | 115 | """Get usage information from Swift Recon service.""" | ||
313 | 116 | if not host: | ||
314 | 117 | return | ||
315 | 118 | |||
316 | 119 | scout = Scout("diskusage") | ||
317 | 120 | # Perform the actual call | ||
318 | 121 | _, disk_usage, code = scout.scout(host) | ||
319 | 122 | if code == 200: | ||
320 | 123 | return disk_usage | ||
321 | 124 | |||
322 | 125 | def _handle_usage(self, disk_usage): | ||
323 | 126 | timestamp = int(self._create_time()) | ||
324 | 127 | |||
325 | 128 | devices = set() | ||
326 | 129 | for usage in disk_usage: | ||
327 | 130 | if not usage["mounted"]: | ||
328 | 131 | continue | ||
329 | 132 | |||
330 | 133 | device = usage["device"] | ||
331 | 134 | devices.add(device) | ||
332 | 135 | |||
333 | 136 | step_values = [] | ||
334 | 137 | for key in ("size", "avail", "used"): | ||
335 | 138 | # Store values in tree so it's easy to delete all values for a | ||
336 | 139 | # device | ||
337 | 140 | persist_key = "usage.%s.%s" % (device, key) | ||
338 | 141 | step_value = self._accumulate( | ||
339 | 142 | timestamp, usage[key], persist_key) | ||
340 | 143 | step_values.append(step_value) | ||
341 | 144 | |||
342 | 145 | if all(step_values): | ||
343 | 146 | point = [step_value[0], device] # accumulated timestamp | ||
344 | 147 | point.extend(int(step_value[1]) for step_value in step_values) | ||
345 | 148 | self._swift_usage_points.append(tuple(point)) | ||
346 | 149 | |||
347 | 150 | # Update device list and remove usage for devices that no longer exist. | ||
348 | 151 | current_devices = set(self._persist.get("devices", ())) | ||
349 | 152 | for device in current_devices - devices: | ||
350 | 153 | self._persist.remove("usage.%s" % device) | ||
351 | 154 | self._persist.set("devices", list(devices)) | ||
352 | 184 | 155 | ||
353 | === renamed file 'landscape/monitor/tests/test_swiftdeviceinfo.py' => 'landscape/monitor/tests/test_swiftusage.py' | |||
354 | --- landscape/monitor/tests/test_swiftdeviceinfo.py 2013-07-12 13:41:07 +0000 | |||
355 | +++ landscape/monitor/tests/test_swiftusage.py 2014-05-21 06:46:43 +0000 | |||
356 | @@ -1,388 +1,263 @@ | |||
357 | 1 | from twisted.internet.defer import succeed | 1 | from twisted.internet.defer import succeed |
358 | 2 | 2 | ||
361 | 3 | from landscape.lib.fetch import HTTPCodeError | 3 | from landscape.monitor.swiftusage import SwiftUsage |
360 | 4 | from landscape.monitor.swiftdeviceinfo import SwiftDeviceInfo | ||
362 | 5 | from landscape.tests.helpers import LandscapeTest, MonitorHelper | 4 | from landscape.tests.helpers import LandscapeTest, MonitorHelper |
363 | 6 | from landscape.tests.mocker import ANY | 5 | from landscape.tests.mocker import ANY |
364 | 7 | 6 | ||
365 | 8 | 7 | ||
367 | 9 | class FakeRingInfo(object): | 8 | class FakeRing(object): |
368 | 10 | def __init__(self, ip_port_tuples=[]): | 9 | def __init__(self, ip_port_tuples=[]): |
376 | 11 | self.devs = [] | 10 | self.devs = [ |
377 | 12 | for ip, port in ip_port_tuples: | 11 | {"ip": ip, "port": port} |
378 | 13 | self.devs.append({"ip": ip, "port": port}) | 12 | for ip, port in ip_port_tuples] |
379 | 14 | 13 | ||
380 | 15 | 14 | ||
381 | 16 | class SwiftDeviceInfoTest(LandscapeTest): | 15 | class SwiftUsageTest(LandscapeTest): |
382 | 17 | """Tests for swift-device-info plugin.""" | 16 | """Tests for swift-usage plugin.""" |
383 | 18 | 17 | ||
384 | 19 | helpers = [MonitorHelper] | 18 | helpers = [MonitorHelper] |
385 | 20 | 19 | ||
386 | 21 | def setUp(self): | 20 | def setUp(self): |
387 | 22 | LandscapeTest.setUp(self) | 21 | LandscapeTest.setUp(self) |
389 | 23 | self.mstore.set_accepted_types(["swift-device-info"]) | 22 | self.mstore.set_accepted_types(["swift-usage"]) |
390 | 23 | self.plugin = SwiftUsage( | ||
391 | 24 | create_time=self.reactor.time, swift_ring=self.makeFile("ring")) | ||
392 | 25 | self.plugin._has_swift = True | ||
393 | 26 | |||
394 | 27 | def test_wb_should_run_not_active(self): | ||
395 | 28 | """ | ||
396 | 29 | L{SwiftUsage._should_run} returns C{False} if plugin is not active. | ||
397 | 30 | """ | ||
398 | 31 | plugin = SwiftUsage(create_time=self.reactor.time) | ||
399 | 32 | plugin.active = False | ||
400 | 33 | self.assertFalse(plugin._should_run()) | ||
401 | 34 | |||
402 | 35 | def test_wb_should_run_no_swift(self): | ||
403 | 36 | """ | ||
404 | 37 | L{SwiftUsage._should_run} returns C{False} if Swift client library is | ||
405 | 38 | not available. It also disables the plugin. | ||
406 | 39 | """ | ||
407 | 40 | plugin = SwiftUsage(create_time=self.reactor.time) | ||
408 | 41 | plugin._has_swift = False | ||
409 | 42 | self.assertFalse(plugin._should_run()) | ||
410 | 43 | self.assertFalse(plugin.active) | ||
411 | 44 | |||
412 | 45 | def test_wb_should_run_no_swift_ring(self): | ||
413 | 46 | """ | ||
414 | 47 | L{SwiftUsage._should_run} returns C{False} if Swift ring configuration | ||
415 | 48 | file is not found. | ||
416 | 49 | """ | ||
417 | 50 | plugin = SwiftUsage( | ||
418 | 51 | create_time=self.reactor.time, swift_ring=self.makeFile()) | ||
419 | 52 | plugin._has_swift = True | ||
420 | 53 | self.assertFalse(plugin._should_run()) | ||
421 | 54 | |||
422 | 55 | def test_wb_should_run(self): | ||
423 | 56 | """ | ||
424 | 57 | L{SwiftUsage._should_run} returns C{True} if everything if the Swift | ||
425 | 58 | ring is properly configured. | ||
426 | 59 | """ | ||
427 | 60 | plugin = SwiftUsage( | ||
428 | 61 | create_time=self.reactor.time, swift_ring=self.makeFile("ring")) | ||
429 | 62 | plugin._has_swift = True | ||
430 | 63 | self.assertTrue(plugin._should_run()) | ||
431 | 24 | 64 | ||
432 | 25 | def test_exchange_messages(self): | 65 | def test_exchange_messages(self): |
433 | 26 | """ | 66 | """ |
438 | 27 | The swift_device_info plugin queues message when manager.exchange() | 67 | The plugin queues message when manager.exchange() is called. |
439 | 28 | is called. Each message should be aligned to a step boundary; | 68 | Each message should be aligned to a step boundary; only a sing message |
440 | 29 | only a sing message with the latest swift device information will | 69 | with the latest swift device information will be delivered in a single |
441 | 30 | be delivered in a single message. | 70 | message. |
442 | 31 | """ | 71 | """ |
460 | 32 | def fake_swift_devices(): | 72 | points = [(1234, "sdb", 100000, 80000, 20000), |
461 | 33 | return [{"device": "/dev/hdf", "mounted": True}, | 73 | (1234, "sdc", 200000, 120000, 800000)] |
462 | 34 | {"device": "/dev/hda2", "mounted": False}] | 74 | self.plugin._swift_usage_points = points |
463 | 35 | 75 | ||
464 | 36 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | 76 | self.monitor.add(self.plugin) |
448 | 37 | plugin._get_swift_devices = fake_swift_devices | ||
449 | 38 | |||
450 | 39 | step_size = self.monitor.step_size | ||
451 | 40 | self.monitor.add(plugin) | ||
452 | 41 | |||
453 | 42 | # Exchange should trigger a flush of the persist database | ||
454 | 43 | registry_mocker = self.mocker.replace(plugin.registry) | ||
455 | 44 | registry_mocker.flush() | ||
456 | 45 | self.mocker.result(None) | ||
457 | 46 | self.mocker.replay() | ||
458 | 47 | |||
459 | 48 | self.reactor.advance(step_size * 2) | ||
465 | 49 | self.monitor.exchange() | 77 | self.monitor.exchange() |
466 | 50 | 78 | ||
467 | 51 | messages = self.mstore.get_pending_messages() | 79 | messages = self.mstore.get_pending_messages() |
468 | 52 | self.assertEqual(len(messages), 1) | 80 | self.assertEqual(len(messages), 1) |
488 | 53 | expected_message_content = [ | 81 | data_points = messages[0]["data-points"] |
489 | 54 | {"device": "/dev/hdf", "mounted": True}, | 82 | self.assertEqual(points, data_points) |
490 | 55 | {"device": "/dev/hda2", "mounted": False}] | 83 | |
491 | 56 | 84 | def test_no_exchange_empty_messages(self): | |
492 | 57 | swift_devices = messages[0]["swift-device-info"] | 85 | """ |
493 | 58 | self.assertEqual(swift_devices, expected_message_content) | 86 | If no usage data is available, no message is exchanged. |
494 | 59 | 87 | """ | |
495 | 60 | def test_messaging_flushes(self): | 88 | self.monitor.add(self.plugin) |
496 | 61 | """ | 89 | self.monitor.exchange() |
497 | 62 | Duplicate message should never be created. If no data is | 90 | |
498 | 63 | available, None will be returned when messages are created. | 91 | self.assertEqual([], self.mstore.get_pending_messages()) |
499 | 64 | """ | 92 | |
500 | 65 | def fake_swift_devices(): | 93 | def test_create_message(self): |
501 | 66 | return [{"device": "/dev/hdf", "mounted": True}, | 94 | """L{SwiftUsage.create_message} returns a 'swift-usage' message.""" |
502 | 67 | {"device": "/dev/hda2", "mounted": False}] | 95 | points = [(1234, "sdb", 100000, 80000, 20000), |
503 | 68 | 96 | (1234, "sdc", 200000, 120000, 80000)] | |
504 | 69 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | 97 | self.plugin._swift_usage_points = points |
505 | 70 | self.monitor.add(plugin) | 98 | message = self.plugin.create_message() |
506 | 71 | plugin._get_swift_devices = fake_swift_devices | 99 | self.assertEqual( |
507 | 100 | {"type": "swift-usage", "data-points": points}, message) | ||
508 | 101 | |||
509 | 102 | def test_create_message_empty(self): | ||
510 | 103 | """ | ||
511 | 104 | L{SwiftUsage.create_message} returns C{None} if no data are available. | ||
512 | 105 | """ | ||
513 | 106 | self.assertIs(None, self.plugin.create_message()) | ||
514 | 107 | |||
515 | 108 | def test_crate_message_flushes(self): | ||
516 | 109 | """Duplicate message should never be created.""" | ||
517 | 110 | self.monitor.add(self.plugin) | ||
518 | 111 | self.plugin._swift_usage_points = [(1234, "sdb", 100000, 80000, 20000)] | ||
519 | 112 | self.plugin._get_recon_host = lambda: ("192.168.1.10", 6000) | ||
520 | 72 | 113 | ||
521 | 73 | self.reactor.advance(self.monitor.step_size) | 114 | self.reactor.advance(self.monitor.step_size) |
597 | 74 | 115 | message = self.plugin.create_message() | |
598 | 75 | message = plugin.create_swift_device_info_message() | 116 | self.assertIsNot(None, message) |
599 | 76 | self.assertEqual(message.keys(), ["swift-device-info", "type"]) | 117 | message = self.plugin.create_message() |
600 | 77 | 118 | self.assertIs(None, message) | |
526 | 78 | message = plugin.create_swift_device_info_message() | ||
527 | 79 | self.assertEqual(message, None) | ||
528 | 80 | |||
529 | 81 | def test_never_exchange_empty_messages(self): | ||
530 | 82 | """ | ||
531 | 83 | When the plugin has no data, its various create_X_message() | ||
532 | 84 | methods will return None. Empty or null messages should never | ||
533 | 85 | be queued. | ||
534 | 86 | """ | ||
535 | 87 | self.mstore.set_accepted_types(["load-average"]) | ||
536 | 88 | |||
537 | 89 | plugin = SwiftDeviceInfo() | ||
538 | 90 | self.monitor.add(plugin) | ||
539 | 91 | self.monitor.exchange() | ||
540 | 92 | self.assertEqual(len(self.mstore.get_pending_messages()), 0) | ||
541 | 93 | |||
542 | 94 | def test_messages_with_swift_data(self): | ||
543 | 95 | """ | ||
544 | 96 | All swift-affiliated devices are sent in swift-device-info messages. | ||
545 | 97 | Both mounted and unmounted swift devices send data. | ||
546 | 98 | """ | ||
547 | 99 | def fake_swift_devices(): | ||
548 | 100 | return [{"device": "/dev/hdf", "mounted": True}, | ||
549 | 101 | {"device": "/dev/hda2", "mounted": False}] | ||
550 | 102 | |||
551 | 103 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | ||
552 | 104 | |||
553 | 105 | plugin._get_swift_devices = fake_swift_devices | ||
554 | 106 | |||
555 | 107 | step_size = self.monitor.step_size | ||
556 | 108 | self.monitor.add(plugin) | ||
557 | 109 | plugin.run() | ||
558 | 110 | |||
559 | 111 | self.reactor.advance(step_size) | ||
560 | 112 | self.monitor.exchange() | ||
561 | 113 | |||
562 | 114 | messages = self.mstore.get_pending_messages() | ||
563 | 115 | self.assertEqual(len(messages), 1) | ||
564 | 116 | |||
565 | 117 | # Need to see both mounted and unmounted swift device info | ||
566 | 118 | self.assertEqual( | ||
567 | 119 | messages[0].get("swift-device-info"), | ||
568 | 120 | [{'device': u'/dev/hdf', 'mounted': True}, | ||
569 | 121 | {'device': u'/dev/hda2', 'mounted': False}]) | ||
570 | 122 | |||
571 | 123 | def test_resynchronize(self): | ||
572 | 124 | """ | ||
573 | 125 | On the reactor "resynchronize" event, new swift-device-info messages | ||
574 | 126 | should be sent. | ||
575 | 127 | """ | ||
576 | 128 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | ||
577 | 129 | |||
578 | 130 | def fake_swift_devices(): | ||
579 | 131 | return [{"device": "/dev/hdf", "mounted": True}, | ||
580 | 132 | {"device": "/dev/hda2", "mounted": False}] | ||
581 | 133 | |||
582 | 134 | self.monitor.add(plugin) | ||
583 | 135 | plugin._get_swift_devices = fake_swift_devices | ||
584 | 136 | |||
585 | 137 | plugin.run() | ||
586 | 138 | plugin.exchange() | ||
587 | 139 | self.reactor.fire("resynchronize", scopes=["storage"]) | ||
588 | 140 | plugin.run() | ||
589 | 141 | plugin.exchange() | ||
590 | 142 | messages = self.mstore.get_pending_messages() | ||
591 | 143 | expected_message = { | ||
592 | 144 | "type": "swift-device-info", | ||
593 | 145 | "swift-device-info": [ | ||
594 | 146 | {"device": "/dev/hdf", "mounted": True}, | ||
595 | 147 | {"device": "/dev/hda2", "mounted": False}]} | ||
596 | 148 | self.assertMessages(messages, [expected_message, expected_message]) | ||
601 | 149 | 119 | ||
602 | 150 | def test_no_message_if_not_accepted(self): | 120 | def test_no_message_if_not_accepted(self): |
603 | 151 | """ | 121 | """ |
606 | 152 | Don't add any messages at all if the broker isn't currently | 122 | No message is sent if the broker isn't currently accepting their type. |
605 | 153 | accepting their type. | ||
607 | 154 | """ | 123 | """ |
608 | 124 | self.plugin._swift_usage_points = [(1234, "sdb", 100000, 80000, 20000)] | ||
609 | 125 | self.plugin._get_recon_host = lambda: ("192.168.1.10", 6000) | ||
610 | 126 | |||
611 | 155 | self.mstore.set_accepted_types([]) | 127 | self.mstore.set_accepted_types([]) |
620 | 156 | 128 | self.monitor.add(self.plugin) | |
613 | 157 | def fake_swift_devices(): | ||
614 | 158 | return [{"device": "/dev/hdf", "mounted": True}, | ||
615 | 159 | {"device": "/dev/hda2", "mounted": False}] | ||
616 | 160 | |||
617 | 161 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | ||
618 | 162 | self.monitor.add(plugin) | ||
619 | 163 | plugin._get_swift_devices = fake_swift_devices | ||
621 | 164 | 129 | ||
622 | 165 | self.reactor.advance(self.monitor.step_size * 2) | 130 | self.reactor.advance(self.monitor.step_size * 2) |
623 | 166 | self.monitor.exchange() | 131 | self.monitor.exchange() |
624 | 167 | 132 | ||
625 | 168 | self.mstore.set_accepted_types(["swift-device-info"]) | ||
626 | 169 | self.assertMessages(list(self.mstore.get_pending_messages()), []) | 133 | self.assertMessages(list(self.mstore.get_pending_messages()), []) |
627 | 170 | 134 | ||
628 | 171 | def test_call_on_accepted(self): | 135 | def test_call_on_accepted(self): |
629 | 172 | """ | 136 | """ |
631 | 173 | When message type acceptance is added for swift-device-info, | 137 | When message type acceptance is added for 'swift' message, |
632 | 174 | send_message gets called. | 138 | send_message gets called. |
633 | 175 | """ | 139 | """ |
643 | 176 | def fake_swift_devices(): | 140 | self.plugin._swift_usage_points = [(1234, "sdb", 100000, 80000, 20000)] |
644 | 177 | return [{"device": "/dev/hdf", "mounted": True}, | 141 | self.plugin._get_recon_host = lambda: ("192.168.1.10", 6000) |
645 | 178 | {"device": "/dev/hda2", "mounted": False}] | 142 | |
646 | 179 | 143 | self.monitor.add(self.plugin) | |
647 | 180 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | 144 | self.reactor.advance(self.plugin.run_interval) |
639 | 181 | self.monitor.add(plugin) | ||
640 | 182 | plugin._get_swift_devices = fake_swift_devices | ||
641 | 183 | |||
642 | 184 | self.reactor.advance(plugin.run_interval) | ||
648 | 185 | 145 | ||
649 | 186 | remote_broker_mock = self.mocker.replace(self.remote) | 146 | remote_broker_mock = self.mocker.replace(self.remote) |
650 | 187 | remote_broker_mock.send_message(ANY, ANY, urgent=True) | 147 | remote_broker_mock.send_message(ANY, ANY, urgent=True) |
651 | 188 | self.mocker.result(succeed(None)) | 148 | self.mocker.result(succeed(None)) |
653 | 189 | self.mocker.count(1) # 1 send message is called for swift-device-info | 149 | self.mocker.count(1) # 1 send message is called |
654 | 190 | self.mocker.replay() | 150 | self.mocker.replay() |
655 | 191 | 151 | ||
656 | 192 | self.reactor.fire( | 152 | self.reactor.fire( |
853 | 193 | ("message-type-acceptance-changed", "swift-device-info"), True) | 153 | ("message-type-acceptance-changed", "swift-usage"), True) |
854 | 194 | 154 | ||
855 | 195 | def test_persist_deltas(self): | 155 | def test_message_only_mounted_devices(self): |
856 | 196 | """ | 156 | """ |
857 | 197 | Swift persistent device info drops old devices from persist storage if | 157 | The plugin only collects usage for mounted devices. |
858 | 198 | the device no longer exists in the current device list. | 158 | """ |
859 | 199 | """ | 159 | recon_response = [ |
860 | 200 | def fake_swift_devices(): | 160 | {"device": "vdb", |
861 | 201 | return [{"device": "/dev/hdf", "mounted": True}, | 161 | "mounted": True, |
862 | 202 | {"device": "/dev/hda2", "mounted": False}] | 162 | "size": 100000, |
863 | 203 | 163 | "avail": 80000, | |
864 | 204 | def fake_swift_devices_no_hdf(): | 164 | "used": 20000}, |
865 | 205 | return [{"device": "/dev/hda2", "mounted": False}] | 165 | {"device": "vdc", |
866 | 206 | 166 | "mounted": False, | |
867 | 207 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | 167 | "size": "", |
868 | 208 | self.monitor.add(plugin) | 168 | "avail": "", |
869 | 209 | plugin._get_swift_devices = fake_swift_devices | 169 | "used": ""}, |
870 | 210 | plugin.run() | 170 | {"device": "vdd", |
871 | 211 | plugin.exchange() # To persist swift recon data | 171 | "mounted": True, |
872 | 212 | self.assertEqual( | 172 | "size": 200000, |
873 | 213 | plugin._persist.get("swift-device-info"), | 173 | "avail": 10000, |
874 | 214 | {"/dev/hdf": {"device": "/dev/hdf", "mounted": True}, | 174 | "used": 190000}] |
875 | 215 | "/dev/hda2": {"device": "/dev/hda2", "mounted": False}}) | 175 | self.plugin._perform_recon_call = lambda host: succeed(recon_response) |
876 | 216 | 176 | self.plugin._get_recon_host = lambda: ("192.168.1.10", 6000) | |
877 | 217 | # Drop a device | 177 | |
878 | 218 | plugin._get_swift_devices = fake_swift_devices_no_hdf | 178 | self.monitor.add(self.plugin) |
879 | 219 | plugin.run() | 179 | self.reactor.advance(self.plugin._interval) |
880 | 220 | plugin.exchange() | 180 | self.plugin._handle_usage(recon_response) |
881 | 221 | self.assertEqual( | 181 | |
882 | 222 | plugin._persist.get("swift-device-info"), | 182 | self.assertEqual( |
883 | 223 | {"/dev/hda2": {"device": "/dev/hda2", "mounted": False}}) | 183 | [(30, "vdb", 100000, 80000, 20000), |
884 | 224 | 184 | (30, "vdd", 200000, 10000, 190000)], | |
885 | 225 | # Run again, calling create_swift_device_info_message which purges info | 185 | self.plugin._swift_usage_points) |
886 | 226 | plugin.run() | 186 | self.assertEqual(["vdb", "vdd"], self.plugin._persist.get("devices")) |
887 | 227 | plugin.exchange() | 187 | self.assertNotIn("vdc", self.plugin._persist.get("usage")) |
888 | 228 | message3 = plugin.create_swift_device_info_message() | 188 | |
889 | 229 | self.assertIdentical(message3, None) | 189 | def test_message_remove_disappeared_devices(self): |
890 | 230 | 190 | """ | |
891 | 231 | def test_persist_timing(self): | 191 | Usage for devices that have disappeared are removed from the persist. |
892 | 232 | """Swift device info is only persisted when exchange happens. | 192 | """ |
893 | 233 | 193 | recon_response = [ | |
894 | 234 | If an event happened between the persist and the exchange, the server | 194 | {"device": "vdb", |
895 | 235 | didn't get the mount info at all. This test ensures that mount info are | 195 | "mounted": True, |
896 | 236 | only saved when exchange happens. | 196 | "size": 100000, |
897 | 237 | """ | 197 | "avail": 80000, |
898 | 238 | def fake_swift_devices(): | 198 | "used": 20000}, |
899 | 239 | return [{"device": "/dev/hdf", "mounted": True}, | 199 | {"device": "vdc", |
900 | 240 | {"device": "/dev/hda2", "mounted": False}] | 200 | "mounted": True, |
901 | 241 | 201 | "size": 200000, | |
902 | 242 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | 202 | "avail": 10000, |
903 | 243 | self.monitor.add(plugin) | 203 | "used": 190000}] |
904 | 244 | plugin._get_swift_devices = fake_swift_devices | 204 | self.plugin._perform_recon_call = lambda host: succeed(recon_response) |
905 | 245 | plugin.run() | 205 | self.plugin._get_recon_host = lambda: ("192.168.1.10", 6000) |
906 | 246 | message1 = plugin.create_swift_device_info_message() | 206 | |
907 | 247 | self.assertEqual( | 207 | self.monitor.add(self.plugin) |
908 | 248 | message1.get("swift-device-info"), | 208 | self.reactor.advance(self.monitor.step_size) |
909 | 249 | [{"device": "/dev/hdf", "mounted": True}, | 209 | self.plugin._handle_usage(recon_response) |
910 | 250 | {"device": "/dev/hda2", "mounted": False}]) | 210 | self.assertEqual( |
911 | 251 | plugin.run() | 211 | ["vdb", "vdc"], sorted(self.plugin._persist.get("devices"))) |
912 | 252 | message2 = plugin.create_swift_device_info_message() | 212 | |
913 | 253 | self.assertEqual( | 213 | recon_response = [ |
914 | 254 | message2.get("swift-device-info"), | 214 | {"device": "vdb", |
915 | 255 | [{"device": "/dev/hdf", "mounted": True}, | 215 | "mounted": True, |
916 | 256 | {"device": "/dev/hda2", "mounted": False}]) | 216 | "size": 100000, |
917 | 257 | # Run again, calling create_swift_device_info_message which purges info | 217 | "avail": 70000, |
918 | 258 | plugin.run() | 218 | "used": 30000}] |
919 | 259 | plugin.exchange() | 219 | self.reactor.advance(self.monitor.step_size) |
920 | 260 | plugin.run() | 220 | self.plugin._handle_usage(recon_response) |
921 | 261 | message3 = plugin.create_swift_device_info_message() | 221 | self.assertNotIn("vdc", self.plugin._persist.get("usage")) |
922 | 262 | self.assertIdentical(message3, None) | 222 | self.assertEqual(["vdb"], self.plugin._persist.get("devices")) |
923 | 263 | 223 | ||
924 | 264 | def test_wb_get_swift_devices_when_not_a_swift_node(self): | 224 | def test_message_remove_unmounted_devices(self): |
925 | 265 | """ | 225 | """ |
926 | 266 | When not a swift node, _get_swift_devices returns an empty list and | 226 | Usage for devices that are no longer mounted are removed from the |
927 | 267 | no error messages. | 227 | persist. |
928 | 268 | """ | 228 | """ |
929 | 269 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | 229 | recon_response = [ |
930 | 270 | self.assertEqual(plugin._get_swift_devices(), []) | 230 | {"device": "vdb", |
931 | 271 | 231 | "mounted": True, | |
932 | 272 | def test_wb_get_swift_devices_when_on_a_swift_node(self): | 232 | "size": 100000, |
933 | 273 | """ | 233 | "avail": 80000, |
934 | 274 | When on a swift node, _get_swift_devices reports a warning if the ring | 234 | "used": 20000}, |
935 | 275 | files don't exist yet. | 235 | {"device": "vdc", |
936 | 276 | """ | 236 | "mounted": True, |
937 | 277 | plugin = SwiftDeviceInfo(create_time=self.reactor.time, | 237 | "size": 200000, |
938 | 278 | swift_config="/etc/hosts") | 238 | "avail": 10000, |
939 | 279 | logging_mock = self.mocker.replace("logging.warning") | 239 | "used": 190000}] |
940 | 280 | logging_mock("Swift ring files are not available yet.") | 240 | self.plugin._perform_recon_call = lambda host: succeed(recon_response) |
941 | 281 | self.mocker.replay() | 241 | self.plugin._get_recon_host = lambda: ("192.168.1.10", 6000) |
942 | 282 | self.assertEqual(plugin._get_swift_devices(), []) | 242 | |
943 | 283 | 243 | self.monitor.add(self.plugin) | |
944 | 284 | def test_run_disabled_when_missing_swift_config(self): | 244 | self.reactor.advance(self.monitor.step_size) |
945 | 285 | """ | 245 | self.plugin._handle_usage(recon_response) |
946 | 286 | When on a node that doesn't have the appropriate swift config file. The | 246 | self.assertEqual( |
947 | 287 | plugin logs an info message and is disabled. | 247 | ["vdb", "vdc"], sorted(self.plugin._persist.get("devices"))) |
948 | 288 | """ | 248 | |
949 | 289 | plugin = SwiftDeviceInfo(create_time=self.reactor.time, | 249 | recon_response = [ |
950 | 290 | swift_config="/config/file/doesnotexist") | 250 | {"device": "vdb", |
951 | 291 | logging_mock = self.mocker.replace("logging.info") | 251 | "mounted": True, |
952 | 292 | logging_mock("This does not appear to be a swift storage server. " | 252 | "size": 100000, |
953 | 293 | "'swift-device-info' plugin has been disabled.") | 253 | "avail": 70000, |
954 | 294 | self.mocker.replay() | 254 | "used": 30000}, |
955 | 295 | self.monitor.add(plugin) | 255 | {"device": "vdc", |
956 | 296 | self.assertEqual(plugin.enabled, True) | 256 | "mounted": False, |
957 | 297 | plugin.run() | 257 | "size": "", |
958 | 298 | self.assertEqual(plugin.enabled, False) | 258 | "avail": "", |
959 | 299 | 259 | "used": ""}] | |
960 | 300 | def test_wb_get_swift_devices_no_swift_python_libs_available(self): | 260 | self.reactor.advance(self.monitor.step_size) |
961 | 301 | """ | 261 | self.plugin._handle_usage(recon_response) |
962 | 302 | The plugin logs an error and doesn't find swift devices when it can't | 262 | self.assertNotIn("vdc", self.plugin._persist.get("usage")) |
963 | 303 | import the swift python libs which it requires. | 263 | self.assertEqual(["vdb"], self.plugin._persist.get("devices")) |
768 | 304 | """ | ||
769 | 305 | plugin = SwiftDeviceInfo(create_time=self.reactor.time, | ||
770 | 306 | swift_config="/etc/hosts", | ||
771 | 307 | swift_ring="/etc/hosts") | ||
772 | 308 | |||
773 | 309 | logging_mock = self.mocker.replace("logging.error") | ||
774 | 310 | logging_mock("Swift python common libraries not found. " | ||
775 | 311 | "'swift-device-info' plugin has been disabled.") | ||
776 | 312 | self.mocker.replay() | ||
777 | 313 | |||
778 | 314 | self.assertEqual(plugin._get_swift_devices(), []) | ||
779 | 315 | |||
780 | 316 | def test_wb_get_swift_disk_usage_when_no_swift_service_running(self): | ||
781 | 317 | """ | ||
782 | 318 | When the swift service is running, but recon middleware is not active, | ||
783 | 319 | the Swift storage usage logs an error. | ||
784 | 320 | """ | ||
785 | 321 | self.log_helper.ignore_errors(".*") | ||
786 | 322 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | ||
787 | 323 | plugin._swift_recon_url = "http://localhost:12654" | ||
788 | 324 | result = plugin._get_swift_disk_usage() | ||
789 | 325 | self.assertIs(None, result) | ||
790 | 326 | self.assertIn( | ||
791 | 327 | "Swift service not available at %s." % plugin._swift_recon_url, | ||
792 | 328 | self.logfile.getvalue()) | ||
793 | 329 | |||
794 | 330 | def test_wb_get_swift_disk_usage_when_no_recon_service_configured(self): | ||
795 | 331 | """ | ||
796 | 332 | When the swift service is running, but recon middleware is not active, | ||
797 | 333 | an error is logged. | ||
798 | 334 | """ | ||
799 | 335 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | ||
800 | 336 | |||
801 | 337 | plugin._swift_recon_url = "http://localhost:12654" | ||
802 | 338 | |||
803 | 339 | def fetch_error(url): | ||
804 | 340 | raise HTTPCodeError(400, "invalid path: /recon/diskusage") | ||
805 | 341 | plugin._fetch = fetch_error | ||
806 | 342 | |||
807 | 343 | logging_mock = self.mocker.replace("logging.error", passthrough=False) | ||
808 | 344 | logging_mock( | ||
809 | 345 | "Swift service is running without swift-recon enabled. " | ||
810 | 346 | "'swift-device-info' plugin has been disabled.") | ||
811 | 347 | self.mocker.result(None) | ||
812 | 348 | self.mocker.replay() | ||
813 | 349 | |||
814 | 350 | result = plugin._get_swift_disk_usage() | ||
815 | 351 | self.assertIs(None, result) | ||
816 | 352 | |||
817 | 353 | def test_wb_get_swift_usage_no_information(self): | ||
818 | 354 | """ | ||
819 | 355 | When the swift recon service returns no disk usage information, | ||
820 | 356 | the _get_swift_disk_usage method returns None. | ||
821 | 357 | """ | ||
822 | 358 | plugin = SwiftDeviceInfo(create_time=self.reactor.time) | ||
823 | 359 | |||
824 | 360 | def fetch_none(url): | ||
825 | 361 | return None | ||
826 | 362 | |||
827 | 363 | plugin._fetch = fetch_none | ||
828 | 364 | |||
829 | 365 | result = plugin._get_swift_disk_usage() | ||
830 | 366 | self.assertEqual(None, result) | ||
831 | 367 | |||
832 | 368 | def test_wb_get_swift_devices_no_matched_local_service(self): | ||
833 | 369 | """ | ||
834 | 370 | The plugin logs an error when the swift ring file does not represent | ||
835 | 371 | a swift service running local IP address on the current node. | ||
836 | 372 | """ | ||
837 | 373 | plugin = SwiftDeviceInfo(create_time=self.reactor.time, | ||
838 | 374 | swift_config="/etc/hosts") | ||
839 | 375 | |||
840 | 376 | def get_fake_ring(): | ||
841 | 377 | return FakeRingInfo([("192.168.1.10", 6000)]) | ||
842 | 378 | plugin._get_ring = get_fake_ring | ||
843 | 379 | |||
844 | 380 | def local_network_devices(): | ||
845 | 381 | return [{"ip_address": "10.1.2.3"}] | ||
846 | 382 | plugin._get_network_devices = local_network_devices | ||
847 | 383 | |||
848 | 384 | logging_mock = self.mocker.replace("logging.error") | ||
849 | 385 | logging_mock("Local swift service not found. " | ||
850 | 386 | "'swift-device-info' plugin has been disabled.") | ||
851 | 387 | self.mocker.replay() | ||
852 | 388 | self.assertEqual(plugin._get_swift_devices(), []) |
+1
[0] monitor/ swiftusage. py - Line 23
landscape/
This only works if the client runs on a Swift note. It requires the
Typo - "note" -> "node"
[1] monitor/ swiftusage. py - Line 47
landscape/
I'd say "Swift device usage" rather than "Swift devices usage"