Merge lp:~terceiro/lava-dashboard-tool/absorbed-by-lava-tool into lp:lava-dashboard-tool
- absorbed-by-lava-tool
- Merge into trunk
Proposed by
Antonio Terceiro
Status: | Merged |
---|---|
Approved by: | Senthil Kumaran S |
Approved revision: | 160 |
Merged at revision: | 160 |
Proposed branch: | lp:~terceiro/lava-dashboard-tool/absorbed-by-lava-tool |
Merge into: | lp:lava-dashboard-tool |
Diff against target: |
1230 lines (+4/-1176) 6 files modified
lava_dashboard_tool/__init__.py (+0/-23) lava_dashboard_tool/commands.py (+0/-981) lava_dashboard_tool/main.py (+0/-37) lava_dashboard_tool/tests/__init__.py (+0/-52) lava_dashboard_tool/tests/test_commands.py (+0/-44) setup.py (+4/-39) |
To merge this branch: | bzr merge lp:~terceiro/lava-dashboard-tool/absorbed-by-lava-tool |
Related bugs: | |
Related blueprints: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Senthil Kumaran S | Approve | ||
Review via email: mp+160193@code.launchpad.net |
Commit message
Description of the change
implementation of https:/
To post a comment you must log in.
Revision history for this message
Michael Hudson-Doyle (mwhudson) wrote : | # |
Revision history for this message
Antonio Terceiro (terceiro) wrote : | # |
On Tue, Apr 23, 2013 at 12:58:22AM -0000, Michael Hudson-Doyle wrote:
> Seems fine to me -- is this really needed though? Couldn't one just
> remove lava-dashboard-tool from the manifest?
People have them installed with pypi and APT, and we need to provide a
clean upgrade path so that they don't get two packages providing
lava_dashboard_tool Python module at the same time. For this we need to
make a last release of the deprecated packages with no code at all that
depends on the package/version that provides that code.
--
Antonio Terceiro
Software Engineer - Linaro
http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === removed directory 'lava_dashboard_tool' | |||
2 | === removed file 'lava_dashboard_tool/__init__.py' | |||
3 | --- lava_dashboard_tool/__init__.py 2012-03-22 18:38:32 +0000 | |||
4 | +++ lava_dashboard_tool/__init__.py 1970-01-01 00:00:00 +0000 | |||
5 | @@ -1,23 +0,0 @@ | |||
6 | 1 | # Copyright (C) 2010,2011 Linaro Limited | ||
7 | 2 | # | ||
8 | 3 | # Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org> | ||
9 | 4 | # | ||
10 | 5 | # This file is part of lava-dashboard-tool. | ||
11 | 6 | # | ||
12 | 7 | # lava-dashboard-tool is free software: you can redistribute it and/or modify | ||
13 | 8 | # it under the terms of the GNU Lesser General Public License version 3 | ||
14 | 9 | # as published by the Free Software Foundation | ||
15 | 10 | # | ||
16 | 11 | # lava-dashboard-tool is distributed in the hope that it will be useful, | ||
17 | 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
18 | 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
19 | 14 | # GNU General Public License for more details. | ||
20 | 15 | # | ||
21 | 16 | # You should have received a copy of the GNU Lesser General Public License | ||
22 | 17 | # along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>. | ||
23 | 18 | |||
24 | 19 | """ | ||
25 | 20 | Launch Control Tool package | ||
26 | 21 | """ | ||
27 | 22 | |||
28 | 23 | __version__ = (0, 8, 0, "dev", 0) | ||
29 | 24 | 0 | ||
30 | === removed file 'lava_dashboard_tool/commands.py' | |||
31 | --- lava_dashboard_tool/commands.py 2012-03-22 18:12:14 +0000 | |||
32 | +++ lava_dashboard_tool/commands.py 1970-01-01 00:00:00 +0000 | |||
33 | @@ -1,981 +0,0 @@ | |||
34 | 1 | # Copyright (C) 2010,2011 Linaro Limited | ||
35 | 2 | # | ||
36 | 3 | # Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org> | ||
37 | 4 | # | ||
38 | 5 | # This file is part of lava-dashboard-tool. | ||
39 | 6 | # | ||
40 | 7 | # lava-dashboard-tool is free software: you can redistribute it and/or modify | ||
41 | 8 | # it under the terms of the GNU Lesser General Public License version 3 | ||
42 | 9 | # as published by the Free Software Foundation | ||
43 | 10 | # | ||
44 | 11 | # lava-dashboard-tool is distributed in the hope that it will be useful, | ||
45 | 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
46 | 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
47 | 14 | # GNU General Public License for more details. | ||
48 | 15 | # | ||
49 | 16 | # You should have received a copy of the GNU Lesser General Public License | ||
50 | 17 | # along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>. | ||
51 | 18 | |||
52 | 19 | """ | ||
53 | 20 | Module with command-line tool commands that interact with the dashboard | ||
54 | 21 | server. All commands listed here should have counterparts in the | ||
55 | 22 | launch_control.dashboard_app.xml_rpc package. | ||
56 | 23 | """ | ||
57 | 24 | |||
58 | 25 | import argparse | ||
59 | 26 | import contextlib | ||
60 | 27 | import errno | ||
61 | 28 | import os | ||
62 | 29 | import re | ||
63 | 30 | import socket | ||
64 | 31 | import sys | ||
65 | 32 | import urllib | ||
66 | 33 | import urlparse | ||
67 | 34 | import xmlrpclib | ||
68 | 35 | |||
69 | 36 | import simplejson | ||
70 | 37 | from json_schema_validator.extensions import datetime_extension | ||
71 | 38 | |||
72 | 39 | from lava_tool.authtoken import AuthenticatingServerProxy, KeyringAuthBackend | ||
73 | 40 | from lava.tool.commands import ExperimentalCommandMixIn | ||
74 | 41 | from lava.tool.command import Command, CommandGroup | ||
75 | 42 | |||
76 | 43 | |||
77 | 44 | class dashboard(CommandGroup): | ||
78 | 45 | """ | ||
79 | 46 | Commands for interacting with LAVA Dashboard | ||
80 | 47 | """ | ||
81 | 48 | |||
82 | 49 | namespace = "lava.dashboard.commands" | ||
83 | 50 | |||
84 | 51 | |||
85 | 52 | class InsufficientServerVersion(Exception): | ||
86 | 53 | """ | ||
87 | 54 | Exception raised when server version that a command interacts with is too | ||
88 | 55 | old to support required features. | ||
89 | 56 | """ | ||
90 | 57 | def __init__(self, server_version, required_version): | ||
91 | 58 | self.server_version = server_version | ||
92 | 59 | self.required_version = required_version | ||
93 | 60 | |||
94 | 61 | |||
95 | 62 | class DataSetRenderer(object): | ||
96 | 63 | """ | ||
97 | 64 | Support class for rendering a table out of list of dictionaries. | ||
98 | 65 | |||
99 | 66 | It supports several features that make printing tabular data easier. | ||
100 | 67 | * Automatic layout | ||
101 | 68 | * Custom column headers | ||
102 | 69 | * Custom cell formatting | ||
103 | 70 | * Custom table captions | ||
104 | 71 | * Custom column ordering | ||
105 | 72 | * Custom Column separators | ||
106 | 73 | * Custom dataset notification | ||
107 | 74 | |||
108 | 75 | The primary method is render() which does all of the above. You | ||
109 | 76 | need to pass a dataset argument which is a list of dictionaries. | ||
110 | 77 | Each dictionary must have the same keys. In particular the first row | ||
111 | 78 | is used to determine columns. | ||
112 | 79 | """ | ||
113 | 80 | def __init__(self, column_map=None, row_formatter=None, empty=None, | ||
114 | 81 | order=None, caption=None, separator=" ", header_separator=None): | ||
115 | 82 | if column_map is None: | ||
116 | 83 | column_map = {} | ||
117 | 84 | if row_formatter is None: | ||
118 | 85 | row_formatter = {} | ||
119 | 86 | if empty is None: | ||
120 | 87 | empty = "There is no data to display" | ||
121 | 88 | self.column_map = column_map | ||
122 | 89 | self.row_formatter = row_formatter | ||
123 | 90 | self.empty = empty | ||
124 | 91 | self.order = order | ||
125 | 92 | self.separator = separator | ||
126 | 93 | self.caption = caption | ||
127 | 94 | self.header_separator = header_separator | ||
128 | 95 | |||
129 | 96 | def _analyze_dataset(self, dataset): | ||
130 | 97 | """ | ||
131 | 98 | Determine the columns that will be displayed and the maximum | ||
132 | 99 | length of each of those columns. | ||
133 | 100 | |||
134 | 101 | Returns a tuple (dataset, columms, maxlen) where columns is a | ||
135 | 102 | list of column names and maxlen is a dictionary mapping from | ||
136 | 103 | column name to maximum length of any value in the row or the | ||
137 | 104 | column header and the dataset is a copy of the dataset altered | ||
138 | 105 | as necessary. | ||
139 | 106 | |||
140 | 107 | Some examples: | ||
141 | 108 | |||
142 | 109 | First the dataset, an array of dictionaries | ||
143 | 110 | >>> dataset = [ | ||
144 | 111 | ... {'a': 'shorter', 'bee': ''}, | ||
145 | 112 | ... {'a': 'little longer', 'bee': 'b'}] | ||
146 | 113 | |||
147 | 114 | Note that column 'bee' is actually three characters long as the | ||
148 | 115 | column name made it wider. | ||
149 | 116 | >>> dataset_out, columns, maxlen = DataSetRenderer( | ||
150 | 117 | ... )._analyze_dataset(dataset) | ||
151 | 118 | |||
152 | 119 | Unless you format rows with a custom function the data is not altered. | ||
153 | 120 | >>> dataset_out is dataset | ||
154 | 121 | True | ||
155 | 122 | |||
156 | 123 | Columns come out in sorted alphabetic order | ||
157 | 124 | >>> columns | ||
158 | 125 | ['a', 'bee'] | ||
159 | 126 | |||
160 | 127 | Maximum length determines the width of each column. Note that | ||
161 | 128 | the header affects the column width. | ||
162 | 129 | >>> maxlen | ||
163 | 130 | {'a': 13, 'bee': 3} | ||
164 | 131 | |||
165 | 132 | You can constrain or reorder columns. In that case columns you | ||
166 | 133 | decided to ignore are simply left out of the output. | ||
167 | 134 | >>> dataset_out, columns, maxlen = DataSetRenderer( | ||
168 | 135 | ... order=['bee'])._analyze_dataset(dataset) | ||
169 | 136 | >>> columns | ||
170 | 137 | ['bee'] | ||
171 | 138 | >>> maxlen | ||
172 | 139 | {'bee': 3} | ||
173 | 140 | |||
174 | 141 | You can format values anyway you like: | ||
175 | 142 | >>> dataset_out, columns, maxlen = DataSetRenderer(row_formatter={ | ||
176 | 143 | ... 'bee': lambda value: "%10s" % value} | ||
177 | 144 | ... )._analyze_dataset(dataset) | ||
178 | 145 | |||
179 | 146 | Dataset is altered to take account of the row formatting | ||
180 | 147 | function. The original dataset argument is copied. | ||
181 | 148 | >>> dataset_out | ||
182 | 149 | [{'a': 'shorter', 'bee': ' '}, {'a': 'little longer', 'bee': ' b'}] | ||
183 | 150 | >>> dataset_out is not dataset | ||
184 | 151 | True | ||
185 | 152 | |||
186 | 153 | Columns stay the same though: | ||
187 | 154 | >>> columns | ||
188 | 155 | ['a', 'bee'] | ||
189 | 156 | |||
190 | 157 | Note how formatting altered the width of the column 'bee' | ||
191 | 158 | >>> maxlen | ||
192 | 159 | {'a': 13, 'bee': 10} | ||
193 | 160 | |||
194 | 161 | You can also format columns (with nice aliases).Note how | ||
195 | 162 | column 'bee' maximum width is now dominated by the long column | ||
196 | 163 | name: | ||
197 | 164 | >>> dataset_out, columns, maxlen = DataSetRenderer(column_map={ | ||
198 | 165 | ... 'bee': "Column B"})._analyze_dataset(dataset) | ||
199 | 166 | >>> maxlen | ||
200 | 167 | {'a': 13, 'bee': 8} | ||
201 | 168 | """ | ||
202 | 169 | if self.order: | ||
203 | 170 | columns = self.order | ||
204 | 171 | else: | ||
205 | 172 | columns = sorted(dataset[0].keys()) | ||
206 | 173 | if self.row_formatter: | ||
207 | 174 | dataset_out = [dict(row) for row in dataset] | ||
208 | 175 | else: | ||
209 | 176 | dataset_out = dataset | ||
210 | 177 | for row in dataset_out: | ||
211 | 178 | for column in row: | ||
212 | 179 | if column in self.row_formatter: | ||
213 | 180 | row[column] = self.row_formatter[column](row[column]) | ||
214 | 181 | maxlen = dict( | ||
215 | 182 | [(column, max( | ||
216 | 183 | len(self.column_map.get(column, column)), | ||
217 | 184 | max([ | ||
218 | 185 | len(str(row[column])) for row in dataset_out]))) | ||
219 | 186 | for column in columns]) | ||
220 | 187 | return dataset_out, columns, maxlen | ||
221 | 188 | |||
222 | 189 | def _render_header(self, dataset, columns, maxlen): | ||
223 | 190 | """ | ||
224 | 191 | Render a header, possibly with a caption string | ||
225 | 192 | |||
226 | 193 | Caption is controlled by the constructor. | ||
227 | 194 | >>> dataset = [ | ||
228 | 195 | ... {'a': 'shorter', 'bee': ''}, | ||
229 | 196 | ... {'a': 'little longer', 'bee': 'b'}] | ||
230 | 197 | >>> columns = ['a', 'bee'] | ||
231 | 198 | >>> maxlen = {'a': 13, 'bee': 3} | ||
232 | 199 | |||
233 | 200 | By default there is no caption, just column names: | ||
234 | 201 | >>> DataSetRenderer()._render_header( | ||
235 | 202 | ... dataset, columns, maxlen) | ||
236 | 203 | a bee | ||
237 | 204 | |||
238 | 205 | If you enable the header separator then column names will be visually | ||
239 | 206 | separated from the first row of data. | ||
240 | 207 | >>> DataSetRenderer(header_separator=True)._render_header( | ||
241 | 208 | ... dataset, columns, maxlen) | ||
242 | 209 | a bee | ||
243 | 210 | ----------------- | ||
244 | 211 | |||
245 | 212 | If you provide a caption it gets rendered as a centered | ||
246 | 213 | underlined text before the data: | ||
247 | 214 | >>> DataSetRenderer(caption="Dataset")._render_header( | ||
248 | 215 | ... dataset, columns, maxlen) | ||
249 | 216 | Dataset | ||
250 | 217 | ================= | ||
251 | 218 | a bee | ||
252 | 219 | |||
253 | 220 | You can use both caption and header separator | ||
254 | 221 | >>> DataSetRenderer(caption="Dataset", header_separator=True)._render_header( | ||
255 | 222 | ... dataset, columns, maxlen) | ||
256 | 223 | Dataset | ||
257 | 224 | ================= | ||
258 | 225 | a bee | ||
259 | 226 | ----------------- | ||
260 | 227 | |||
261 | 228 | Observe how the total length of the output horizontal line | ||
262 | 229 | depends on the separator! Also note the columns labels are | ||
263 | 230 | aligned to the center of the column | ||
264 | 231 | >>> DataSetRenderer(caption="Dataset", separator=" | ")._render_header( | ||
265 | 232 | ... dataset, columns, maxlen) | ||
266 | 233 | Dataset | ||
267 | 234 | =================== | ||
268 | 235 | a | bee | ||
269 | 236 | """ | ||
270 | 237 | total_len = sum(maxlen.itervalues()) | ||
271 | 238 | if len(columns): | ||
272 | 239 | total_len += len(self.separator) * (len(columns) - 1) | ||
273 | 240 | # Print the caption | ||
274 | 241 | if self.caption: | ||
275 | 242 | print "{0:^{1}}".format(self.caption, total_len) | ||
276 | 243 | print "=" * total_len | ||
277 | 244 | # Now print the coulum names | ||
278 | 245 | print self.separator.join([ | ||
279 | 246 | "{0:^{1}}".format(self.column_map.get(column, column), | ||
280 | 247 | maxlen[column]) for column in columns]) | ||
281 | 248 | # Finally print the header separator | ||
282 | 249 | if self.header_separator: | ||
283 | 250 | print "-" * total_len | ||
284 | 251 | |||
285 | 252 | def _render_rows(self, dataset, columns, maxlen): | ||
286 | 253 | """ | ||
287 | 254 | Render rows of the dataset. | ||
288 | 255 | |||
289 | 256 | Each row is printed on one line using the maxlen argument to | ||
290 | 257 | determine correct column size. Text is aligned left. | ||
291 | 258 | |||
292 | 259 | First the dataset, columns and maxlen as produced by | ||
293 | 260 | _analyze_dataset() | ||
294 | 261 | >>> dataset = [ | ||
295 | 262 | ... {'a': 'shorter', 'bee': ''}, | ||
296 | 263 | ... {'a': 'little longer', 'bee': 'b'}] | ||
297 | 264 | >>> columns = ['a', 'bee'] | ||
298 | 265 | >>> maxlen = {'a': 13, 'bee': 3} | ||
299 | 266 | |||
300 | 267 | Now a plain table. Note! To really understand this test | ||
301 | 268 | you should check out the length of the strings below. There | ||
302 | 269 | are two more spaces after 'b' in the second row | ||
303 | 270 | >>> DataSetRenderer()._render_rows(dataset, columns, maxlen) | ||
304 | 271 | shorter | ||
305 | 272 | little longer b | ||
306 | 273 | """ | ||
307 | 274 | for row in dataset: | ||
308 | 275 | print self.separator.join([ | ||
309 | 276 | "{0!s:{1}}".format(row[column], maxlen[column]) | ||
310 | 277 | for column in columns]) | ||
311 | 278 | |||
312 | 279 | def _render_dataset(self, dataset): | ||
313 | 280 | """ | ||
314 | 281 | Render the header followed by the rows of data. | ||
315 | 282 | """ | ||
316 | 283 | dataset, columns, maxlen = self._analyze_dataset(dataset) | ||
317 | 284 | self._render_header(dataset, columns, maxlen) | ||
318 | 285 | self._render_rows(dataset, columns, maxlen) | ||
319 | 286 | |||
320 | 287 | def _render_empty_dataset(self): | ||
321 | 288 | """ | ||
322 | 289 | Render empty dataset. | ||
323 | 290 | |||
324 | 291 | By default it just prints out a fixed sentence: | ||
325 | 292 | >>> DataSetRenderer()._render_empty_dataset() | ||
326 | 293 | There is no data to display | ||
327 | 294 | |||
328 | 295 | This can be changed by passing an argument to the constructor | ||
329 | 296 | >>> DataSetRenderer(empty="there is no data")._render_empty_dataset() | ||
330 | 297 | there is no data | ||
331 | 298 | """ | ||
332 | 299 | print self.empty | ||
333 | 300 | |||
334 | 301 | def render(self, dataset): | ||
335 | 302 | if len(dataset) > 0: | ||
336 | 303 | self._render_dataset(dataset) | ||
337 | 304 | else: | ||
338 | 305 | self._render_empty_dataset() | ||
339 | 306 | |||
340 | 307 | |||
341 | 308 | class XMLRPCCommand(Command): | ||
342 | 309 | """ | ||
343 | 310 | Abstract base class for commands that interact with dashboard server | ||
344 | 311 | over XML-RPC. | ||
345 | 312 | |||
346 | 313 | The only difference is that you should implement invoke_remote() | ||
347 | 314 | instead of invoke(). The provided implementation catches several | ||
348 | 315 | socket and XML-RPC errors and prints a pretty error message. | ||
349 | 316 | """ | ||
350 | 317 | |||
351 | 318 | @staticmethod | ||
352 | 319 | def _construct_xml_rpc_url(url): | ||
353 | 320 | """ | ||
354 | 321 | Construct URL to the XML-RPC service out of the given URL | ||
355 | 322 | """ | ||
356 | 323 | parts = urlparse.urlsplit(url) | ||
357 | 324 | if not parts.path.endswith("/RPC2/"): | ||
358 | 325 | path = parts.path.rstrip("/") + "/xml-rpc/" | ||
359 | 326 | else: | ||
360 | 327 | path = parts.path | ||
361 | 328 | return urlparse.urlunsplit( | ||
362 | 329 | (parts.scheme, parts.netloc, path, "", "")) | ||
363 | 330 | |||
364 | 331 | @staticmethod | ||
365 | 332 | def _strict_server_version(version): | ||
366 | 333 | """ | ||
367 | 334 | Calculate strict server version (as defined by | ||
368 | 335 | distutils.version.StrictVersion). This works by discarding .candidate | ||
369 | 336 | and .dev release-levels. | ||
370 | 337 | >>> XMLRPCCommand._strict_server_version("0.4.0.candidate.5") | ||
371 | 338 | '0.4.0' | ||
372 | 339 | >>> XMLRPCCommand._strict_server_version("0.4.0.dev.126") | ||
373 | 340 | '0.4.0' | ||
374 | 341 | >>> XMLRPCCommand._strict_server_version("0.4.0.alpha.1") | ||
375 | 342 | '0.4.0a1' | ||
376 | 343 | >>> XMLRPCCommand._strict_server_version("0.4.0.beta.2") | ||
377 | 344 | '0.4.0b2' | ||
378 | 345 | """ | ||
379 | 346 | try: | ||
380 | 347 | major, minor, micro, releaselevel, serial = version.split(".") | ||
381 | 348 | except ValueError: | ||
382 | 349 | raise ValueError( | ||
383 | 350 | ("version %r does not follow pattern " | ||
384 | 351 | "'major.minor.micro.releaselevel.serial'") % version) | ||
385 | 352 | if releaselevel in ["dev", "candidate", "final"]: | ||
386 | 353 | return "%s.%s.%s" % (major, minor, micro) | ||
387 | 354 | elif releaselevel == "alpha": | ||
388 | 355 | return "%s.%s.%sa%s" % (major, minor, micro, serial) | ||
389 | 356 | elif releaselevel == "beta": | ||
390 | 357 | return "%s.%s.%sb%s" % (major, minor, micro, serial) | ||
391 | 358 | else: | ||
392 | 359 | raise ValueError( | ||
393 | 360 | ("releaselevel %r is not one of 'final', 'alpha', 'beta', " | ||
394 | 361 | "'candidate' or 'final'") % releaselevel) | ||
395 | 362 | |||
396 | 363 | def _check_server_version(self, server_obj, required_version): | ||
397 | 364 | """ | ||
398 | 365 | Check that server object has is at least required_version. | ||
399 | 366 | |||
400 | 367 | This method may raise InsufficientServerVersion. | ||
401 | 368 | """ | ||
402 | 369 | from distutils.version import StrictVersion, LooseVersion | ||
403 | 370 | # For backwards compatibility the server reports | ||
404 | 371 | # major.minor.micro.releaselevel.serial which is not PEP-386 compliant | ||
405 | 372 | server_version = StrictVersion( | ||
406 | 373 | self._strict_server_version(server_obj.version())) | ||
407 | 374 | required_version = StrictVersion(required_version) | ||
408 | 375 | if server_version < required_version: | ||
409 | 376 | raise InsufficientServerVersion(server_version, required_version) | ||
410 | 377 | |||
411 | 378 | def __init__(self, parser, args): | ||
412 | 379 | super(XMLRPCCommand, self).__init__(parser, args) | ||
413 | 380 | xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url) | ||
414 | 381 | self.server = AuthenticatingServerProxy( | ||
415 | 382 | xml_rpc_url, | ||
416 | 383 | verbose=args.verbose_xml_rpc, | ||
417 | 384 | allow_none=True, | ||
418 | 385 | use_datetime=True, | ||
419 | 386 | auth_backend=KeyringAuthBackend()) | ||
420 | 387 | |||
421 | 388 | def use_non_legacy_api_if_possible(self, name='server'): | ||
422 | 389 | # Legacy APIs are registered in top-level object, non-legacy APIs are | ||
423 | 390 | # prefixed with extension name. | ||
424 | 391 | if "dashboard.version" in getattr(self, name).system.listMethods(): | ||
425 | 392 | setattr(self, name, getattr(self, name).dashboard) | ||
426 | 393 | |||
427 | 394 | @classmethod | ||
428 | 395 | def register_arguments(cls, parser): | ||
429 | 396 | dashboard_group = parser.add_argument_group("dashboard specific arguments") | ||
430 | 397 | default_dashboard_url = os.getenv("DASHBOARD_URL") | ||
431 | 398 | if default_dashboard_url: | ||
432 | 399 | dashboard_group.add_argument("--dashboard-url", | ||
433 | 400 | metavar="URL", help="URL of your validation dashboard (currently %(default)s)", | ||
434 | 401 | default=default_dashboard_url) | ||
435 | 402 | else: | ||
436 | 403 | dashboard_group.add_argument("--dashboard-url", required=True, | ||
437 | 404 | metavar="URL", help="URL of your validation dashboard") | ||
438 | 405 | debug_group = parser.add_argument_group("debugging arguments") | ||
439 | 406 | debug_group.add_argument("--verbose-xml-rpc", | ||
440 | 407 | action="store_true", default=False, | ||
441 | 408 | help="Show XML-RPC data") | ||
442 | 409 | return dashboard_group | ||
443 | 410 | |||
444 | 411 | @contextlib.contextmanager | ||
445 | 412 | def safety_net(self): | ||
446 | 413 | try: | ||
447 | 414 | yield | ||
448 | 415 | except socket.error as ex: | ||
449 | 416 | print >> sys.stderr, "Unable to connect to server at %s" % ( | ||
450 | 417 | self.args.dashboard_url,) | ||
451 | 418 | # It seems that some errors are reported as -errno | ||
452 | 419 | # while others as +errno. | ||
453 | 420 | ex.errno = abs(ex.errno) | ||
454 | 421 | if ex.errno == errno.ECONNREFUSED: | ||
455 | 422 | print >> sys.stderr, "Connection was refused." | ||
456 | 423 | parts = urlparse.urlsplit(self.args.dashboard_url) | ||
457 | 424 | if parts.netloc == "localhost:8000": | ||
458 | 425 | print >> sys.stderr, "Perhaps the server is not running?" | ||
459 | 426 | elif ex.errno == errno.ENOENT: | ||
460 | 427 | print >> sys.stderr, "Unable to resolve address" | ||
461 | 428 | else: | ||
462 | 429 | print >> sys.stderr, "Socket %d: %s" % (ex.errno, ex.strerror) | ||
463 | 430 | except xmlrpclib.ProtocolError as ex: | ||
464 | 431 | print >> sys.stderr, "Unable to exchange XML-RPC message with dashboard server" | ||
465 | 432 | print >> sys.stderr, "HTTP error code: %d/%s" % (ex.errcode, ex.errmsg) | ||
466 | 433 | except xmlrpclib.Fault as ex: | ||
467 | 434 | self.handle_xmlrpc_fault(ex.faultCode, ex.faultString) | ||
468 | 435 | except InsufficientServerVersion as ex: | ||
469 | 436 | print >> sys.stderr, ("This command requires at least server version " | ||
470 | 437 | "%s, actual server version is %s" % | ||
471 | 438 | (ex.required_version, ex.server_version)) | ||
472 | 439 | |||
473 | 440 | def invoke(self): | ||
474 | 441 | with self.safety_net(): | ||
475 | 442 | self.use_non_legacy_api_if_possible() | ||
476 | 443 | return self.invoke_remote() | ||
477 | 444 | |||
478 | 445 | def handle_xmlrpc_fault(self, faultCode, faultString): | ||
479 | 446 | if faultCode == 500: | ||
480 | 447 | print >> sys.stderr, "Dashboard server has experienced internal error" | ||
481 | 448 | print >> sys.stderr, faultString | ||
482 | 449 | else: | ||
483 | 450 | print >> sys.stderr, "XML-RPC error %d: %s" % (faultCode, faultString) | ||
484 | 451 | |||
485 | 452 | def invoke_remote(self): | ||
486 | 453 | raise NotImplementedError() | ||
487 | 454 | |||
488 | 455 | |||
489 | 456 | class server_version(XMLRPCCommand): | ||
490 | 457 | """ | ||
491 | 458 | Display dashboard server version | ||
492 | 459 | """ | ||
493 | 460 | |||
494 | 461 | def invoke_remote(self): | ||
495 | 462 | print "Dashboard server version: %s" % (self.server.version(),) | ||
496 | 463 | |||
497 | 464 | |||
498 | 465 | class put(XMLRPCCommand): | ||
499 | 466 | """ | ||
500 | 467 | Upload a bundle on the server | ||
501 | 468 | """ | ||
502 | 469 | |||
503 | 470 | @classmethod | ||
504 | 471 | def register_arguments(cls, parser): | ||
505 | 472 | super(put, cls).register_arguments(parser) | ||
506 | 473 | parser.add_argument("LOCAL", | ||
507 | 474 | type=argparse.FileType("rb"), | ||
508 | 475 | help="pathname on the local file system") | ||
509 | 476 | parser.add_argument("REMOTE", | ||
510 | 477 | default="/anonymous/", nargs='?', | ||
511 | 478 | help="pathname on the server") | ||
512 | 479 | |||
513 | 480 | def invoke_remote(self): | ||
514 | 481 | content = self.args.LOCAL.read() | ||
515 | 482 | filename = self.args.LOCAL.name | ||
516 | 483 | pathname = self.args.REMOTE | ||
517 | 484 | content_sha1 = self.server.put(content, filename, pathname) | ||
518 | 485 | print "Stored as bundle {0}".format(content_sha1) | ||
519 | 486 | |||
520 | 487 | def handle_xmlrpc_fault(self, faultCode, faultString): | ||
521 | 488 | if faultCode == 404: | ||
522 | 489 | print >> sys.stderr, "Bundle stream %s does not exist" % ( | ||
523 | 490 | self.args.REMOTE) | ||
524 | 491 | elif faultCode == 409: | ||
525 | 492 | print >> sys.stderr, "You have already uploaded this bundle to the dashboard" | ||
526 | 493 | else: | ||
527 | 494 | super(put, self).handle_xmlrpc_fault(faultCode, faultString) | ||
528 | 495 | |||
529 | 496 | |||
530 | 497 | class get(XMLRPCCommand): | ||
531 | 498 | """ | ||
532 | 499 | Download a bundle from the server | ||
533 | 500 | """ | ||
534 | 501 | |||
535 | 502 | @classmethod | ||
536 | 503 | def register_arguments(cls, parser): | ||
537 | 504 | super(get, cls).register_arguments(parser) | ||
538 | 505 | parser.add_argument("SHA1", | ||
539 | 506 | type=str, | ||
540 | 507 | help="SHA1 of the bundle to download") | ||
541 | 508 | parser.add_argument("--overwrite", | ||
542 | 509 | action="store_true", | ||
543 | 510 | help="Overwrite files on the local disk") | ||
544 | 511 | parser.add_argument("--output", "-o", | ||
545 | 512 | type=argparse.FileType("wb"), | ||
546 | 513 | default=None, | ||
547 | 514 | help="Alternate name of the output file") | ||
548 | 515 | |||
549 | 516 | def invoke_remote(self): | ||
550 | 517 | response = self.server.get(self.args.SHA1) | ||
551 | 518 | if self.args.output is None: | ||
552 | 519 | filename = self.args.SHA1 | ||
553 | 520 | if os.path.exists(filename) and not self.args.overwrite: | ||
554 | 521 | print >> sys.stderr, "File {filename!r} already exists".format( | ||
555 | 522 | filename=filename) | ||
556 | 523 | print >> sys.stderr, "You may pass --overwrite to write over it" | ||
557 | 524 | return -1 | ||
558 | 525 | stream = open(filename, "wb") | ||
559 | 526 | else: | ||
560 | 527 | stream = self.args.output | ||
561 | 528 | filename = self.args.output.name | ||
562 | 529 | stream.write(response['content']) | ||
563 | 530 | print "Downloaded bundle {0} to file {1!r}".format( | ||
564 | 531 | self.args.SHA1, filename) | ||
565 | 532 | |||
566 | 533 | def handle_xmlrpc_fault(self, faultCode, faultString): | ||
567 | 534 | if faultCode == 404: | ||
568 | 535 | print >> sys.stderr, "Bundle {sha1} does not exist".format( | ||
569 | 536 | sha1=self.args.SHA1) | ||
570 | 537 | else: | ||
571 | 538 | super(get, self).handle_xmlrpc_fault(faultCode, faultString) | ||
572 | 539 | |||
573 | 540 | |||
574 | 541 | class deserialize(XMLRPCCommand): | ||
575 | 542 | """ | ||
576 | 543 | Deserialize a bundle on the server | ||
577 | 544 | """ | ||
578 | 545 | |||
579 | 546 | @classmethod | ||
580 | 547 | def register_arguments(cls, parser): | ||
581 | 548 | super(deserialize, cls).register_arguments(parser) | ||
582 | 549 | parser.add_argument("SHA1", | ||
583 | 550 | type=str, | ||
584 | 551 | help="SHA1 of the bundle to deserialize") | ||
585 | 552 | |||
586 | 553 | def invoke_remote(self): | ||
587 | 554 | response = self.server.deserialize(self.args.SHA1) | ||
588 | 555 | print "Bundle {sha1} deserialized".format( | ||
589 | 556 | sha1=self.args.SHA1) | ||
590 | 557 | |||
591 | 558 | def handle_xmlrpc_fault(self, faultCode, faultString): | ||
592 | 559 | if faultCode == 404: | ||
593 | 560 | print >> sys.stderr, "Bundle {sha1} does not exist".format( | ||
594 | 561 | sha1=self.args.SHA1) | ||
595 | 562 | elif faultCode == 409: | ||
596 | 563 | print >> sys.stderr, "Unable to deserialize bundle {sha1}".format( | ||
597 | 564 | sha1=self.args.SHA1) | ||
598 | 565 | print >> sys.stderr, faultString | ||
599 | 566 | else: | ||
600 | 567 | super(deserialize, self).handle_xmlrpc_fault(faultCode, faultString) | ||
601 | 568 | |||
602 | 569 | |||
603 | 570 | def _get_pretty_renderer(**kwargs): | ||
604 | 571 | if "separator" not in kwargs: | ||
605 | 572 | kwargs["separator"] = " | " | ||
606 | 573 | if "header_separator" not in kwargs: | ||
607 | 574 | kwargs["header_separator"] = True | ||
608 | 575 | return DataSetRenderer(**kwargs) | ||
609 | 576 | |||
610 | 577 | |||
611 | 578 | class streams(XMLRPCCommand): | ||
612 | 579 | """ | ||
613 | 580 | Show streams you have access to | ||
614 | 581 | """ | ||
615 | 582 | |||
616 | 583 | renderer = _get_pretty_renderer( | ||
617 | 584 | order=('pathname', 'bundle_count', 'name'), | ||
618 | 585 | column_map={ | ||
619 | 586 | 'pathname': 'Pathname', | ||
620 | 587 | 'bundle_count': 'Number of bundles', | ||
621 | 588 | 'name': 'Name'}, | ||
622 | 589 | row_formatter={ | ||
623 | 590 | 'name': lambda name: name or "(not set)"}, | ||
624 | 591 | empty="There are no streams you can access on the server", | ||
625 | 592 | caption="Bundle streams") | ||
626 | 593 | |||
627 | 594 | def invoke_remote(self): | ||
628 | 595 | self.renderer.render(self.server.streams()) | ||
629 | 596 | |||
630 | 597 | |||
631 | 598 | class bundles(XMLRPCCommand): | ||
632 | 599 | """ | ||
633 | 600 | Show bundles in the specified stream | ||
634 | 601 | """ | ||
635 | 602 | |||
636 | 603 | renderer = _get_pretty_renderer( | ||
637 | 604 | column_map={ | ||
638 | 605 | 'uploaded_by': 'Uploader', | ||
639 | 606 | 'uploaded_on': 'Upload date', | ||
640 | 607 | 'content_filename': 'File name', | ||
641 | 608 | 'content_sha1': 'SHA1', | ||
642 | 609 | 'is_deserialized': "Deserialized?"}, | ||
643 | 610 | row_formatter={ | ||
644 | 611 | 'is_deserialized': lambda x: "yes" if x else "no", | ||
645 | 612 | 'uploaded_by': lambda x: x or "(anonymous)", | ||
646 | 613 | 'uploaded_on': lambda x: x.strftime("%Y-%m-%d %H:%M:%S")}, | ||
647 | 614 | order=('content_sha1', 'content_filename', 'uploaded_by', | ||
648 | 615 | 'uploaded_on', 'is_deserialized'), | ||
649 | 616 | empty="There are no bundles in this stream", | ||
650 | 617 | caption="Bundles", | ||
651 | 618 | separator=" | ") | ||
652 | 619 | |||
653 | 620 | @classmethod | ||
654 | 621 | def register_arguments(cls, parser): | ||
655 | 622 | super(bundles, cls).register_arguments(parser) | ||
656 | 623 | parser.add_argument("PATHNAME", | ||
657 | 624 | default="/anonymous/", nargs='?', | ||
658 | 625 | help="pathname on the server (defaults to %(default)s)") | ||
659 | 626 | |||
660 | 627 | def invoke_remote(self): | ||
661 | 628 | self.renderer.render(self.server.bundles(self.args.PATHNAME)) | ||
662 | 629 | |||
663 | 630 | def handle_xmlrpc_fault(self, faultCode, faultString): | ||
664 | 631 | if faultCode == 404: | ||
665 | 632 | print >> sys.stderr, "Bundle stream %s does not exist" % ( | ||
666 | 633 | self.args.PATHNAME) | ||
667 | 634 | else: | ||
668 | 635 | super(bundles, self).handle_xmlrpc_fault(faultCode, faultString) | ||
669 | 636 | |||
670 | 637 | |||
671 | 638 | class make_stream(XMLRPCCommand): | ||
672 | 639 | """ | ||
673 | 640 | Create a bundle stream on the server | ||
674 | 641 | """ | ||
675 | 642 | |||
676 | 643 | @classmethod | ||
677 | 644 | def register_arguments(cls, parser): | ||
678 | 645 | super(make_stream, cls).register_arguments(parser) | ||
679 | 646 | parser.add_argument( | ||
680 | 647 | "pathname", | ||
681 | 648 | type=str, | ||
682 | 649 | help="Pathname of the bundle stream to create") | ||
683 | 650 | parser.add_argument( | ||
684 | 651 | "--name", | ||
685 | 652 | type=str, | ||
686 | 653 | default="", | ||
687 | 654 | help="Name of the bundle stream (description)") | ||
688 | 655 | |||
689 | 656 | def invoke_remote(self): | ||
690 | 657 | self._check_server_version(self.server, "0.3") | ||
691 | 658 | pathname = self.server.make_stream(self.args.pathname, self.args.name) | ||
692 | 659 | print "Bundle stream {pathname} created".format(pathname=pathname) | ||
693 | 660 | |||
694 | 661 | |||
695 | 662 | class backup(XMLRPCCommand): | ||
696 | 663 | """ | ||
697 | 664 | Backup data uploaded to a dashboard instance. | ||
698 | 665 | |||
699 | 666 | 0 | ||
700 | 667 | Not all data is preserved. The following data is lost: identity of the user | ||
701 | 668 | that uploaded each bundle, time of uploading and deserialization on the | ||
702 | 669 | server, name of the bundle stream that contained the data | ||
703 | 670 | """ | ||
704 | 671 | |||
705 | 672 | @classmethod | ||
706 | 673 | def register_arguments(cls, parser): | ||
707 | 674 | super(backup, cls).register_arguments(parser) | ||
708 | 675 | parser.add_argument("BACKUP_DIR", type=str, | ||
709 | 676 | help="Directory to backup to") | ||
710 | 677 | |||
711 | 678 | def invoke_remote(self): | ||
712 | 679 | if not os.path.exists(self.args.BACKUP_DIR): | ||
713 | 680 | os.mkdir(self.args.BACKUP_DIR) | ||
714 | 681 | for bundle_stream in self.server.streams(): | ||
715 | 682 | print "Processing stream %s" % bundle_stream["pathname"] | ||
716 | 683 | bundle_stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(bundle_stream["pathname"])) | ||
717 | 684 | if not os.path.exists(bundle_stream_dir): | ||
718 | 685 | os.mkdir(bundle_stream_dir) | ||
719 | 686 | with open(os.path.join(bundle_stream_dir, "metadata.json"), "wt") as stream: | ||
720 | 687 | simplejson.dump({ | ||
721 | 688 | "pathname": bundle_stream["pathname"], | ||
722 | 689 | "name": bundle_stream["name"], | ||
723 | 690 | "user": bundle_stream["user"], | ||
724 | 691 | "group": bundle_stream["group"], | ||
725 | 692 | }, stream) | ||
726 | 693 | for bundle in self.server.bundles(bundle_stream["pathname"]): | ||
727 | 694 | print " * Backing up bundle %s" % bundle["content_sha1"] | ||
728 | 695 | data = self.server.get(bundle["content_sha1"]) | ||
729 | 696 | bundle_pathname = os.path.join(bundle_stream_dir, bundle["content_sha1"]) | ||
730 | 697 | # Note: we write bundles as binary data to preserve anything the user might have dumped on us | ||
731 | 698 | with open(bundle_pathname + ".json", "wb") as stream: | ||
732 | 699 | stream.write(data["content"]) | ||
733 | 700 | with open(bundle_pathname + ".metadata.json", "wt") as stream: | ||
734 | 701 | simplejson.dump({ | ||
735 | 702 | "uploaded_by": bundle["uploaded_by"], | ||
736 | 703 | "uploaded_on": datetime_extension.to_json(bundle["uploaded_on"]), | ||
737 | 704 | "content_filename": bundle["content_filename"], | ||
738 | 705 | "content_sha1": bundle["content_sha1"], | ||
739 | 706 | "content_size": bundle["content_size"], | ||
740 | 707 | }, stream) | ||
741 | 708 | |||
742 | 709 | |||
743 | 710 | class restore(XMLRPCCommand): | ||
744 | 711 | """ | ||
745 | 712 | Restore a dashboard instance from backup | ||
746 | 713 | """ | ||
747 | 714 | |||
748 | 715 | @classmethod | ||
749 | 716 | def register_arguments(cls, parser): | ||
750 | 717 | super(restore, cls).register_arguments(parser) | ||
751 | 718 | parser.add_argument("BACKUP_DIR", type=str, | ||
752 | 719 | help="Directory to backup from") | ||
753 | 720 | |||
754 | 721 | def invoke_remote(self): | ||
755 | 722 | self._check_server_version(self.server, "0.3") | ||
756 | 723 | for stream_pathname_quoted in os.listdir(self.args.BACKUP_DIR): | ||
757 | 724 | filesystem_stream_pathname = os.path.join(self.args.BACKUP_DIR, stream_pathname_quoted) | ||
758 | 725 | if not os.path.isdir(filesystem_stream_pathname): | ||
759 | 726 | continue | ||
760 | 727 | stream_pathname = urllib.unquote(stream_pathname_quoted) | ||
761 | 728 | if os.path.exists(os.path.join(filesystem_stream_pathname, "metadata.json")): | ||
762 | 729 | with open(os.path.join(filesystem_stream_pathname, "metadata.json"), "rt") as stream: | ||
763 | 730 | stream_metadata = simplejson.load(stream) | ||
764 | 731 | else: | ||
765 | 732 | stream_metadata = {} | ||
766 | 733 | print "Processing stream %s" % stream_pathname | ||
767 | 734 | try: | ||
768 | 735 | self.server.make_stream(stream_pathname, stream_metadata.get("name", "Restored from backup")) | ||
769 | 736 | except xmlrpclib.Fault as ex: | ||
770 | 737 | if ex.faultCode != 409: | ||
771 | 738 | raise | ||
772 | 739 | for content_sha1 in [item[:-len(".json")] for item in os.listdir(filesystem_stream_pathname) if item.endswith(".json") and not item.endswith(".metadata.json") and item != "metadata.json"]: | ||
773 | 740 | filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_sha1 + ".json") | ||
774 | 741 | if not os.path.isfile(filesystem_content_filename): | ||
775 | 742 | continue | ||
776 | 743 | with open(os.path.join(filesystem_stream_pathname, content_sha1) + ".metadata.json", "rt") as stream: | ||
777 | 744 | bundle_metadata = simplejson.load(stream) | ||
778 | 745 | with open(filesystem_content_filename, "rb") as stream: | ||
779 | 746 | content = stream.read() | ||
780 | 747 | print " * Restoring bundle %s" % content_sha1 | ||
781 | 748 | try: | ||
782 | 749 | self.server.put(content, bundle_metadata["content_filename"], stream_pathname) | ||
783 | 750 | except xmlrpclib.Fault as ex: | ||
784 | 751 | if ex.faultCode != 409: | ||
785 | 752 | raise | ||
786 | 753 | |||
787 | 754 | |||
788 | 755 | class pull(ExperimentalCommandMixIn, XMLRPCCommand): | ||
789 | 756 | """ | ||
790 | 757 | Copy bundles and bundle streams from one dashboard to another. | ||
791 | 758 | |||
792 | 759 | 1 | ||
793 | 760 | This command checks for two environment varialbes: | ||
794 | 761 | The value of DASHBOARD_URL is used as a replacement for --dashbard-url. | ||
795 | 762 | The value of REMOTE_DASHBOARD_URL as a replacement for FROM. | ||
796 | 763 | Their presence automatically makes the corresponding argument optional. | ||
797 | 764 | """ | ||
798 | 765 | |||
799 | 766 | def __init__(self, parser, args): | ||
800 | 767 | super(pull, self).__init__(parser, args) | ||
801 | 768 | remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM) | ||
802 | 769 | self.remote_server = AuthenticatingServerProxy( | ||
803 | 770 | remote_xml_rpc_url, | ||
804 | 771 | verbose=args.verbose_xml_rpc, | ||
805 | 772 | use_datetime=True, | ||
806 | 773 | allow_none=True, | ||
807 | 774 | auth_backend=KeyringAuthBackend()) | ||
808 | 775 | self.use_non_legacy_api_if_possible('remote_server') | ||
809 | 776 | |||
810 | 777 | @classmethod | ||
811 | 778 | def register_arguments(cls, parser): | ||
812 | 779 | group = super(pull, cls).register_arguments(parser) | ||
813 | 780 | default_remote_dashboard_url = os.getenv("REMOTE_DASHBOARD_URL") | ||
814 | 781 | if default_remote_dashboard_url: | ||
815 | 782 | group.add_argument( | ||
816 | 783 | "FROM", nargs="?", | ||
817 | 784 | help="URL of the remote validation dashboard (currently %(default)s)", | ||
818 | 785 | default=default_remote_dashboard_url) | ||
819 | 786 | else: | ||
820 | 787 | group.add_argument( | ||
821 | 788 | "FROM", | ||
822 | 789 | help="URL of the remote validation dashboard)") | ||
823 | 790 | group.add_argument("STREAM", nargs="*", help="Streams to pull from (all by default)") | ||
824 | 791 | |||
825 | 792 | @staticmethod | ||
826 | 793 | def _filesizeformat(num_bytes): | ||
827 | 794 | """ | ||
828 | 795 | Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB, | ||
829 | 796 | 102 num_bytes, etc). | ||
830 | 797 | """ | ||
831 | 798 | try: | ||
832 | 799 | num_bytes = float(num_bytes) | ||
833 | 800 | except (TypeError, ValueError, UnicodeDecodeError): | ||
834 | 801 | return "%(size)d byte", "%(size)d num_bytes" % {'size': 0} | ||
835 | 802 | |||
836 | 803 | filesize_number_format = lambda value: "%0.2f" % (round(value, 1),) | ||
837 | 804 | |||
838 | 805 | if num_bytes < 1024: | ||
839 | 806 | return "%(size)d bytes" % {'size': num_bytes} | ||
840 | 807 | if num_bytes < 1024 * 1024: | ||
841 | 808 | return "%s KB" % filesize_number_format(num_bytes / 1024) | ||
842 | 809 | if num_bytes < 1024 * 1024 * 1024: | ||
843 | 810 | return "%s MB" % filesize_number_format(num_bytes / (1024 * 1024)) | ||
844 | 811 | return "%s GB" % filesize_number_format(num_bytes / (1024 * 1024 * 1024)) | ||
845 | 812 | |||
846 | 813 | def invoke_remote(self): | ||
847 | 814 | self._check_server_version(self.server, "0.3") | ||
848 | 815 | |||
849 | 816 | print "Checking local and remote streams" | ||
850 | 817 | remote = self.remote_server.streams() | ||
851 | 818 | if self.args.STREAM: | ||
852 | 819 | # Check that all requested streams are available remotely | ||
853 | 820 | requested_set = frozenset(self.args.STREAM) | ||
854 | 821 | remote_set = frozenset((stream["pathname"] for stream in remote)) | ||
855 | 822 | unavailable_set = requested_set - remote_set | ||
856 | 823 | if unavailable_set: | ||
857 | 824 | print >> sys.stderr, "Remote stream not found: %s" % ", ".join(unavailable_set) | ||
858 | 825 | return -1 | ||
859 | 826 | # Limit to requested streams if necessary | ||
860 | 827 | remote = [stream for stream in remote if stream["pathname"] in requested_set] | ||
861 | 828 | local = self.server.streams() | ||
862 | 829 | missing_pathnames = set([stream["pathname"] for stream in remote]) - set([stream["pathname"] for stream in local]) | ||
863 | 830 | for stream in remote: | ||
864 | 831 | if stream["pathname"] in missing_pathnames: | ||
865 | 832 | self.server.make_stream(stream["pathname"], stream["name"]) | ||
866 | 833 | local_bundles = [] | ||
867 | 834 | else: | ||
868 | 835 | local_bundles = [bundle for bundle in self.server.bundles(stream["pathname"])] | ||
869 | 836 | remote_bundles = [bundle for bundle in self.remote_server.bundles(stream["pathname"])] | ||
870 | 837 | missing_bundles = set((bundle["content_sha1"] for bundle in remote_bundles)) | ||
871 | 838 | missing_bundles -= set((bundle["content_sha1"] for bundle in local_bundles)) | ||
872 | 839 | try: | ||
873 | 840 | missing_bytes = sum( | ||
874 | 841 | (bundle["content_size"] | ||
875 | 842 | for bundle in remote_bundles | ||
876 | 843 | if bundle["content_sha1"] in missing_bundles)) | ||
877 | 844 | except KeyError as ex: | ||
878 | 845 | # Older servers did not return content_size so this part is optional | ||
879 | 846 | missing_bytes = None | ||
880 | 847 | if missing_bytes: | ||
881 | 848 | print "Stream %s needs update (%s)" % (stream["pathname"], self._filesizeformat(missing_bytes)) | ||
882 | 849 | elif missing_bundles: | ||
883 | 850 | print "Stream %s needs update (no estimate available)" % (stream["pathname"],) | ||
884 | 851 | else: | ||
885 | 852 | print "Stream %s is up to date" % (stream["pathname"],) | ||
886 | 853 | for content_sha1 in missing_bundles: | ||
887 | 854 | print "Getting %s" % (content_sha1,), | ||
888 | 855 | sys.stdout.flush() | ||
889 | 856 | data = self.remote_server.get(content_sha1) | ||
890 | 857 | print "got %s, storing" % (self._filesizeformat(len(data["content"]))), | ||
891 | 858 | sys.stdout.flush() | ||
892 | 859 | try: | ||
893 | 860 | self.server.put(data["content"], data["content_filename"], stream["pathname"]) | ||
894 | 861 | except xmlrpclib.Fault as ex: | ||
895 | 862 | if ex.faultCode == 409: # duplicate | ||
896 | 863 | print "already present (in another stream)" | ||
897 | 864 | else: | ||
898 | 865 | raise | ||
899 | 866 | else: | ||
900 | 867 | print "done" | ||
901 | 868 | |||
902 | 869 | |||
903 | 870 | class data_views(ExperimentalCommandMixIn, XMLRPCCommand): | ||
904 | 871 | """ | ||
905 | 872 | Show data views defined on the server | ||
906 | 873 | """ | ||
907 | 874 | renderer = _get_pretty_renderer( | ||
908 | 875 | column_map={ | ||
909 | 876 | 'name': 'Name', | ||
910 | 877 | 'summary': 'Summary', | ||
911 | 878 | }, | ||
912 | 879 | order=('name', 'summary'), | ||
913 | 880 | empty="There are no data views defined yet", | ||
914 | 881 | caption="Data Views") | ||
915 | 882 | |||
916 | 883 | def invoke_remote(self): | ||
917 | 884 | self._check_server_version(self.server, "0.4") | ||
918 | 885 | self.renderer.render(self.server.data_views()) | ||
919 | 886 | |||
920 | 887 | print "Tip: to invoke a data view try `lc-tool query-data-view`" | ||
921 | 888 | |||
922 | 889 | |||
923 | 890 | class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand): | ||
924 | 891 | """ | ||
925 | 892 | Invoke a specified data view | ||
926 | 893 | """ | ||
927 | 894 | @classmethod | ||
928 | 895 | def register_arguments(cls, parser): | ||
929 | 896 | super(query_data_view, cls).register_arguments(parser) | ||
930 | 897 | parser.add_argument("QUERY", metavar="QUERY", nargs="...", | ||
931 | 898 | help="Data view name and any optional and required arguments") | ||
932 | 899 | |||
933 | 900 | def _probe_data_views(self): | ||
934 | 901 | """ | ||
935 | 902 | Probe the server for information about data views | ||
936 | 903 | """ | ||
937 | 904 | with self.safety_net(): | ||
938 | 905 | self.use_non_legacy_api_if_possible() | ||
939 | 906 | self._check_server_version(self.server, "0.4") | ||
940 | 907 | return self.server.data_views() | ||
941 | 908 | |||
942 | 909 | def reparse_arguments(self, parser, raw_args): | ||
943 | 910 | self.data_views = self._probe_data_views() | ||
944 | 911 | if self.data_views is None: | ||
945 | 912 | return | ||
946 | 913 | # Here we hack a little, the last actuin is the QUERY action added | ||
947 | 914 | # in register_arguments above. By removing it we make the output | ||
948 | 915 | # of lc-tool query-data-view NAME --help more consistent. | ||
949 | 916 | del parser._actions[-1] | ||
950 | 917 | subparsers = parser.add_subparsers( | ||
951 | 918 | title="Data views available on the server") | ||
952 | 919 | for data_view in self.data_views: | ||
953 | 920 | data_view_parser = subparsers.add_parser( | ||
954 | 921 | data_view["name"], | ||
955 | 922 | help=data_view["summary"], | ||
956 | 923 | epilog=data_view["documentation"]) | ||
957 | 924 | data_view_parser.set_defaults(data_view=data_view) | ||
958 | 925 | group = data_view_parser.add_argument_group("Data view parameters") | ||
959 | 926 | for argument in data_view["arguments"]: | ||
960 | 927 | if argument["default"] is None: | ||
961 | 928 | group.add_argument( | ||
962 | 929 | "--{name}".format(name=argument["name"].replace("_", "-")), | ||
963 | 930 | dest=argument["name"], | ||
964 | 931 | help=argument["help"], | ||
965 | 932 | type=str, | ||
966 | 933 | required=True) | ||
967 | 934 | else: | ||
968 | 935 | group.add_argument( | ||
969 | 936 | "--{name}".format(name=argument["name"].replace("_", "-")), | ||
970 | 937 | dest=argument["name"], | ||
971 | 938 | help=argument["help"], | ||
972 | 939 | type=str, | ||
973 | 940 | default=argument["default"]) | ||
974 | 941 | self.args = self.parser.parse_args(raw_args) | ||
975 | 942 | |||
976 | 943 | def invoke(self): | ||
977 | 944 | # Override and _not_ call 'use_non_legacy_api_if_possible' as we | ||
978 | 945 | # already did this reparse_arguments | ||
979 | 946 | with self.safety_net(): | ||
980 | 947 | return self.invoke_remote() | ||
981 | 948 | |||
982 | 949 | def invoke_remote(self): | ||
983 | 950 | if self.data_views is None: | ||
984 | 951 | return -1 | ||
985 | 952 | self._check_server_version(self.server, "0.4") | ||
986 | 953 | # Build a collection of arguments for data view | ||
987 | 954 | data_view_args = {} | ||
988 | 955 | for argument in self.args.data_view["arguments"]: | ||
989 | 956 | arg_name = argument["name"] | ||
990 | 957 | if arg_name in self.args: | ||
991 | 958 | data_view_args[arg_name] = getattr(self.args, arg_name) | ||
992 | 959 | # Invoke the data view | ||
993 | 960 | response = self.server.query_data_view(self.args.data_view["name"], data_view_args) | ||
994 | 961 | # Create a pretty-printer | ||
995 | 962 | renderer = _get_pretty_renderer( | ||
996 | 963 | caption=self.args.data_view["summary"], | ||
997 | 964 | order=[item["name"] for item in response["columns"]]) | ||
998 | 965 | # Post-process the data so that it fits the printer | ||
999 | 966 | data_for_renderer = [ | ||
1000 | 967 | dict(zip( | ||
1001 | 968 | [column["name"] for column in response["columns"]], | ||
1002 | 969 | row)) | ||
1003 | 970 | for row in response["rows"]] | ||
1004 | 971 | # Print the data | ||
1005 | 972 | renderer.render(data_for_renderer) | ||
1006 | 973 | |||
1007 | 974 | |||
1008 | 975 | class version(Command): | ||
1009 | 976 | """ | ||
1010 | 977 | Show dashboard client version | ||
1011 | 978 | """ | ||
1012 | 979 | def invoke(self): | ||
1013 | 980 | import versiontools | ||
1014 | 981 | from lava_dashboard_tool import __version__ | ||
1015 | 982 | print "Dashboard client version: {version}".format( | ||
1016 | 983 | version=versiontools.format_version(__version__)) | ||
1017 | 984 | 2 | ||
1018 | === removed file 'lava_dashboard_tool/main.py' | |||
1019 | --- lava_dashboard_tool/main.py 2011-06-23 11:23:24 +0000 | |||
1020 | +++ lava_dashboard_tool/main.py 1970-01-01 00:00:00 +0000 | |||
1021 | @@ -1,37 +0,0 @@ | |||
1022 | 1 | # Copyright (C) 2011 Linaro Limited | ||
1023 | 2 | # | ||
1024 | 3 | # Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org> | ||
1025 | 4 | # Author: Michael Hudson-Doyle <michael.hudson@linaro.org> | ||
1026 | 5 | # | ||
1027 | 6 | # This file is part of lava-dashboard-tool. | ||
1028 | 7 | # | ||
1029 | 8 | # lava-dashboard-tool is free software: you can redistribute it and/or modify | ||
1030 | 9 | # it under the terms of the GNU Lesser General Public License version 3 | ||
1031 | 10 | # as published by the Free Software Foundation | ||
1032 | 11 | # | ||
1033 | 12 | # lava-dashboard-tool is distributed in the hope that it will be useful, | ||
1034 | 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
1035 | 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
1036 | 15 | # GNU General Public License for more details. | ||
1037 | 16 | # | ||
1038 | 17 | # You should have received a copy of the GNU Lesser General Public License | ||
1039 | 18 | # along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>. | ||
1040 | 19 | |||
1041 | 20 | |||
1042 | 21 | from lava_tool.dispatcher import LavaDispatcher, run_with_dispatcher_class | ||
1043 | 22 | |||
1044 | 23 | |||
1045 | 24 | class LaunchControlDispatcher(LavaDispatcher): | ||
1046 | 25 | |||
1047 | 26 | toolname = 'lava_dashboard_tool' | ||
1048 | 27 | description = """ | ||
1049 | 28 | Command line tool for interacting with Launch Control | ||
1050 | 29 | """ | ||
1051 | 30 | epilog = """ | ||
1052 | 31 | Please report all bugs using the Launchpad bug tracker: | ||
1053 | 32 | http://bugs.launchpad.net/lava-dashboard-tool/+filebug | ||
1054 | 33 | """ | ||
1055 | 34 | |||
1056 | 35 | |||
1057 | 36 | def main(): | ||
1058 | 37 | run_with_dispatcher_class(LaunchControlDispatcher) | ||
1059 | 38 | 0 | ||
1060 | === removed directory 'lava_dashboard_tool/tests' | |||
1061 | === removed file 'lava_dashboard_tool/tests/__init__.py' | |||
1062 | --- lava_dashboard_tool/tests/__init__.py 2011-06-23 11:23:24 +0000 | |||
1063 | +++ lava_dashboard_tool/tests/__init__.py 1970-01-01 00:00:00 +0000 | |||
1064 | @@ -1,52 +0,0 @@ | |||
1065 | 1 | # Copyright (C) 2010,2011 Linaro Limited | ||
1066 | 2 | # | ||
1067 | 3 | # Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org> | ||
1068 | 4 | # | ||
1069 | 5 | # This file is part of lava-dashboard-tool. | ||
1070 | 6 | # | ||
1071 | 7 | # lava-dashboard-tool is free software: you can redistribute it and/or modify | ||
1072 | 8 | # it under the terms of the GNU Lesser General Public License version 3 | ||
1073 | 9 | # as published by the Free Software Foundation | ||
1074 | 10 | # | ||
1075 | 11 | # lava-dashboard-tool is distributed in the hope that it will be useful, | ||
1076 | 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
1077 | 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
1078 | 14 | # GNU General Public License for more details. | ||
1079 | 15 | # | ||
1080 | 16 | # You should have received a copy of the GNU Lesser General Public License | ||
1081 | 17 | # along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>. | ||
1082 | 18 | |||
1083 | 19 | """ | ||
1084 | 20 | Package with unit tests for lava_dashboard_tool | ||
1085 | 21 | """ | ||
1086 | 22 | |||
1087 | 23 | import doctest | ||
1088 | 24 | import unittest | ||
1089 | 25 | |||
1090 | 26 | |||
1091 | 27 | def app_modules(): | ||
1092 | 28 | return [ | ||
1093 | 29 | 'lava_dashboard_tool.commands', | ||
1094 | 30 | ] | ||
1095 | 31 | |||
1096 | 32 | |||
1097 | 33 | def test_modules(): | ||
1098 | 34 | return [ | ||
1099 | 35 | 'lava_dashboard_tool.tests.test_commands', | ||
1100 | 36 | ] | ||
1101 | 37 | |||
1102 | 38 | |||
1103 | 39 | def test_suite(): | ||
1104 | 40 | """ | ||
1105 | 41 | Build an unittest.TestSuite() object with all the tests in _modules. | ||
1106 | 42 | Each module is harvested for both regular unittests and doctests | ||
1107 | 43 | """ | ||
1108 | 44 | modules = app_modules() + test_modules() | ||
1109 | 45 | suite = unittest.TestSuite() | ||
1110 | 46 | loader = unittest.TestLoader() | ||
1111 | 47 | for name in modules: | ||
1112 | 48 | unit_suite = loader.loadTestsFromName(name) | ||
1113 | 49 | suite.addTests(unit_suite) | ||
1114 | 50 | doc_suite = doctest.DocTestSuite(name) | ||
1115 | 51 | suite.addTests(doc_suite) | ||
1116 | 52 | return suite | ||
1117 | 53 | 0 | ||
1118 | === removed file 'lava_dashboard_tool/tests/test_commands.py' | |||
1119 | --- lava_dashboard_tool/tests/test_commands.py 2011-06-23 11:23:24 +0000 | |||
1120 | +++ lava_dashboard_tool/tests/test_commands.py 1970-01-01 00:00:00 +0000 | |||
1121 | @@ -1,44 +0,0 @@ | |||
1122 | 1 | # Copyright (C) 2010,2011 Linaro Limited | ||
1123 | 2 | # | ||
1124 | 3 | # Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org> | ||
1125 | 4 | # | ||
1126 | 5 | # This file is part of lava-dashboard-tool. | ||
1127 | 6 | # | ||
1128 | 7 | # lava-dashboard-tool is free software: you can redistribute it and/or modify | ||
1129 | 8 | # it under the terms of the GNU Lesser General Public License version 3 | ||
1130 | 9 | # as published by the Free Software Foundation | ||
1131 | 10 | # | ||
1132 | 11 | # lava-dashboard-tool is distributed in the hope that it will be useful, | ||
1133 | 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
1134 | 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
1135 | 14 | # GNU General Public License for more details. | ||
1136 | 15 | # | ||
1137 | 16 | # You should have received a copy of the GNU Lesser General Public License | ||
1138 | 17 | # along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>. | ||
1139 | 18 | |||
1140 | 19 | """ | ||
1141 | 20 | Unit tests for the launch_control.commands package | ||
1142 | 21 | """ | ||
1143 | 22 | |||
1144 | 23 | from unittest import TestCase | ||
1145 | 24 | |||
1146 | 25 | from lava_dashboard_tool.commands import XMLRPCCommand | ||
1147 | 26 | |||
1148 | 27 | |||
1149 | 28 | class XMLRPCCommandTestCase(TestCase): | ||
1150 | 29 | |||
1151 | 30 | def test_construct_xml_rpc_url_preserves_path(self): | ||
1152 | 31 | self.assertEqual( | ||
1153 | 32 | XMLRPCCommand._construct_xml_rpc_url("http://domain/path"), | ||
1154 | 33 | "http://domain/path/xml-rpc/") | ||
1155 | 34 | self.assertEqual( | ||
1156 | 35 | XMLRPCCommand._construct_xml_rpc_url("http://domain/path/"), | ||
1157 | 36 | "http://domain/path/xml-rpc/") | ||
1158 | 37 | |||
1159 | 38 | def test_construct_xml_rpc_url_adds_proper_suffix(self): | ||
1160 | 39 | self.assertEqual( | ||
1161 | 40 | XMLRPCCommand._construct_xml_rpc_url("http://domain/"), | ||
1162 | 41 | "http://domain/xml-rpc/") | ||
1163 | 42 | self.assertEqual( | ||
1164 | 43 | XMLRPCCommand._construct_xml_rpc_url("http://domain"), | ||
1165 | 44 | "http://domain/xml-rpc/") | ||
1166 | 45 | 0 | ||
1167 | === modified file 'setup.py' | |||
1168 | --- setup.py 2012-03-22 18:12:14 +0000 | |||
1169 | +++ setup.py 2013-04-22 19:30:37 +0000 | |||
1170 | @@ -23,48 +23,14 @@ | |||
1171 | 23 | 23 | ||
1172 | 24 | setup( | 24 | setup( |
1173 | 25 | name='lava-dashboard-tool', | 25 | name='lava-dashboard-tool', |
1175 | 26 | version=":versiontools:lava_dashboard_tool:__version__", | 26 | version="0.8", |
1176 | 27 | author="Zygmunt Krynicki", | 27 | author="Zygmunt Krynicki", |
1177 | 28 | author_email="zygmunt.krynicki@linaro.org", | 28 | author_email="zygmunt.krynicki@linaro.org", |
1178 | 29 | packages=find_packages(), | 29 | packages=find_packages(), |
1180 | 30 | description="Command line utility for Launch Control", | 30 | description="Command line utility for Launch Control (deprecated)", |
1181 | 31 | url='https://launchpad.net/lava-dashboard-tool', | 31 | url='https://launchpad.net/lava-dashboard-tool', |
1182 | 32 | test_suite='lava_dashboard_tool.tests.test_suite', | 32 | test_suite='lava_dashboard_tool.tests.test_suite', |
1183 | 33 | license="LGPLv3", | 33 | license="LGPLv3", |
1184 | 34 | entry_points=""" | ||
1185 | 35 | [console_scripts] | ||
1186 | 36 | lava-dashboard-tool=lava_dashboard_tool.main:main | ||
1187 | 37 | [lava.commands] | ||
1188 | 38 | dashboard=lava_dashboard_tool.commands:dashboard | ||
1189 | 39 | [lava.dashboard.commands] | ||
1190 | 40 | backup=lava_dashboard_tool.commands:backup | ||
1191 | 41 | bundles=lava_dashboard_tool.commands:bundles | ||
1192 | 42 | data_views=lava_dashboard_tool.commands:data_views | ||
1193 | 43 | deserialize=lava_dashboard_tool.commands:deserialize | ||
1194 | 44 | get=lava_dashboard_tool.commands:get | ||
1195 | 45 | make_stream=lava_dashboard_tool.commands:make_stream | ||
1196 | 46 | pull=lava_dashboard_tool.commands:pull | ||
1197 | 47 | put=lava_dashboard_tool.commands:put | ||
1198 | 48 | query_data_view=lava_dashboard_tool.commands:query_data_view | ||
1199 | 49 | restore=lava_dashboard_tool.commands:restore | ||
1200 | 50 | server_version=lava_dashboard_tool.commands:server_version | ||
1201 | 51 | streams=lava_dashboard_tool.commands:streams | ||
1202 | 52 | version=lava_dashboard_tool.commands:version | ||
1203 | 53 | [lava_dashboard_tool.commands] | ||
1204 | 54 | backup=lava_dashboard_tool.commands:backup | ||
1205 | 55 | bundles=lava_dashboard_tool.commands:bundles | ||
1206 | 56 | data_views=lava_dashboard_tool.commands:data_views | ||
1207 | 57 | deserialize=lava_dashboard_tool.commands:deserialize | ||
1208 | 58 | get=lava_dashboard_tool.commands:get | ||
1209 | 59 | make_stream=lava_dashboard_tool.commands:make_stream | ||
1210 | 60 | pull=lava_dashboard_tool.commands:pull | ||
1211 | 61 | put=lava_dashboard_tool.commands:put | ||
1212 | 62 | query_data_view=lava_dashboard_tool.commands:query_data_view | ||
1213 | 63 | restore=lava_dashboard_tool.commands:restore | ||
1214 | 64 | server_version=lava_dashboard_tool.commands:server_version | ||
1215 | 65 | streams=lava_dashboard_tool.commands:streams | ||
1216 | 66 | version=lava_dashboard_tool.commands:version | ||
1217 | 67 | """, | ||
1218 | 68 | classifiers=[ | 34 | classifiers=[ |
1219 | 69 | "Development Status :: 4 - Beta", | 35 | "Development Status :: 4 - Beta", |
1220 | 70 | "Intended Audience :: Developers", | 36 | "Intended Audience :: Developers", |
1221 | @@ -75,9 +41,8 @@ | |||
1222 | 75 | "Programming Language :: Python :: 2.7", | 41 | "Programming Language :: Python :: 2.7", |
1223 | 76 | "Topic :: Software Development :: Testing"], | 42 | "Topic :: Software Development :: Testing"], |
1224 | 77 | install_requires=[ | 43 | install_requires=[ |
1228 | 78 | 'lava-tool [auth] >= 0.4', | 44 | 'lava-tool >= 0.7.dev', |
1229 | 79 | 'json-schema-validator >= 2.0', | 45 | ], |
1227 | 80 | 'versiontools >= 1.3.1'], | ||
1230 | 81 | setup_requires=['versiontools >= 1.3.1'], | 46 | setup_requires=['versiontools >= 1.3.1'], |
1231 | 82 | tests_require=[], | 47 | tests_require=[], |
1232 | 83 | zip_safe=True) | 48 | zip_safe=True) |
Seems fine to me -- is this really needed though? Couldn't one just
remove lava-dashboard-tool from the manifest?