Merge lp:~terceiro/lava-dashboard-tool/absorbed-by-lava-tool into lp:lava-dashboard-tool

Proposed by Antonio Terceiro
Status: Merged
Approved by: Senthil Kumaran S
Approved revision: 160
Merged at revision: 160
Proposed branch: lp:~terceiro/lava-dashboard-tool/absorbed-by-lava-tool
Merge into: lp:lava-dashboard-tool
Diff against target: 1230 lines (+4/-1176)
6 files modified
lava_dashboard_tool/__init__.py (+0/-23)
lava_dashboard_tool/commands.py (+0/-981)
lava_dashboard_tool/main.py (+0/-37)
lava_dashboard_tool/tests/__init__.py (+0/-52)
lava_dashboard_tool/tests/test_commands.py (+0/-44)
setup.py (+4/-39)
To merge this branch: bzr merge lp:~terceiro/lava-dashboard-tool/absorbed-by-lava-tool
Reviewer Review Type Date Requested Status
Senthil Kumaran S Approve
Review via email: mp+160193@code.launchpad.net

Description of the change

To post a comment you must log in.
Revision history for this message
Michael Hudson-Doyle (mwhudson) wrote :

Seems fine to me -- is this really needed though? Couldn't one just
remove lava-dashboard-tool from the manifest?

Revision history for this message
Antonio Terceiro (terceiro) wrote :

On Tue, Apr 23, 2013 at 12:58:22AM -0000, Michael Hudson-Doyle wrote:
> Seems fine to me -- is this really needed though? Couldn't one just
> remove lava-dashboard-tool from the manifest?

People have them installed with pypi and APT, and we need to provide a
clean upgrade path so that they don't get two packages providing
lava_dashboard_tool Python module at the same time. For this we need to
make a last release of the deprecated packages with no code at all that
depends on the package/version that provides that code.

--
Antonio Terceiro
Software Engineer - Linaro
http://www.linaro.org

Revision history for this message
Senthil Kumaran S (stylesen) wrote :

+1

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== removed directory 'lava_dashboard_tool'
=== removed file 'lava_dashboard_tool/__init__.py'
--- lava_dashboard_tool/__init__.py 2012-03-22 18:38:32 +0000
+++ lava_dashboard_tool/__init__.py 1970-01-01 00:00:00 +0000
@@ -1,23 +0,0 @@
1# Copyright (C) 2010,2011 Linaro Limited
2#
3# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
4#
5# This file is part of lava-dashboard-tool.
6#
7# lava-dashboard-tool is free software: you can redistribute it and/or modify
8# it under the terms of the GNU Lesser General Public License version 3
9# as published by the Free Software Foundation
10#
11# lava-dashboard-tool is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU Lesser General Public License
17# along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>.
18
19"""
20Launch Control Tool package
21"""
22
23__version__ = (0, 8, 0, "dev", 0)
240
=== removed file 'lava_dashboard_tool/commands.py'
--- lava_dashboard_tool/commands.py 2012-03-22 18:12:14 +0000
+++ lava_dashboard_tool/commands.py 1970-01-01 00:00:00 +0000
@@ -1,981 +0,0 @@
1# Copyright (C) 2010,2011 Linaro Limited
2#
3# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
4#
5# This file is part of lava-dashboard-tool.
6#
7# lava-dashboard-tool is free software: you can redistribute it and/or modify
8# it under the terms of the GNU Lesser General Public License version 3
9# as published by the Free Software Foundation
10#
11# lava-dashboard-tool is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU Lesser General Public License
17# along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>.
18
19"""
20Module with command-line tool commands that interact with the dashboard
21server. All commands listed here should have counterparts in the
22launch_control.dashboard_app.xml_rpc package.
23"""
24
25import argparse
26import contextlib
27import errno
28import os
29import re
30import socket
31import sys
32import urllib
33import urlparse
34import xmlrpclib
35
36import simplejson
37from json_schema_validator.extensions import datetime_extension
38
39from lava_tool.authtoken import AuthenticatingServerProxy, KeyringAuthBackend
40from lava.tool.commands import ExperimentalCommandMixIn
41from lava.tool.command import Command, CommandGroup
42
43
44class dashboard(CommandGroup):
45 """
46 Commands for interacting with LAVA Dashboard
47 """
48
49 namespace = "lava.dashboard.commands"
50
51
52class InsufficientServerVersion(Exception):
53 """
54 Exception raised when server version that a command interacts with is too
55 old to support required features.
56 """
57 def __init__(self, server_version, required_version):
58 self.server_version = server_version
59 self.required_version = required_version
60
61
62class DataSetRenderer(object):
63 """
64 Support class for rendering a table out of list of dictionaries.
65
66 It supports several features that make printing tabular data easier.
67 * Automatic layout
68 * Custom column headers
69 * Custom cell formatting
70 * Custom table captions
71 * Custom column ordering
72 * Custom Column separators
73 * Custom dataset notification
74
75 The primary method is render() which does all of the above. You
76 need to pass a dataset argument which is a list of dictionaries.
77 Each dictionary must have the same keys. In particular the first row
78 is used to determine columns.
79 """
80 def __init__(self, column_map=None, row_formatter=None, empty=None,
81 order=None, caption=None, separator=" ", header_separator=None):
82 if column_map is None:
83 column_map = {}
84 if row_formatter is None:
85 row_formatter = {}
86 if empty is None:
87 empty = "There is no data to display"
88 self.column_map = column_map
89 self.row_formatter = row_formatter
90 self.empty = empty
91 self.order = order
92 self.separator = separator
93 self.caption = caption
94 self.header_separator = header_separator
95
96 def _analyze_dataset(self, dataset):
97 """
98 Determine the columns that will be displayed and the maximum
99 length of each of those columns.
100
101 Returns a tuple (dataset, columms, maxlen) where columns is a
102 list of column names and maxlen is a dictionary mapping from
103 column name to maximum length of any value in the row or the
104 column header and the dataset is a copy of the dataset altered
105 as necessary.
106
107 Some examples:
108
109 First the dataset, an array of dictionaries
110 >>> dataset = [
111 ... {'a': 'shorter', 'bee': ''},
112 ... {'a': 'little longer', 'bee': 'b'}]
113
114 Note that column 'bee' is actually three characters long as the
115 column name made it wider.
116 >>> dataset_out, columns, maxlen = DataSetRenderer(
117 ... )._analyze_dataset(dataset)
118
119 Unless you format rows with a custom function the data is not altered.
120 >>> dataset_out is dataset
121 True
122
123 Columns come out in sorted alphabetic order
124 >>> columns
125 ['a', 'bee']
126
127 Maximum length determines the width of each column. Note that
128 the header affects the column width.
129 >>> maxlen
130 {'a': 13, 'bee': 3}
131
132 You can constrain or reorder columns. In that case columns you
133 decided to ignore are simply left out of the output.
134 >>> dataset_out, columns, maxlen = DataSetRenderer(
135 ... order=['bee'])._analyze_dataset(dataset)
136 >>> columns
137 ['bee']
138 >>> maxlen
139 {'bee': 3}
140
141 You can format values anyway you like:
142 >>> dataset_out, columns, maxlen = DataSetRenderer(row_formatter={
143 ... 'bee': lambda value: "%10s" % value}
144 ... )._analyze_dataset(dataset)
145
146 Dataset is altered to take account of the row formatting
147 function. The original dataset argument is copied.
148 >>> dataset_out
149 [{'a': 'shorter', 'bee': ' '}, {'a': 'little longer', 'bee': ' b'}]
150 >>> dataset_out is not dataset
151 True
152
153 Columns stay the same though:
154 >>> columns
155 ['a', 'bee']
156
157 Note how formatting altered the width of the column 'bee'
158 >>> maxlen
159 {'a': 13, 'bee': 10}
160
161 You can also format columns (with nice aliases).Note how
162 column 'bee' maximum width is now dominated by the long column
163 name:
164 >>> dataset_out, columns, maxlen = DataSetRenderer(column_map={
165 ... 'bee': "Column B"})._analyze_dataset(dataset)
166 >>> maxlen
167 {'a': 13, 'bee': 8}
168 """
169 if self.order:
170 columns = self.order
171 else:
172 columns = sorted(dataset[0].keys())
173 if self.row_formatter:
174 dataset_out = [dict(row) for row in dataset]
175 else:
176 dataset_out = dataset
177 for row in dataset_out:
178 for column in row:
179 if column in self.row_formatter:
180 row[column] = self.row_formatter[column](row[column])
181 maxlen = dict(
182 [(column, max(
183 len(self.column_map.get(column, column)),
184 max([
185 len(str(row[column])) for row in dataset_out])))
186 for column in columns])
187 return dataset_out, columns, maxlen
188
189 def _render_header(self, dataset, columns, maxlen):
190 """
191 Render a header, possibly with a caption string
192
193 Caption is controlled by the constructor.
194 >>> dataset = [
195 ... {'a': 'shorter', 'bee': ''},
196 ... {'a': 'little longer', 'bee': 'b'}]
197 >>> columns = ['a', 'bee']
198 >>> maxlen = {'a': 13, 'bee': 3}
199
200 By default there is no caption, just column names:
201 >>> DataSetRenderer()._render_header(
202 ... dataset, columns, maxlen)
203 a bee
204
205 If you enable the header separator then column names will be visually
206 separated from the first row of data.
207 >>> DataSetRenderer(header_separator=True)._render_header(
208 ... dataset, columns, maxlen)
209 a bee
210 -----------------
211
212 If you provide a caption it gets rendered as a centered
213 underlined text before the data:
214 >>> DataSetRenderer(caption="Dataset")._render_header(
215 ... dataset, columns, maxlen)
216 Dataset
217 =================
218 a bee
219
220 You can use both caption and header separator
221 >>> DataSetRenderer(caption="Dataset", header_separator=True)._render_header(
222 ... dataset, columns, maxlen)
223 Dataset
224 =================
225 a bee
226 -----------------
227
228 Observe how the total length of the output horizontal line
229 depends on the separator! Also note the columns labels are
230 aligned to the center of the column
231 >>> DataSetRenderer(caption="Dataset", separator=" | ")._render_header(
232 ... dataset, columns, maxlen)
233 Dataset
234 ===================
235 a | bee
236 """
237 total_len = sum(maxlen.itervalues())
238 if len(columns):
239 total_len += len(self.separator) * (len(columns) - 1)
240 # Print the caption
241 if self.caption:
242 print "{0:^{1}}".format(self.caption, total_len)
243 print "=" * total_len
244 # Now print the coulum names
245 print self.separator.join([
246 "{0:^{1}}".format(self.column_map.get(column, column),
247 maxlen[column]) for column in columns])
248 # Finally print the header separator
249 if self.header_separator:
250 print "-" * total_len
251
252 def _render_rows(self, dataset, columns, maxlen):
253 """
254 Render rows of the dataset.
255
256 Each row is printed on one line using the maxlen argument to
257 determine correct column size. Text is aligned left.
258
259 First the dataset, columns and maxlen as produced by
260 _analyze_dataset()
261 >>> dataset = [
262 ... {'a': 'shorter', 'bee': ''},
263 ... {'a': 'little longer', 'bee': 'b'}]
264 >>> columns = ['a', 'bee']
265 >>> maxlen = {'a': 13, 'bee': 3}
266
267 Now a plain table. Note! To really understand this test
268 you should check out the length of the strings below. There
269 are two more spaces after 'b' in the second row
270 >>> DataSetRenderer()._render_rows(dataset, columns, maxlen)
271 shorter
272 little longer b
273 """
274 for row in dataset:
275 print self.separator.join([
276 "{0!s:{1}}".format(row[column], maxlen[column])
277 for column in columns])
278
279 def _render_dataset(self, dataset):
280 """
281 Render the header followed by the rows of data.
282 """
283 dataset, columns, maxlen = self._analyze_dataset(dataset)
284 self._render_header(dataset, columns, maxlen)
285 self._render_rows(dataset, columns, maxlen)
286
287 def _render_empty_dataset(self):
288 """
289 Render empty dataset.
290
291 By default it just prints out a fixed sentence:
292 >>> DataSetRenderer()._render_empty_dataset()
293 There is no data to display
294
295 This can be changed by passing an argument to the constructor
296 >>> DataSetRenderer(empty="there is no data")._render_empty_dataset()
297 there is no data
298 """
299 print self.empty
300
301 def render(self, dataset):
302 if len(dataset) > 0:
303 self._render_dataset(dataset)
304 else:
305 self._render_empty_dataset()
306
307
308class XMLRPCCommand(Command):
309 """
310 Abstract base class for commands that interact with dashboard server
311 over XML-RPC.
312
313 The only difference is that you should implement invoke_remote()
314 instead of invoke(). The provided implementation catches several
315 socket and XML-RPC errors and prints a pretty error message.
316 """
317
318 @staticmethod
319 def _construct_xml_rpc_url(url):
320 """
321 Construct URL to the XML-RPC service out of the given URL
322 """
323 parts = urlparse.urlsplit(url)
324 if not parts.path.endswith("/RPC2/"):
325 path = parts.path.rstrip("/") + "/xml-rpc/"
326 else:
327 path = parts.path
328 return urlparse.urlunsplit(
329 (parts.scheme, parts.netloc, path, "", ""))
330
331 @staticmethod
332 def _strict_server_version(version):
333 """
334 Calculate strict server version (as defined by
335 distutils.version.StrictVersion). This works by discarding .candidate
336 and .dev release-levels.
337 >>> XMLRPCCommand._strict_server_version("0.4.0.candidate.5")
338 '0.4.0'
339 >>> XMLRPCCommand._strict_server_version("0.4.0.dev.126")
340 '0.4.0'
341 >>> XMLRPCCommand._strict_server_version("0.4.0.alpha.1")
342 '0.4.0a1'
343 >>> XMLRPCCommand._strict_server_version("0.4.0.beta.2")
344 '0.4.0b2'
345 """
346 try:
347 major, minor, micro, releaselevel, serial = version.split(".")
348 except ValueError:
349 raise ValueError(
350 ("version %r does not follow pattern "
351 "'major.minor.micro.releaselevel.serial'") % version)
352 if releaselevel in ["dev", "candidate", "final"]:
353 return "%s.%s.%s" % (major, minor, micro)
354 elif releaselevel == "alpha":
355 return "%s.%s.%sa%s" % (major, minor, micro, serial)
356 elif releaselevel == "beta":
357 return "%s.%s.%sb%s" % (major, minor, micro, serial)
358 else:
359 raise ValueError(
360 ("releaselevel %r is not one of 'final', 'alpha', 'beta', "
361 "'candidate' or 'final'") % releaselevel)
362
363 def _check_server_version(self, server_obj, required_version):
364 """
365 Check that server object has is at least required_version.
366
367 This method may raise InsufficientServerVersion.
368 """
369 from distutils.version import StrictVersion, LooseVersion
370 # For backwards compatibility the server reports
371 # major.minor.micro.releaselevel.serial which is not PEP-386 compliant
372 server_version = StrictVersion(
373 self._strict_server_version(server_obj.version()))
374 required_version = StrictVersion(required_version)
375 if server_version < required_version:
376 raise InsufficientServerVersion(server_version, required_version)
377
378 def __init__(self, parser, args):
379 super(XMLRPCCommand, self).__init__(parser, args)
380 xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url)
381 self.server = AuthenticatingServerProxy(
382 xml_rpc_url,
383 verbose=args.verbose_xml_rpc,
384 allow_none=True,
385 use_datetime=True,
386 auth_backend=KeyringAuthBackend())
387
388 def use_non_legacy_api_if_possible(self, name='server'):
389 # Legacy APIs are registered in top-level object, non-legacy APIs are
390 # prefixed with extension name.
391 if "dashboard.version" in getattr(self, name).system.listMethods():
392 setattr(self, name, getattr(self, name).dashboard)
393
394 @classmethod
395 def register_arguments(cls, parser):
396 dashboard_group = parser.add_argument_group("dashboard specific arguments")
397 default_dashboard_url = os.getenv("DASHBOARD_URL")
398 if default_dashboard_url:
399 dashboard_group.add_argument("--dashboard-url",
400 metavar="URL", help="URL of your validation dashboard (currently %(default)s)",
401 default=default_dashboard_url)
402 else:
403 dashboard_group.add_argument("--dashboard-url", required=True,
404 metavar="URL", help="URL of your validation dashboard")
405 debug_group = parser.add_argument_group("debugging arguments")
406 debug_group.add_argument("--verbose-xml-rpc",
407 action="store_true", default=False,
408 help="Show XML-RPC data")
409 return dashboard_group
410
411 @contextlib.contextmanager
412 def safety_net(self):
413 try:
414 yield
415 except socket.error as ex:
416 print >> sys.stderr, "Unable to connect to server at %s" % (
417 self.args.dashboard_url,)
418 # It seems that some errors are reported as -errno
419 # while others as +errno.
420 ex.errno = abs(ex.errno)
421 if ex.errno == errno.ECONNREFUSED:
422 print >> sys.stderr, "Connection was refused."
423 parts = urlparse.urlsplit(self.args.dashboard_url)
424 if parts.netloc == "localhost:8000":
425 print >> sys.stderr, "Perhaps the server is not running?"
426 elif ex.errno == errno.ENOENT:
427 print >> sys.stderr, "Unable to resolve address"
428 else:
429 print >> sys.stderr, "Socket %d: %s" % (ex.errno, ex.strerror)
430 except xmlrpclib.ProtocolError as ex:
431 print >> sys.stderr, "Unable to exchange XML-RPC message with dashboard server"
432 print >> sys.stderr, "HTTP error code: %d/%s" % (ex.errcode, ex.errmsg)
433 except xmlrpclib.Fault as ex:
434 self.handle_xmlrpc_fault(ex.faultCode, ex.faultString)
435 except InsufficientServerVersion as ex:
436 print >> sys.stderr, ("This command requires at least server version "
437 "%s, actual server version is %s" %
438 (ex.required_version, ex.server_version))
439
440 def invoke(self):
441 with self.safety_net():
442 self.use_non_legacy_api_if_possible()
443 return self.invoke_remote()
444
445 def handle_xmlrpc_fault(self, faultCode, faultString):
446 if faultCode == 500:
447 print >> sys.stderr, "Dashboard server has experienced internal error"
448 print >> sys.stderr, faultString
449 else:
450 print >> sys.stderr, "XML-RPC error %d: %s" % (faultCode, faultString)
451
452 def invoke_remote(self):
453 raise NotImplementedError()
454
455
456class server_version(XMLRPCCommand):
457 """
458 Display dashboard server version
459 """
460
461 def invoke_remote(self):
462 print "Dashboard server version: %s" % (self.server.version(),)
463
464
465class put(XMLRPCCommand):
466 """
467 Upload a bundle on the server
468 """
469
470 @classmethod
471 def register_arguments(cls, parser):
472 super(put, cls).register_arguments(parser)
473 parser.add_argument("LOCAL",
474 type=argparse.FileType("rb"),
475 help="pathname on the local file system")
476 parser.add_argument("REMOTE",
477 default="/anonymous/", nargs='?',
478 help="pathname on the server")
479
480 def invoke_remote(self):
481 content = self.args.LOCAL.read()
482 filename = self.args.LOCAL.name
483 pathname = self.args.REMOTE
484 content_sha1 = self.server.put(content, filename, pathname)
485 print "Stored as bundle {0}".format(content_sha1)
486
487 def handle_xmlrpc_fault(self, faultCode, faultString):
488 if faultCode == 404:
489 print >> sys.stderr, "Bundle stream %s does not exist" % (
490 self.args.REMOTE)
491 elif faultCode == 409:
492 print >> sys.stderr, "You have already uploaded this bundle to the dashboard"
493 else:
494 super(put, self).handle_xmlrpc_fault(faultCode, faultString)
495
496
497class get(XMLRPCCommand):
498 """
499 Download a bundle from the server
500 """
501
502 @classmethod
503 def register_arguments(cls, parser):
504 super(get, cls).register_arguments(parser)
505 parser.add_argument("SHA1",
506 type=str,
507 help="SHA1 of the bundle to download")
508 parser.add_argument("--overwrite",
509 action="store_true",
510 help="Overwrite files on the local disk")
511 parser.add_argument("--output", "-o",
512 type=argparse.FileType("wb"),
513 default=None,
514 help="Alternate name of the output file")
515
516 def invoke_remote(self):
517 response = self.server.get(self.args.SHA1)
518 if self.args.output is None:
519 filename = self.args.SHA1
520 if os.path.exists(filename) and not self.args.overwrite:
521 print >> sys.stderr, "File {filename!r} already exists".format(
522 filename=filename)
523 print >> sys.stderr, "You may pass --overwrite to write over it"
524 return -1
525 stream = open(filename, "wb")
526 else:
527 stream = self.args.output
528 filename = self.args.output.name
529 stream.write(response['content'])
530 print "Downloaded bundle {0} to file {1!r}".format(
531 self.args.SHA1, filename)
532
533 def handle_xmlrpc_fault(self, faultCode, faultString):
534 if faultCode == 404:
535 print >> sys.stderr, "Bundle {sha1} does not exist".format(
536 sha1=self.args.SHA1)
537 else:
538 super(get, self).handle_xmlrpc_fault(faultCode, faultString)
539
540
541class deserialize(XMLRPCCommand):
542 """
543 Deserialize a bundle on the server
544 """
545
546 @classmethod
547 def register_arguments(cls, parser):
548 super(deserialize, cls).register_arguments(parser)
549 parser.add_argument("SHA1",
550 type=str,
551 help="SHA1 of the bundle to deserialize")
552
553 def invoke_remote(self):
554 response = self.server.deserialize(self.args.SHA1)
555 print "Bundle {sha1} deserialized".format(
556 sha1=self.args.SHA1)
557
558 def handle_xmlrpc_fault(self, faultCode, faultString):
559 if faultCode == 404:
560 print >> sys.stderr, "Bundle {sha1} does not exist".format(
561 sha1=self.args.SHA1)
562 elif faultCode == 409:
563 print >> sys.stderr, "Unable to deserialize bundle {sha1}".format(
564 sha1=self.args.SHA1)
565 print >> sys.stderr, faultString
566 else:
567 super(deserialize, self).handle_xmlrpc_fault(faultCode, faultString)
568
569
570def _get_pretty_renderer(**kwargs):
571 if "separator" not in kwargs:
572 kwargs["separator"] = " | "
573 if "header_separator" not in kwargs:
574 kwargs["header_separator"] = True
575 return DataSetRenderer(**kwargs)
576
577
578class streams(XMLRPCCommand):
579 """
580 Show streams you have access to
581 """
582
583 renderer = _get_pretty_renderer(
584 order=('pathname', 'bundle_count', 'name'),
585 column_map={
586 'pathname': 'Pathname',
587 'bundle_count': 'Number of bundles',
588 'name': 'Name'},
589 row_formatter={
590 'name': lambda name: name or "(not set)"},
591 empty="There are no streams you can access on the server",
592 caption="Bundle streams")
593
594 def invoke_remote(self):
595 self.renderer.render(self.server.streams())
596
597
598class bundles(XMLRPCCommand):
599 """
600 Show bundles in the specified stream
601 """
602
603 renderer = _get_pretty_renderer(
604 column_map={
605 'uploaded_by': 'Uploader',
606 'uploaded_on': 'Upload date',
607 'content_filename': 'File name',
608 'content_sha1': 'SHA1',
609 'is_deserialized': "Deserialized?"},
610 row_formatter={
611 'is_deserialized': lambda x: "yes" if x else "no",
612 'uploaded_by': lambda x: x or "(anonymous)",
613 'uploaded_on': lambda x: x.strftime("%Y-%m-%d %H:%M:%S")},
614 order=('content_sha1', 'content_filename', 'uploaded_by',
615 'uploaded_on', 'is_deserialized'),
616 empty="There are no bundles in this stream",
617 caption="Bundles",
618 separator=" | ")
619
620 @classmethod
621 def register_arguments(cls, parser):
622 super(bundles, cls).register_arguments(parser)
623 parser.add_argument("PATHNAME",
624 default="/anonymous/", nargs='?',
625 help="pathname on the server (defaults to %(default)s)")
626
627 def invoke_remote(self):
628 self.renderer.render(self.server.bundles(self.args.PATHNAME))
629
630 def handle_xmlrpc_fault(self, faultCode, faultString):
631 if faultCode == 404:
632 print >> sys.stderr, "Bundle stream %s does not exist" % (
633 self.args.PATHNAME)
634 else:
635 super(bundles, self).handle_xmlrpc_fault(faultCode, faultString)
636
637
638class make_stream(XMLRPCCommand):
639 """
640 Create a bundle stream on the server
641 """
642
643 @classmethod
644 def register_arguments(cls, parser):
645 super(make_stream, cls).register_arguments(parser)
646 parser.add_argument(
647 "pathname",
648 type=str,
649 help="Pathname of the bundle stream to create")
650 parser.add_argument(
651 "--name",
652 type=str,
653 default="",
654 help="Name of the bundle stream (description)")
655
656 def invoke_remote(self):
657 self._check_server_version(self.server, "0.3")
658 pathname = self.server.make_stream(self.args.pathname, self.args.name)
659 print "Bundle stream {pathname} created".format(pathname=pathname)
660
661
662class backup(XMLRPCCommand):
663 """
664 Backup data uploaded to a dashboard instance.
665
6660
667 Not all data is preserved. The following data is lost: identity of the user
668 that uploaded each bundle, time of uploading and deserialization on the
669 server, name of the bundle stream that contained the data
670 """
671
672 @classmethod
673 def register_arguments(cls, parser):
674 super(backup, cls).register_arguments(parser)
675 parser.add_argument("BACKUP_DIR", type=str,
676 help="Directory to backup to")
677
678 def invoke_remote(self):
679 if not os.path.exists(self.args.BACKUP_DIR):
680 os.mkdir(self.args.BACKUP_DIR)
681 for bundle_stream in self.server.streams():
682 print "Processing stream %s" % bundle_stream["pathname"]
683 bundle_stream_dir = os.path.join(self.args.BACKUP_DIR, urllib.quote_plus(bundle_stream["pathname"]))
684 if not os.path.exists(bundle_stream_dir):
685 os.mkdir(bundle_stream_dir)
686 with open(os.path.join(bundle_stream_dir, "metadata.json"), "wt") as stream:
687 simplejson.dump({
688 "pathname": bundle_stream["pathname"],
689 "name": bundle_stream["name"],
690 "user": bundle_stream["user"],
691 "group": bundle_stream["group"],
692 }, stream)
693 for bundle in self.server.bundles(bundle_stream["pathname"]):
694 print " * Backing up bundle %s" % bundle["content_sha1"]
695 data = self.server.get(bundle["content_sha1"])
696 bundle_pathname = os.path.join(bundle_stream_dir, bundle["content_sha1"])
697 # Note: we write bundles as binary data to preserve anything the user might have dumped on us
698 with open(bundle_pathname + ".json", "wb") as stream:
699 stream.write(data["content"])
700 with open(bundle_pathname + ".metadata.json", "wt") as stream:
701 simplejson.dump({
702 "uploaded_by": bundle["uploaded_by"],
703 "uploaded_on": datetime_extension.to_json(bundle["uploaded_on"]),
704 "content_filename": bundle["content_filename"],
705 "content_sha1": bundle["content_sha1"],
706 "content_size": bundle["content_size"],
707 }, stream)
708
709
710class restore(XMLRPCCommand):
711 """
712 Restore a dashboard instance from backup
713 """
714
715 @classmethod
716 def register_arguments(cls, parser):
717 super(restore, cls).register_arguments(parser)
718 parser.add_argument("BACKUP_DIR", type=str,
719 help="Directory to backup from")
720
721 def invoke_remote(self):
722 self._check_server_version(self.server, "0.3")
723 for stream_pathname_quoted in os.listdir(self.args.BACKUP_DIR):
724 filesystem_stream_pathname = os.path.join(self.args.BACKUP_DIR, stream_pathname_quoted)
725 if not os.path.isdir(filesystem_stream_pathname):
726 continue
727 stream_pathname = urllib.unquote(stream_pathname_quoted)
728 if os.path.exists(os.path.join(filesystem_stream_pathname, "metadata.json")):
729 with open(os.path.join(filesystem_stream_pathname, "metadata.json"), "rt") as stream:
730 stream_metadata = simplejson.load(stream)
731 else:
732 stream_metadata = {}
733 print "Processing stream %s" % stream_pathname
734 try:
735 self.server.make_stream(stream_pathname, stream_metadata.get("name", "Restored from backup"))
736 except xmlrpclib.Fault as ex:
737 if ex.faultCode != 409:
738 raise
739 for content_sha1 in [item[:-len(".json")] for item in os.listdir(filesystem_stream_pathname) if item.endswith(".json") and not item.endswith(".metadata.json") and item != "metadata.json"]:
740 filesystem_content_filename = os.path.join(filesystem_stream_pathname, content_sha1 + ".json")
741 if not os.path.isfile(filesystem_content_filename):
742 continue
743 with open(os.path.join(filesystem_stream_pathname, content_sha1) + ".metadata.json", "rt") as stream:
744 bundle_metadata = simplejson.load(stream)
745 with open(filesystem_content_filename, "rb") as stream:
746 content = stream.read()
747 print " * Restoring bundle %s" % content_sha1
748 try:
749 self.server.put(content, bundle_metadata["content_filename"], stream_pathname)
750 except xmlrpclib.Fault as ex:
751 if ex.faultCode != 409:
752 raise
753
754
755class pull(ExperimentalCommandMixIn, XMLRPCCommand):
756 """
757 Copy bundles and bundle streams from one dashboard to another.
758
7591
760 This command checks for two environment varialbes:
761 The value of DASHBOARD_URL is used as a replacement for --dashbard-url.
762 The value of REMOTE_DASHBOARD_URL as a replacement for FROM.
763 Their presence automatically makes the corresponding argument optional.
764 """
765
766 def __init__(self, parser, args):
767 super(pull, self).__init__(parser, args)
768 remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM)
769 self.remote_server = AuthenticatingServerProxy(
770 remote_xml_rpc_url,
771 verbose=args.verbose_xml_rpc,
772 use_datetime=True,
773 allow_none=True,
774 auth_backend=KeyringAuthBackend())
775 self.use_non_legacy_api_if_possible('remote_server')
776
777 @classmethod
778 def register_arguments(cls, parser):
779 group = super(pull, cls).register_arguments(parser)
780 default_remote_dashboard_url = os.getenv("REMOTE_DASHBOARD_URL")
781 if default_remote_dashboard_url:
782 group.add_argument(
783 "FROM", nargs="?",
784 help="URL of the remote validation dashboard (currently %(default)s)",
785 default=default_remote_dashboard_url)
786 else:
787 group.add_argument(
788 "FROM",
789 help="URL of the remote validation dashboard)")
790 group.add_argument("STREAM", nargs="*", help="Streams to pull from (all by default)")
791
792 @staticmethod
793 def _filesizeformat(num_bytes):
794 """
795 Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
796 102 num_bytes, etc).
797 """
798 try:
799 num_bytes = float(num_bytes)
800 except (TypeError, ValueError, UnicodeDecodeError):
801 return "%(size)d byte", "%(size)d num_bytes" % {'size': 0}
802
803 filesize_number_format = lambda value: "%0.2f" % (round(value, 1),)
804
805 if num_bytes < 1024:
806 return "%(size)d bytes" % {'size': num_bytes}
807 if num_bytes < 1024 * 1024:
808 return "%s KB" % filesize_number_format(num_bytes / 1024)
809 if num_bytes < 1024 * 1024 * 1024:
810 return "%s MB" % filesize_number_format(num_bytes / (1024 * 1024))
811 return "%s GB" % filesize_number_format(num_bytes / (1024 * 1024 * 1024))
812
813 def invoke_remote(self):
814 self._check_server_version(self.server, "0.3")
815
816 print "Checking local and remote streams"
817 remote = self.remote_server.streams()
818 if self.args.STREAM:
819 # Check that all requested streams are available remotely
820 requested_set = frozenset(self.args.STREAM)
821 remote_set = frozenset((stream["pathname"] for stream in remote))
822 unavailable_set = requested_set - remote_set
823 if unavailable_set:
824 print >> sys.stderr, "Remote stream not found: %s" % ", ".join(unavailable_set)
825 return -1
826 # Limit to requested streams if necessary
827 remote = [stream for stream in remote if stream["pathname"] in requested_set]
828 local = self.server.streams()
829 missing_pathnames = set([stream["pathname"] for stream in remote]) - set([stream["pathname"] for stream in local])
830 for stream in remote:
831 if stream["pathname"] in missing_pathnames:
832 self.server.make_stream(stream["pathname"], stream["name"])
833 local_bundles = []
834 else:
835 local_bundles = [bundle for bundle in self.server.bundles(stream["pathname"])]
836 remote_bundles = [bundle for bundle in self.remote_server.bundles(stream["pathname"])]
837 missing_bundles = set((bundle["content_sha1"] for bundle in remote_bundles))
838 missing_bundles -= set((bundle["content_sha1"] for bundle in local_bundles))
839 try:
840 missing_bytes = sum(
841 (bundle["content_size"]
842 for bundle in remote_bundles
843 if bundle["content_sha1"] in missing_bundles))
844 except KeyError as ex:
845 # Older servers did not return content_size so this part is optional
846 missing_bytes = None
847 if missing_bytes:
848 print "Stream %s needs update (%s)" % (stream["pathname"], self._filesizeformat(missing_bytes))
849 elif missing_bundles:
850 print "Stream %s needs update (no estimate available)" % (stream["pathname"],)
851 else:
852 print "Stream %s is up to date" % (stream["pathname"],)
853 for content_sha1 in missing_bundles:
854 print "Getting %s" % (content_sha1,),
855 sys.stdout.flush()
856 data = self.remote_server.get(content_sha1)
857 print "got %s, storing" % (self._filesizeformat(len(data["content"]))),
858 sys.stdout.flush()
859 try:
860 self.server.put(data["content"], data["content_filename"], stream["pathname"])
861 except xmlrpclib.Fault as ex:
862 if ex.faultCode == 409: # duplicate
863 print "already present (in another stream)"
864 else:
865 raise
866 else:
867 print "done"
868
869
870class data_views(ExperimentalCommandMixIn, XMLRPCCommand):
871 """
872 Show data views defined on the server
873 """
874 renderer = _get_pretty_renderer(
875 column_map={
876 'name': 'Name',
877 'summary': 'Summary',
878 },
879 order=('name', 'summary'),
880 empty="There are no data views defined yet",
881 caption="Data Views")
882
883 def invoke_remote(self):
884 self._check_server_version(self.server, "0.4")
885 self.renderer.render(self.server.data_views())
886 print
887 print "Tip: to invoke a data view try `lc-tool query-data-view`"
888
889
890class query_data_view(ExperimentalCommandMixIn, XMLRPCCommand):
891 """
892 Invoke a specified data view
893 """
894 @classmethod
895 def register_arguments(cls, parser):
896 super(query_data_view, cls).register_arguments(parser)
897 parser.add_argument("QUERY", metavar="QUERY", nargs="...",
898 help="Data view name and any optional and required arguments")
899
900 def _probe_data_views(self):
901 """
902 Probe the server for information about data views
903 """
904 with self.safety_net():
905 self.use_non_legacy_api_if_possible()
906 self._check_server_version(self.server, "0.4")
907 return self.server.data_views()
908
909 def reparse_arguments(self, parser, raw_args):
910 self.data_views = self._probe_data_views()
911 if self.data_views is None:
912 return
913 # Here we hack a little, the last actuin is the QUERY action added
914 # in register_arguments above. By removing it we make the output
915 # of lc-tool query-data-view NAME --help more consistent.
916 del parser._actions[-1]
917 subparsers = parser.add_subparsers(
918 title="Data views available on the server")
919 for data_view in self.data_views:
920 data_view_parser = subparsers.add_parser(
921 data_view["name"],
922 help=data_view["summary"],
923 epilog=data_view["documentation"])
924 data_view_parser.set_defaults(data_view=data_view)
925 group = data_view_parser.add_argument_group("Data view parameters")
926 for argument in data_view["arguments"]:
927 if argument["default"] is None:
928 group.add_argument(
929 "--{name}".format(name=argument["name"].replace("_", "-")),
930 dest=argument["name"],
931 help=argument["help"],
932 type=str,
933 required=True)
934 else:
935 group.add_argument(
936 "--{name}".format(name=argument["name"].replace("_", "-")),
937 dest=argument["name"],
938 help=argument["help"],
939 type=str,
940 default=argument["default"])
941 self.args = self.parser.parse_args(raw_args)
942
943 def invoke(self):
944 # Override and _not_ call 'use_non_legacy_api_if_possible' as we
945 # already did this reparse_arguments
946 with self.safety_net():
947 return self.invoke_remote()
948
949 def invoke_remote(self):
950 if self.data_views is None:
951 return -1
952 self._check_server_version(self.server, "0.4")
953 # Build a collection of arguments for data view
954 data_view_args = {}
955 for argument in self.args.data_view["arguments"]:
956 arg_name = argument["name"]
957 if arg_name in self.args:
958 data_view_args[arg_name] = getattr(self.args, arg_name)
959 # Invoke the data view
960 response = self.server.query_data_view(self.args.data_view["name"], data_view_args)
961 # Create a pretty-printer
962 renderer = _get_pretty_renderer(
963 caption=self.args.data_view["summary"],
964 order=[item["name"] for item in response["columns"]])
965 # Post-process the data so that it fits the printer
966 data_for_renderer = [
967 dict(zip(
968 [column["name"] for column in response["columns"]],
969 row))
970 for row in response["rows"]]
971 # Print the data
972 renderer.render(data_for_renderer)
973
974
975class version(Command):
976 """
977 Show dashboard client version
978 """
979 def invoke(self):
980 import versiontools
981 from lava_dashboard_tool import __version__
982 print "Dashboard client version: {version}".format(
983 version=versiontools.format_version(__version__))
9842
=== removed file 'lava_dashboard_tool/main.py'
--- lava_dashboard_tool/main.py 2011-06-23 11:23:24 +0000
+++ lava_dashboard_tool/main.py 1970-01-01 00:00:00 +0000
@@ -1,37 +0,0 @@
1# Copyright (C) 2011 Linaro Limited
2#
3# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
4# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
5#
6# This file is part of lava-dashboard-tool.
7#
8# lava-dashboard-tool is free software: you can redistribute it and/or modify
9# it under the terms of the GNU Lesser General Public License version 3
10# as published by the Free Software Foundation
11#
12# lava-dashboard-tool is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU Lesser General Public License
18# along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>.
19
20
21from lava_tool.dispatcher import LavaDispatcher, run_with_dispatcher_class
22
23
24class LaunchControlDispatcher(LavaDispatcher):
25
26 toolname = 'lava_dashboard_tool'
27 description = """
28 Command line tool for interacting with Launch Control
29 """
30 epilog = """
31 Please report all bugs using the Launchpad bug tracker:
32 http://bugs.launchpad.net/lava-dashboard-tool/+filebug
33 """
34
35
36def main():
37 run_with_dispatcher_class(LaunchControlDispatcher)
380
=== removed directory 'lava_dashboard_tool/tests'
=== removed file 'lava_dashboard_tool/tests/__init__.py'
--- lava_dashboard_tool/tests/__init__.py 2011-06-23 11:23:24 +0000
+++ lava_dashboard_tool/tests/__init__.py 1970-01-01 00:00:00 +0000
@@ -1,52 +0,0 @@
1# Copyright (C) 2010,2011 Linaro Limited
2#
3# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
4#
5# This file is part of lava-dashboard-tool.
6#
7# lava-dashboard-tool is free software: you can redistribute it and/or modify
8# it under the terms of the GNU Lesser General Public License version 3
9# as published by the Free Software Foundation
10#
11# lava-dashboard-tool is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU Lesser General Public License
17# along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>.
18
19"""
20Package with unit tests for lava_dashboard_tool
21"""
22
23import doctest
24import unittest
25
26
27def app_modules():
28 return [
29 'lava_dashboard_tool.commands',
30 ]
31
32
33def test_modules():
34 return [
35 'lava_dashboard_tool.tests.test_commands',
36 ]
37
38
39def test_suite():
40 """
41 Build an unittest.TestSuite() object with all the tests in _modules.
42 Each module is harvested for both regular unittests and doctests
43 """
44 modules = app_modules() + test_modules()
45 suite = unittest.TestSuite()
46 loader = unittest.TestLoader()
47 for name in modules:
48 unit_suite = loader.loadTestsFromName(name)
49 suite.addTests(unit_suite)
50 doc_suite = doctest.DocTestSuite(name)
51 suite.addTests(doc_suite)
52 return suite
530
=== removed file 'lava_dashboard_tool/tests/test_commands.py'
--- lava_dashboard_tool/tests/test_commands.py 2011-06-23 11:23:24 +0000
+++ lava_dashboard_tool/tests/test_commands.py 1970-01-01 00:00:00 +0000
@@ -1,44 +0,0 @@
1# Copyright (C) 2010,2011 Linaro Limited
2#
3# Author: Zygmunt Krynicki <zygmunt.krynicki@linaro.org>
4#
5# This file is part of lava-dashboard-tool.
6#
7# lava-dashboard-tool is free software: you can redistribute it and/or modify
8# it under the terms of the GNU Lesser General Public License version 3
9# as published by the Free Software Foundation
10#
11# lava-dashboard-tool is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU Lesser General Public License
17# along with lava-dashboard-tool. If not, see <http://www.gnu.org/licenses/>.
18
19"""
20Unit tests for the launch_control.commands package
21"""
22
23from unittest import TestCase
24
25from lava_dashboard_tool.commands import XMLRPCCommand
26
27
28class XMLRPCCommandTestCase(TestCase):
29
30 def test_construct_xml_rpc_url_preserves_path(self):
31 self.assertEqual(
32 XMLRPCCommand._construct_xml_rpc_url("http://domain/path"),
33 "http://domain/path/xml-rpc/")
34 self.assertEqual(
35 XMLRPCCommand._construct_xml_rpc_url("http://domain/path/"),
36 "http://domain/path/xml-rpc/")
37
38 def test_construct_xml_rpc_url_adds_proper_suffix(self):
39 self.assertEqual(
40 XMLRPCCommand._construct_xml_rpc_url("http://domain/"),
41 "http://domain/xml-rpc/")
42 self.assertEqual(
43 XMLRPCCommand._construct_xml_rpc_url("http://domain"),
44 "http://domain/xml-rpc/")
450
=== modified file 'setup.py'
--- setup.py 2012-03-22 18:12:14 +0000
+++ setup.py 2013-04-22 19:30:37 +0000
@@ -23,48 +23,14 @@
2323
24setup(24setup(
25 name='lava-dashboard-tool',25 name='lava-dashboard-tool',
26 version=":versiontools:lava_dashboard_tool:__version__",26 version="0.8",
27 author="Zygmunt Krynicki",27 author="Zygmunt Krynicki",
28 author_email="zygmunt.krynicki@linaro.org",28 author_email="zygmunt.krynicki@linaro.org",
29 packages=find_packages(),29 packages=find_packages(),
30 description="Command line utility for Launch Control",30 description="Command line utility for Launch Control (deprecated)",
31 url='https://launchpad.net/lava-dashboard-tool',31 url='https://launchpad.net/lava-dashboard-tool',
32 test_suite='lava_dashboard_tool.tests.test_suite',32 test_suite='lava_dashboard_tool.tests.test_suite',
33 license="LGPLv3",33 license="LGPLv3",
34 entry_points="""
35 [console_scripts]
36 lava-dashboard-tool=lava_dashboard_tool.main:main
37 [lava.commands]
38 dashboard=lava_dashboard_tool.commands:dashboard
39 [lava.dashboard.commands]
40 backup=lava_dashboard_tool.commands:backup
41 bundles=lava_dashboard_tool.commands:bundles
42 data_views=lava_dashboard_tool.commands:data_views
43 deserialize=lava_dashboard_tool.commands:deserialize
44 get=lava_dashboard_tool.commands:get
45 make_stream=lava_dashboard_tool.commands:make_stream
46 pull=lava_dashboard_tool.commands:pull
47 put=lava_dashboard_tool.commands:put
48 query_data_view=lava_dashboard_tool.commands:query_data_view
49 restore=lava_dashboard_tool.commands:restore
50 server_version=lava_dashboard_tool.commands:server_version
51 streams=lava_dashboard_tool.commands:streams
52 version=lava_dashboard_tool.commands:version
53 [lava_dashboard_tool.commands]
54 backup=lava_dashboard_tool.commands:backup
55 bundles=lava_dashboard_tool.commands:bundles
56 data_views=lava_dashboard_tool.commands:data_views
57 deserialize=lava_dashboard_tool.commands:deserialize
58 get=lava_dashboard_tool.commands:get
59 make_stream=lava_dashboard_tool.commands:make_stream
60 pull=lava_dashboard_tool.commands:pull
61 put=lava_dashboard_tool.commands:put
62 query_data_view=lava_dashboard_tool.commands:query_data_view
63 restore=lava_dashboard_tool.commands:restore
64 server_version=lava_dashboard_tool.commands:server_version
65 streams=lava_dashboard_tool.commands:streams
66 version=lava_dashboard_tool.commands:version
67 """,
68 classifiers=[34 classifiers=[
69 "Development Status :: 4 - Beta",35 "Development Status :: 4 - Beta",
70 "Intended Audience :: Developers",36 "Intended Audience :: Developers",
@@ -75,9 +41,8 @@
75 "Programming Language :: Python :: 2.7",41 "Programming Language :: Python :: 2.7",
76 "Topic :: Software Development :: Testing"],42 "Topic :: Software Development :: Testing"],
77 install_requires=[43 install_requires=[
78 'lava-tool [auth] >= 0.4',44 'lava-tool >= 0.7.dev',
79 'json-schema-validator >= 2.0',45 ],
80 'versiontools >= 1.3.1'],
81 setup_requires=['versiontools >= 1.3.1'],46 setup_requires=['versiontools >= 1.3.1'],
82 tests_require=[],47 tests_require=[],
83 zip_safe=True)48 zip_safe=True)

Subscribers

People subscribed via source and target branches