Merge lp:~zyga/lava-test/support-lava-command into lp:lava-test/0.0

[Zygmunt Krynicki]

This branch adds support for lava top-level command by duplicating all of the commands there. Some commands are altered to make the user interface better.

The old lava-test implementation is left untouched.

[Marcin Juszkiewicz (hrw)]

Looks nice. I have linaro-json part already applied in my packages.

[Zygmunt Krynicki (zyga)]

ping

[Michael Hudson-Doyle (mwhudson)]

This seems OK, but I assume it's quite a duplication of code?

Unmerged revisions

142. By Zygmunt Krynicki on 2012-03-28

Make lava test parse display what it is looking for

141. By Zygmunt Krynicki on 2012-03-28

Don't import and print version of linaro-json

140. By Zygmunt Krynicki on 2012-03-28

Create hierarchical version of all commands

=== modified file 'doc/changes.rst'
--- doc/changes.rst	2012-03-27 12:42:34 +0000
+++ doc/changes.rst	2012-04-02 16:50:25 +0000
@@ -6,6 +6,15 @@
 Version 0.7 (UNRELEASED)
 ========================
 
+* Add a hierachical version of all commands. Now ``lava-test`` can be invoked as ``lava test``.
+* Rename ``lava-test register-test`` to ``lava test register`` and ``lava-test
+  unregister-test`` to ``lava test unregister``.
+* Rename ``lava-test list-tests`` to ``lava test list``.
+* Merge ``lava test list-installed`` with ``lava test list``. The optional
+  --installed argument will now do what was previously a separate command.
+* Change how ``lava test reset`` displays progress
+* Make ``lava test parse`` display the files it is looking for
+
 .. _version_0_6:
 
 Version 0.6

=== added directory 'lava'
=== added file 'lava/__init__.py'
--- lava/__init__.py	1970-01-01 00:00:00 +0000
+++ lava/__init__.py	2012-04-02 16:50:25 +0000
@@ -0,0 +1,3 @@
+__import__('pkg_resources').declare_namespace(__name__)
+# DO NOT ADD ANYTHING TO THIS FILE!
+# IT MUST STAY AS IS (empty apart from the two lines above)

=== added directory 'lava/test'
=== added file 'lava/test/__init__.py'
=== added file 'lava/test/commands.py'
--- lava/test/commands.py	1970-01-01 00:00:00 +0000
+++ lava/test/commands.py	2012-04-02 16:50:25 +0000
@@ -0,0 +1,453 @@
+"""
+lava.test.commands
+====================
+
+LAVA command definitions used by lava-test
+"""
+
+import logging
+import os
+import shutil
+import subprocess
+
+from lava.tool.command import Command as LavaCommand, CommandGroup
+from lava.tool.errors import CommandError
+
+from lava_test.api.observers import (
+    ITestInstallerObserver,
+    ITestRunnerObserver)
+from lava_test.core.artifacts import TestArtifacts
+from lava_test.core.config import get_config
+from lava_test.core.loader import TestLoader
+from lava_test.utils import Cache
+
+
+__all__ = ["test", "list_", "install", "uninstall", "run", "parse", "show",
+           "register", "unregister", "reset"]
+
+
+class test(CommandGroup):
+    """
+    Access test commands.

+    LAVA Test is a generic test wrapper framework used by the LAVA stack to run
+    third party tests.
+    """
+
+    namespace = "lava.test.commands"
+
+
+class ObserverMethods(ITestInstallerObserver, ITestRunnerObserver):
+    """
+    Helper class that implements observer methods
+    """
+
+    def about_to_install_packages(self, package_list):
+        if self.args.quiet:
+            return
+        self.say("Installing packages: {0}", ", ".join(package_list))
+
+    def about_to_run_shell_command(self, cmd):
+        if self.args.quiet:
+            return
+        self.say("Running shell command: {0!r}", cmd)
+
+    def about_to_download_file(self, url):
+        if self.args.quiet:
+            return
+        self.say("Downloading file from: {0!r}", url)
+
+    def did_install_packages(self, package_list):
+        pass
+
+    def did_run_shell_command(self, cmd, returncode):
+        if returncode is None:
+            self.say("Command {0!r} was terminated prematurely", cmd)
+        elif returncode != 0:
+            self.say("Command {0!r} returned non-zero exit status {1}",
+                     cmd, returncode)
+
+    def did_download_file(self, url):
+        pass
+
+    def display_subprocess_output(self, stream_name, line):
+        if self.args.quiet_subcommands:
+            return
+        if stream_name == 'stdout':
+            self.say('(stdout) {0}', line.rstrip())
+        elif stream_name == 'stderr':
+            self.say('(stderr) {0}', line.rstrip())
+
+
+class Command(LavaCommand, ObserverMethods):
+
+    def __init__(self, parser, args):
+        super(Command, self).__init__(parser, args)
+        self._config = get_config()
+        self._test_loader = TestLoader(self._config)
+        # XXX: check how this interacts with top-level lava logger
+        if self.args.verbose:
+            logging.root.setLevel(logging.DEBUG)
+
+    @classmethod
+    def register_arguments(cls, parser):
+        parser.add_argument(
+           "-v", "--verbose",
+           action="store_true",
+           default=False,
+           help="Be verbose about undertaken actions")
+        parser.add_argument(
+            "-q", "--quiet",
+            action="store_true",
+            default=False,
+            help="Be less verbose about undertaken actions")
+        parser.add_argument(
+            "-Q", "--quiet-subcommands",
+            action="store_true",
+            default=False,
+            help="Hide the output of all sub-commands (including tests)")
+
+
+class TestAffectingCommand(Command):
+
+    INSTALL_REQUIRED = False
+
+    @classmethod
+    def register_arguments(cls, parser):
+        super(TestAffectingCommand, cls).register_arguments(parser)
+        parser.add_argument(
+            "test_id", help="Test or test suite identifier")
+
+    def invoke(self):
+        try:
+            test = self._test_loader[self.args.test_id]
+        except KeyError:
+            try:
+                test = self._test_loader[self.args.test_id.replace('-', '_')]
+            except KeyError:
+                raise CommandError(
+                    "There is no test with the specified ID")
+        return self.invoke_with_test(test)
+
+
+class list_(Command):
+    """
+    List available tests
+
+    .. program:: lava-test list-tests
+
+    Lists all available tests, grouping them by provider.
+    """
+
+    @classmethod
+    def get_name(cls):
+        return "list"
+
+    @classmethod
+    def register_arguments(cls, parser):
+        super(list_, cls).register_arguments(parser)
+        parser.add_argument(
+            "--installed",
+            action="store_true",
+            default=False,
+            help="Display only installed tests")
+
+    def invoke(self):
+        for provider in self._test_loader.get_providers():
+            test_list = [provider[test_id] for test_id in provider]
+            if not test_list:
+                continue
+            self.say("{0}", provider.description)
+            for test in test_list:
+                if self.args.installed and not test.is_installed:
+                    continue
+                self.say(" - {test_id}", test_id=test.test_id)
+
+
+class install(TestAffectingCommand):
+    """
+    Install a test program
+    """
+
+    def invoke_with_test(self, test):
+        if test.is_installed:
+            raise CommandError("This test is already installed")
+        try:
+            test.install(self)
+        except (subprocess.CalledProcessError, RuntimeError) as ex:
+            raise CommandError(str(ex))
+
+
+class uninstall(TestAffectingCommand):
+    """
+    Uninstall a test program
+    """
+
+    def invoke_with_test(self, test):
+        if not test.is_installed:
+            raise CommandError("This test is not installed")
+        test.uninstall()
+
+
+class run(TestAffectingCommand):
+    """
+    Run a previously installed test program
+    """
+
+    @classmethod
+    def register_arguments(cls, parser):
+        super(run, cls).register_arguments(parser)
+        group = parser.add_argument_group("initial bundle configuration")
+        group.add_argument(
+            "-S", "--skip-software-context",
+            default=False,
+            action="store_true",
+            help=("Do not store the software context in the"
+                  " initial bundle. Typically this saves OS"
+                  " image name and all the installed software"
+                  " packages."))
+        group.add_argument(
+            "-H", "--skip-hardware-context",
+            default=False,
+            action="store_true",
+            help=("Do not store the hardware context in the"
+                  " initial bundle. Typically this saves CPU,"
+                  " memory and USB device information."))
+        group.add_argument(
+            "--trusted-time",
+            default=False,
+            action="store_true",
+            help=("Indicate that the real time clock has"
+                  " accurate data. This can differentiate"
+                  " test results created on embedded devices"
+                  " that often have inaccurate real time"
+                  " clock settings."))
+        group.add_argument(
+            "--analyzer-assigned-uuid",
+            default=None,
+            metavar="UUID",
+            help=("Set the analyzer_assigned_uuid to the specified value."
+                  " This will prevent the test device from attempting"
+                  " to generate an UUID by itself. This option may be"
+                  " required if the test device has unreliable real"
+                  " time clock (no battery backed, not ensure to be"
+                  " up-to-date) and unreliable/random hardware ethernet "
+                  " address."))
+
+        group = parser.add_argument_group("complete bundle configuration")
+        group.add_argument(
+            "-o", "--output",
+            default=None,
+            metavar="FILE",
+            help=("After running the test parse the result"
+                  " artifacts, fuse them with the initial"
+                  " bundle and finally save the complete bundle"
+                  " to the  specified FILE."))
+        group.add_argument(
+            "-A", "--skip-attachments",
+            default=False,
+            action="store_true",
+            help=("Do not store standard output and standard"
+                  " error log files as attachments. This"
+                  " option is only affecting the bundle"
+                  " created with --output, the initial bundle"
+                  " is not affected as it never stores any"
+                  " attachments."))
+
+        parser.add_argument(
+            "-t", "--test-options",
+            default=None,
+            help=(
+                "Override the default test options."
+                " The value is passed verbatim to test definition. Typically"
+                " this is simply used in shell commands by expanding the"
+                " string $(OPTIONS). Please refer to the built-in"
+                " peacekeeper.py for examples. Depending on your shell you"
+                " probably have to escape spaces and other special"
+                " characters if you wish to include them in your argument"
+                " options."))
+
+    def invoke_with_test(self, test):
+        # Validate analyzer_assigned_uuid
+        if self.args.analyzer_assigned_uuid:
+            import uuid
+            try:
+                self.analyzer_assigned_uuid = str(
+                    uuid.UUID(self.args.analyzer_assigned_uuid))
+            except ValueError as exc:
+                self.parser.error("--analyzer-assigned-uuid: %s" % exc)
+        if not test.is_installed:
+            raise CommandError("The specified test is not installed")
+        try:
+            artifacts, run_fail = test.run(
+                self, test_options=self.args.test_options)
+        except subprocess.CalledProcessError as ex:
+            if ex.returncode is None:
+                raise CommandError("Command %r was aborted" % ex.cmd)
+            else:
+                raise CommandError(str(ex))
+        except RuntimeError as ex:
+            raise CommandError(str(ex))
+        self.say("run complete, result_id is {0!r}", artifacts.result_id)
+        try:
+            artifacts.create_initial_bundle(
+                self.args.skip_software_context,
+                self.args.skip_hardware_context,
+                self.args.trusted_time,
+                self.args.analyzer_assigned_uuid)
+        except ImportError as exc:
+            msg_template = (
+                "Unable to probe for software context. Install the '%s'"
+                " package or invoke lava-test run with"
+                " '--skip-software-context'")
+            if exc.message == "No module named apt":
+                raise CommandError(msg_template % "python-apt")
+            elif exc.message == "No module named lsb_release":
+                raise CommandError(msg_template % "lsb-release")
+            else:
+                raise
+        artifacts.save_bundle()
+        if self.args.output:
+            parse_results = test.parse(artifacts)
+            artifacts.incorporate_parse_results(parse_results)
+            if not self.args.skip_attachments:
+                artifacts.attach_standard_files_to_bundle()
+            artifacts.save_bundle_as(self.args.output)
+        if run_fail:
+            raise CommandError(
+                'Some of test steps returned non-zero exit code')
+
+
+class parse(TestAffectingCommand):
+    """
+    Parse the results of previous test run
+    """
+
+    @classmethod
+    def register_arguments(cls, parser):
+        super(parse, cls).register_arguments(parser)
+        parser.add_argument(
+            "result_id",
+            help="Test run result identifier")
+        group = parser.add_argument_group("complete bundle configuration")
+        group.add_argument(
+            "-o", "--output",
+            default=None,
+            metavar="FILE",
+            help=("After running the test parse the result"
+                  " artifacts, fuse them with the initial"
+                  " bundle and finally save the complete bundle"
+                  " to the  specified FILE."))
+        group.add_argument(
+            "-A", "--skip-attachments",
+            default=False,
+            action="store_true",
+            help=("Do not store standard output and standard"
+                  " error log files as attachments. This"
+                  " option is only affecting the bundle"
+                  " created with --output, the initial bundle"
+                  " is not affected as it never stores any"
+                  " attachments."))
+
+    def invoke_with_test(self, test):
+        artifacts = TestArtifacts(
+            self.args.test_id, self.args.result_id, self._config)
+        self.say("Loading test data from {0}", artifacts.bundle_pathname)
+        if not os.path.exists(artifacts.bundle_pathname):
+            raise CommandError("Specified result does not exist")
+        artifacts.load_bundle()
+        parse_results = test.parse(artifacts)
+        artifacts.incorporate_parse_results(parse_results)
+        self.say("Parsed {0} test results",
+                 len(artifacts.bundle["test_runs"][0]["test_results"]))
+        logging.debug(artifacts.dumps_bundle())
+        if self.args.output:
+            if not self.args.skip_attachments:
+                artifacts.attach_standard_files_to_bundle()
+            artifacts.save_bundle_as(self.args.output)
+
+
+class show(Command):
+    """
+    Display the output from a previous test run
+    """
+
+    @classmethod
+    def register_arguments(cls, parser):
+        super(show, cls).register_arguments(parser)
+        parser.add_argument(
+            "result_id", help="Test run result identifier")
+
+    def invoke(self):
+        artifacts = TestArtifacts(None, self.args.result_id, self._config)
+        if not os.path.exists(artifacts.results_dir):
+            raise CommandError("Specified result does not exist")
+        if os.path.exists(artifacts.stdout_pathname):
+            with open(artifacts.stdout_pathname, "rt") as stream:
+                for line in iter(stream.readline, ''):
+                    self.display_subprocess_output("stdout", line)
+        if os.path.exists(artifacts.stderr_pathname):
+            with open(artifacts.stderr_pathname, "rt") as stream:
+                for line in iter(stream.readline, ''):
+                    self.display_subprocess_output("stderr", line)
+
+
+class register(Command):
+    """
+    Register remote test
+    """
+
+    @classmethod
+    def register_arguments(cls, parser):
+        super(register, cls).register_arguments(parser)
+        parser.add_argument(
+            "test_url", help="Url for test definition file")
+
+    def invoke(self):
+        try:
+            from lava_test.core.providers import RegistryProvider
+            RegistryProvider.register_remote_test(self.args.test_url)
+        except ValueError as exc:
+            raise CommandError("Unable to register test: %s" % exc)
+        except KeyError:
+            raise CommandError("There is no test_url")
+
+
+class unregister(Command):
+    """
+    Remove a declarative test from the registry

+    This command does the reverse of lava-test register. You need to pass the
+    same URL you've used in `lava-test register-test`
+    """
+
+    @classmethod
+    def register_arguments(cls, parser):
+        super(unregister, cls).register_arguments(parser)
+        parser.add_argument(
+            "url",
+            metavar="URL",
+            help="URL of the test definition file")
+
+    def invoke(self):
+        from lava_test.core.providers import RegistryProvider
+        try:
+            RegistryProvider.unregister_remote_test(self.args.url)
+        except ValueError:
+            raise CommandError("This test is not registered")
+
+
+class reset(Command):
+    """
+    Reset the lava-test environment by removing cached items, all registered
+    simple declarative tests and configuration files
+    """
+
+    def invoke(self):
+        self.say("Deleting {0}", self._config.configdir)
+        shutil.rmtree(self._config.configdir, ignore_errors=True)
+        self.say("Deleting {0}", self._config.installdir)
+        shutil.rmtree(self._config.installdir, ignore_errors=True)
+        self.say("Deleting {0}", self._config.resultsdir)
+        shutil.rmtree(self._config.resultsdir, ignore_errors=True)
+        cache = Cache.get_instance()
+        self.say("Deleting {0}", cache.cache_dir)
+        shutil.rmtree(cache.cache_dir, ignore_errors=True)

=== modified file 'lava_test/commands.py'
--- lava_test/commands.py	2012-03-19 19:24:05 +0000
+++ lava_test/commands.py	2012-04-02 16:50:25 +0000
@@ -400,12 +400,10 @@
         import lava_tool
         import lava_test
         import linaro_dashboard_bundle
-        import linaro_json
         return [
             lava_test,
             lava_tool,
-            linaro_dashboard_bundle,
-            linaro_json]
+            linaro_dashboard_bundle]
 
 
 class register_test(Command):

=== modified file 'setup.py'
--- setup.py	2012-03-02 14:21:13 +0000
+++ setup.py	2012-04-02 16:50:25 +0000
@@ -32,6 +32,18 @@
     entry_points="""
     [console_scripts]
     lava-test=lava_test.main:main
+    [lava.commands]
+    test=lava.test.commands:test
+    [lava.test.commands]
+    list=lava.test.commands:list_
+    install=lava.test.commands:install
+    uninstall=lava.test.commands:uninstall
+    run=lava.test.commands:run
+    parse=lava.test.commands:parse
+    show=lava.test.commands:show
+    register=lava.test.commands:register
+    unregister=lava.test.commands:unregister
+    reset=lava.test.commands:reset
     [lava_test.commands]
     version=lava_test.commands:version
     list-tests=lava_test.commands:list_tests