Merge lp:~pwlars/lava-test/results into lp:lava-test/0.0

[Paul Larson]

  Make a new 'results' module for abrek that contains the subcommands:
   * list
   * rm
   * rename
   * show

  This changes a single command (list-results to results list) and extends
  functionality to further operate on locally stored results

[James Westby (james-w)]

17 - test = abrek.testdef.testloader(testdata['testname'])
18 + test = abrek.testdef.testloader(testdata['test_id'])

This looks unrelated to the other changes in the branch.

Is there a test for it?

57 + from abrek import (builtins, results)

No need for brackets there.

Two lines between classes in the new files please.

Has the set_config to fake the config, or the stdout capturing been
done in other tests?

I would also appreciate tests for failure modes in the non-rename
cases.

Thanks,

James

review: Needs Fixing

[lp:~pwlars/lava-test/results]

27. By Paul Larson on 2010-08-26

remove unneeded parens

28. By Paul Larson on 2010-08-26

style change, add extra blank line between classes

29. By Paul Larson on 2010-08-26

Handle calling results show for a result that doesn't exist

30. By Paul Larson on 2010-08-26

Change error message for consistency

31. By Paul Larson on 2010-08-26

* Split out the FakeConfigTests and FakeOutputTests
* Add several more tests to cover failure cases for results command

[Paul Larson (pwlars)]

On Wed, 2010-08-25 at 14:13 +0000, James Westby wrote:
> Review: Needs Fixing
> 17 - test = abrek.testdef.testloader(testdata['testname'])
> 18 + test = abrek.testdef.testloader(testdata['test_id'])
>
> This looks unrelated to the other changes in the branch.
Yep, I forgot to check it in separately. It's in that parse command
that I use for testing output. The format changed a while back, and
it's one place I missed the change. I plan on removing it eventually,
though I've thought about perhaps keeping it in some form. I'm thinking
when I get through some of the important things I need to add, I may
come back and add a --json or something like that to the results output,
that would spit out the bundle in json as it would be submitted to the
dashboard. I'll add tests for it when that happens. For now, this is
not something that would typically be used when running abrek under
normal conditions.

> 57 + from abrek import (builtins, results)
>
> No need for brackets there.
Fixed

> Two lines between classes in the new files please.
Fixed, and I went ahead and did it for other files as well (sorry if it
increased the diff size, but those should just show up as blank lines in
the patch, so easy to skip past I'm thinking)

> Has the set_config to fake the config, or the stdout capturing been
> done in other tests?
I split off FakeConfigTests and FakeOutputTests into a separate file for
both of those tests to import, since it's used in more than one place
now.

> I would also appreciate tests for failure modes in the non-rename
> cases.
Done

[Paul Larson (pwlars)]

review: Needs Resubmitting

=== modified file 'abrek/builtins.py'
--- abrek/builtins.py	2010-08-18 15:33:32 +0000
+++ abrek/builtins.py	2010-08-26 20:53:40 +0000
@@ -20,6 +20,8 @@
 
 import abrek.command
 import abrek.testdef
+from abrek.config import get_config
+from abrek.utils import read_file
 
 
 class cmd_version(abrek.command.AbrekCmd):
@@ -30,6 +32,7 @@
         import abrek
         print abrek.__version__
 
+
 class cmd_help(abrek.command.AbrekCmd):
     """ Get help on abrek commands
 
@@ -52,6 +55,7 @@
             else:
                 print "No command found for '%s'" % self.args[0]
 
+
 class cmd_install(abrek.command.AbrekCmd):
     """
     Install a test
@@ -69,6 +73,7 @@
             print "Test installation error: %s" % strerror
             sys.exit(1)
 
+
 class cmd_run(abrek.command.AbrekCmd):
     """
     Run tests
@@ -86,6 +91,7 @@
             print "Test execution error: %s" % strerror
             sys.exit(1)
 
+
 class cmd_parse(abrek.command.AbrekCmd):
     def run(self):
         if len(self.args) != 1:
@@ -95,7 +101,7 @@
         resultsdir = os.path.join(config.resultsdir,self.args[0])
         testdatafile = os.path.join(resultsdir,"testdata.json")
         testdata = json.loads(file(testdatafile,'r').read())
-        test = abrek.testdef.testloader(testdata['testname'])
+        test = abrek.testdef.testloader(testdata['test_id'])
         try:
             test.parse(self.args[0])
         except Exception as strerror:
@@ -103,6 +109,7 @@
             sys.exit(1)
         print test.parser.results
 
+
 class cmd_uninstall(abrek.command.AbrekCmd):
     """
     Uninstall a test
@@ -120,12 +127,12 @@
             print "Test uninstall error: %s" % strerror
             sys.exit(1)
 
+
 class cmd_list_installed(abrek.command.AbrekCmd):
     """
     List tests that are currently installed
     """
     def run(self):
-        from abrek.config import get_config
         config = get_config()
         print "Installed tests:"
         try:
@@ -134,6 +141,7 @@
         except OSError:
             print "No tests installed"
 
+
 class cmd_list_tests(abrek.command.AbrekCmd):
     """
     List all known tests
@@ -145,16 +153,3 @@
         for importer, mod, ispkg in walk_packages(test_definitions.__path__):
             print mod
 
-class cmd_list_results(abrek.command.AbrekCmd):
-    """
-    List results of previous runs
-    """
-    def run(self):
-        from abrek.config import get_config
-        config = get_config()
-        print "Saved results:"
-        try:
-            for dir in os.listdir(config.resultsdir):
-                print dir
-        except OSError:
-            print "No results found"

=== modified file 'abrek/command.py'
--- abrek/command.py	2010-08-18 15:33:32 +0000
+++ abrek/command.py	2010-08-26 20:53:40 +0000
@@ -15,6 +15,7 @@
 
 from optparse import OptionParser
 
+
 class _AbrekOptionParser(OptionParser):
     """
     This is just to override the epilog formatter to allow newlines
@@ -22,6 +23,7 @@
     def format_epilog(self, formatter):
         return self.epilog
 
+
 class AbrekCmd(object):
     """ Base class for commands that can be passed to Abrek.
 
@@ -115,8 +117,9 @@
     return cmds
 
 def get_all_cmds():
-    import abrek.builtins
-    cmds = _find_commands(abrek.builtins)
+    from abrek import builtins, results
+    cmds = _find_commands(builtins)
+    cmds.update(_find_commands(results))
     return cmds
 
 def get_command(cmd_name):

=== modified file 'abrek/config.py'
--- abrek/config.py	2010-08-18 15:33:32 +0000
+++ abrek/config.py	2010-08-26 20:53:40 +0000
@@ -15,6 +15,7 @@
 
 import os
 
+
 class AbrekConfig(object):
     def __init__(self):
         home = os.environ.get('HOME', '/')

=== modified file 'abrek/main.py'
--- abrek/main.py	2010-08-18 15:33:32 +0000
+++ abrek/main.py	2010-08-26 20:53:40 +0000
@@ -16,6 +16,7 @@
 import sys
 import abrek.command
 
+
 def main(argv=None):
     if argv is None:
         argv = sys.argv[1:]

=== added file 'abrek/results.py'
--- abrek/results.py	1970-01-01 00:00:00 +0000
+++ abrek/results.py	2010-08-26 20:53:40 +0000
@@ -0,0 +1,111 @@
+# Copyright (c) 2010 Linaro
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import sys
+from optparse import make_option
+
+from abrek.command import AbrekCmd
+from abrek.config import get_config
+from abrek.utils import read_file
+
+
+class subcmd_results_list(AbrekCmd):
+    """
+    List results of previous runs
+    """
+    def run(self):
+        config = get_config()
+        print "Saved results:"
+        try:
+            for dir in os.listdir(config.resultsdir):
+                print dir
+        except OSError:
+            print "No results found"
+
+
+class subcmd_results_show(AbrekCmd):
+    """
+    Display the output from a previous test run
+    """
+    arglist = ['*result']
+    def run(self):
+        if len(self.args) != 1:
+            print "please specify the name of the result dir"
+            sys.exit(1)
+        config = get_config()
+        resultsdir = os.path.join(config.resultsdir,self.args[0])
+        testoutput = os.path.join(resultsdir,"testoutput.log")
+        if not os.path.exists(testoutput):
+            print "No result found for '%s'" % self.args[0]
+            sys.exit(1)
+        print(read_file(testoutput))
+
+
+class subcmd_results_remove(AbrekCmd):
+    """
+    Remove the results of a previous test run
+    """
+    arglist = ['*result']
+    options = [make_option('-f', '--force', action='store_true',
+                           dest='force')]
+    def run(self):
+        if len(self.args) != 1:
+            print "please specify the name of the result dir"
+            sys.exit(1)
+        config = get_config()
+        resultsdir = os.path.join(config.resultsdir,self.args[0])
+        if not os.path.exists(resultsdir):
+            print "No result found for '%s'" % self.args[0]
+            sys.exit(1)
+        if not self.opts.force:
+            print "Delete result '%s' for good? [Y/N]" % self.args[0],
+            response = raw_input()
+            if response[0].upper() != 'Y':
+                sys.exit(0)
+        shutil.rmtree(resultsdir)
+
+
+class subcmd_results_rename(AbrekCmd):
+    """
+    Rename the results from a previous test run
+    """
+    arglist = ['*source', '*destination']
+
+    def run(self):
+        if len(self.args) != 2:
+            print "please specify the name of the result, and the new name"
+            sys.exit(1)
+        config = get_config()
+        srcdir = os.path.join(config.resultsdir,self.args[0])
+        destdir = os.path.join(config.resultsdir,self.args[1])
+        if not os.path.exists(srcdir):
+            print "Result directory not found"
+            sys.exit(1)
+        if os.path.exists(destdir):
+            print "Destination result name already exists"
+            sys.exit(1)
+        shutil.move(srcdir, destdir)
+
+
+class cmd_results(AbrekCmd):
+    """
+    Operate on results of previous test runs stored locally
+    """
+    subcmds = {'list':subcmd_results_list(),
+               'rm':subcmd_results_remove(),
+               'rename':subcmd_results_rename(),
+               'show':subcmd_results_show()}

=== modified file 'abrek/testdef.py'
--- abrek/testdef.py	2010-08-18 15:33:32 +0000
+++ abrek/testdef.py	2010-08-26 20:53:40 +0000
@@ -29,6 +29,7 @@
 from abrek import hwprofile
 from abrek import swprofile
 
+
 class AbrekTest(object):
     """Base class for defining tests.
 
@@ -124,6 +125,7 @@
         os.chdir(self.resultsdir)
         self.parser.parse()
 
+
 class AbrekTestInstaller(object):
     """Base class for defining an installer object.
 
@@ -182,6 +184,7 @@
         self._download()
         self._runsteps()
 
+
 class AbrekTestRunner(object):
     """Base class for defining an test runner object.
 
@@ -209,6 +212,7 @@
         self._runsteps(resultsdir)
         self.endtime = datetime.utcnow()
 
+
 class AbrekTestParser(object):
     """Base class for defining a test parser
 

=== modified file 'tests/__init__.py'
--- tests/__init__.py	2010-08-18 15:33:32 +0000
+++ tests/__init__.py	2010-08-26 20:53:40 +0000
@@ -22,7 +22,8 @@
                     'tests.test_abrektestrunner',
                     'tests.test_abrektestparser',
                     'tests.test_swprofile',
-                    'tests.test_hwprofile']
+                    'tests.test_hwprofile',
+                    'tests.test_results']
     loader = unittest.TestLoader()
     suite = loader.loadTestsFromNames(module_names)
     return suite

=== added file 'tests/faketests.py'
--- tests/faketests.py	1970-01-01 00:00:00 +0000
+++ tests/faketests.py	2010-08-26 20:53:40 +0000
@@ -0,0 +1,47 @@
+# Copyright (c) 2010 Linaro
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+import StringIO
+
+from abrek.config import set_config
+
+class FakeOutputTests(unittest.TestCase):
+    def setUp(self):
+        self.origstdout = sys.stdout
+        sys.stdout = self.fakestdout = StringIO.StringIO()
+
+    def tearDown(self):
+        sys.stdout = self.origstdout
+
+class FakeConfigTests(FakeOutputTests):
+    def setUp(self):
+        super(FakeConfigTests, self).setUp()
+        class fakeconfig:
+            def __init__(self, basedir):
+                self.configdir = os.path.join(basedir, "config")
+                self.installdir = os.path.join(basedir, "install")
+                self.resultsdir = os.path.join(basedir, "results")
+        self.tmpdir = tempfile.mkdtemp()
+        self.config = fakeconfig(self.tmpdir)
+        set_config(self.config)
+
+    def tearDown(self):
+        super(FakeConfigTests, self).tearDown()
+        shutil.rmtree(self.tmpdir)

=== modified file 'tests/test_builtins.py'
--- tests/test_builtins.py	2010-08-18 15:33:32 +0000
+++ tests/test_builtins.py	2010-08-26 20:53:40 +0000
@@ -14,22 +14,9 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import shutil
-import sys
-import tempfile
-import unittest
-import StringIO
 
 import abrek.builtins
-from abrek.config import set_config
-
-class FakeOutputTests(unittest.TestCase):
-    def setUp(self):
-        self.origstdout = sys.stdout
-        sys.stdout = self.fakestdout = StringIO.StringIO()
-
-    def tearDown(self):
-        sys.stdout = self.origstdout
+from faketests import FakeConfigTests, FakeOutputTests
 
 class ListKnown(FakeOutputTests):
     def test_list_tests(self):
@@ -37,32 +24,10 @@
         cmd.run()
         self.assertTrue("stream" in self.fakestdout.getvalue())
 
-class FakeConfigTests(FakeOutputTests):
-    def setUp(self):
-        super(FakeConfigTests, self).setUp()
-        class fakeconfig:
-            def __init__(self, basedir):
-                self.configdir = os.path.join(basedir, "config")
-                self.installdir = os.path.join(basedir, "install")
-                self.resultsdir = os.path.join(basedir, "results")
-        self.tmpdir = tempfile.mkdtemp()
-        self.config = fakeconfig(self.tmpdir)
-        set_config(self.config)
-
-    def tearDown(self):
-        super(FakeConfigTests, self).tearDown()
-        shutil.rmtree(self.tmpdir)
-
+class ListInstalled(FakeConfigTests):
     def test_list_installed(self):
         test_name="test_list_installed000"
         os.makedirs(os.path.join(self.config.installdir, test_name))
         cmd = abrek.builtins.cmd_list_installed()
         cmd.run()
         self.assertTrue(test_name in self.fakestdout.getvalue())
-
-    def test_list_results(self):
-        result_name = "test_list_results000"
-        os.makedirs(os.path.join(self.config.resultsdir, result_name))
-        cmd = abrek.builtins.cmd_list_results()
-        cmd.run()
-        self.assertTrue(result_name in self.fakestdout.getvalue())

=== added file 'tests/test_results.py'
--- tests/test_results.py	1970-01-01 00:00:00 +0000
+++ tests/test_results.py	2010-08-26 20:53:40 +0000
@@ -0,0 +1,95 @@
+import os
+
+import abrek.results
+from abrek.utils import write_file
+from faketests import FakeConfigTests
+
+class ResultsTests(FakeConfigTests):
+    def test_results_list(self):
+        result_name = "test_results_list000"
+        os.makedirs(os.path.join(self.config.resultsdir, result_name))
+        cmd = abrek.results.subcmd_results_list()
+        cmd.run()
+        self.assertTrue(result_name in self.fakestdout.getvalue())
+
+    def test_results_list_nodir(self):
+        errmsg = "No results found"
+        cmd = abrek.results.subcmd_results_list()
+        cmd.run()
+        self.assertTrue(errmsg in self.fakestdout.getvalue())
+
+    def test_results_show(self):
+        result_name = "test_results_show000"
+        result_output = "test result output"
+        result_dir = os.path.join(self.config.resultsdir, result_name)
+        os.makedirs(result_dir)
+        outputfile = os.path.join(result_dir, 'testoutput.log')
+        write_file(result_output, outputfile)
+        cmd = abrek.results.subcmd_results_show()
+        cmd.main(argv=[result_name])
+        self.assertEqual(result_output, self.fakestdout.getvalue().strip())
+
+    def test_results_show_noarg(self):
+        errmsg = "please specify the name of the result dir"
+        cmd = abrek.results.subcmd_results_show()
+        self.assertRaises(SystemExit, cmd.main, argv=[])
+        self.assertEqual(errmsg, self.fakestdout.getvalue().strip())
+
+    def test_results_show_nodir(self):
+        testname = "foo"
+        errmsg = "No result found for '%s'" % testname
+        cmd = abrek.results.subcmd_results_show()
+        self.assertRaises(SystemExit, cmd.main, argv=[testname])
+        self.assertEqual(errmsg, self.fakestdout.getvalue().strip())
+
+    def test_results_remove(self):
+        result_name = "test_results_remove000"
+        result_dir = os.path.join(self.config.resultsdir, result_name)
+        os.makedirs(result_dir)
+        cmd = abrek.results.subcmd_results_remove()
+        cmd.main(argv=[result_name, '-f'])
+        self.assertFalse(os.path.exists(result_dir))
+
+    def test_results_remove_noarg(self):
+        errmsg = "please specify the name of the result dir"
+        cmd = abrek.results.subcmd_results_remove()
+        self.assertRaises(SystemExit, cmd.main, argv=[])
+        self.assertEqual(errmsg, self.fakestdout.getvalue().strip())
+
+    def test_results_remove_nodir(self):
+        testname = "foo"
+        errmsg = "No result found for '%s'" % testname
+        cmd = abrek.results.subcmd_results_remove()
+        self.assertRaises(SystemExit, cmd.main, argv=[testname])
+        self.assertEqual(errmsg, self.fakestdout.getvalue().strip())
+
+    def test_results_rename(self):
+        result_src = "test_results_old"
+        result_dest = "test_results_new"
+        result_srcdir = os.path.join(self.config.resultsdir, result_src)
+        result_destdir = os.path.join(self.config.resultsdir, result_dest)
+        os.makedirs(result_srcdir)
+        cmd = abrek.results.subcmd_results_rename()
+        cmd.main(argv=[result_src, result_dest])
+        self.assertFalse(os.path.exists(result_srcdir))
+        self.assertTrue(os.path.exists(result_destdir))
+
+    def test_results_rename_badsrc(self):
+        errmsg = "Result directory not found"
+        result_src = "test_results_old"
+        result_dest = "test_results_new"
+        cmd = abrek.results.subcmd_results_rename()
+        self.assertRaises(SystemExit, cmd.main, argv=[result_src, result_dest])
+        self.assertEqual(errmsg, self.fakestdout.getvalue().strip())
+
+    def test_results_rename_baddest(self):
+        errmsg = "Destination result name already exists"
+        result_src = "test_results_old"
+        result_dest = "test_results_new"
+        result_srcdir = os.path.join(self.config.resultsdir, result_src)
+        result_destdir = os.path.join(self.config.resultsdir, result_dest)
+        os.makedirs(result_srcdir)
+        os.makedirs(result_destdir)
+        cmd = abrek.results.subcmd_results_rename()
+        self.assertRaises(SystemExit, cmd.main, argv=[result_src, result_dest])
+        self.assertEqual(errmsg, self.fakestdout.getvalue().strip())