Merge lp:~nuclearbob/utah/getfiles into lp:utah

[Max Brustkern]

This branch adds -f and -o options to run_utah_tests.py. -f specified a file or directory to retrieve from the provisioned system, and can be specified multiple times. -o specified the output directory, which defaults to /var/log/utah/{systemname} and will be created if necessary.

[Max Brustkern (nuclearbob)]

There's also a fix for some logic for automatically downloading ISOs that I made while testing this. It's very small, so I wasn't sure if it needed a new branch.

[Joe Talbott (joetalbott)]

Looks good to me. Thanks.

review: Approve

=== modified file 'examples/run_install_test.py'
--- examples/run_install_test.py	2012-11-08 14:51:09 +0000
+++ examples/run_install_test.py	2012-11-16 20:52:18 +0000
@@ -65,6 +65,11 @@
                         help='Enable debug logging')
     parser.add_argument('-j', '--json', action='store_true',
                         help='Enable json logging (default is YAML)')
+    parser.add_argument('-f', '--files', action='append',
+                        help='File or directory to copy from test system ')
+    parser.add_argument('-o', '--outdir', 
+                        help=('Directory to store locally copied files '
+                         '(Default is ' + config.logpath + '/machine-name)'))
     parser.add_argument('--diskbus', metavar='DISKBUS',
                         choices=('virtio', 'sata', 'ide'),
                         help=('Disk bus to use for customvm installation '

=== modified file 'examples/run_test_cobbler.py'
--- examples/run_test_cobbler.py	2012-11-14 19:56:48 +0000
+++ examples/run_test_cobbler.py	2012-11-16 20:52:18 +0000
@@ -46,6 +46,11 @@
                         help='Enable debug logging')
     parser.add_argument('-j', '--json', action='store_true',
                         help='Enable json logging (default is YAML)')
+    parser.add_argument('-f', '--files', action='append',
+                        help='File or directory to copy from test system ')
+    parser.add_argument('-o', '--outdir', 
+                        help=('Directory to store locally copied files '
+                         '(Default is ' + config.logpath + '/machine-name)'))
     parser.add_argument('--rewrite', choices=('all', 'minimal', 'casperonly',
                         'none'), help='Enable or disable automatic '
                         'configuration rewriting')

=== modified file 'examples/run_test_vm.py'
--- examples/run_test_vm.py	2012-11-14 16:14:38 +0000
+++ examples/run_test_vm.py	2012-11-16 20:52:18 +0000
@@ -40,6 +40,11 @@
                         help='Enable debug logging')
     parser.add_argument('-j', '--json', action='store_true',
                         help='Enable json logging (default is YAML)')
+    parser.add_argument('-f', '--files', action='append',
+                        help='File or directory to copy from test system ')
+    parser.add_argument('-o', '--outdir', 
+                        help=('Directory to store locally copied files '
+                         '(Default is ' + config.logpath + '/machine-name)'))
     return parser
 
 

=== modified file 'examples/run_utah_tests.py'
--- examples/run_utah_tests.py	2012-11-14 18:12:13 +0000
+++ examples/run_utah_tests.py	2012-11-16 20:52:18 +0000
@@ -66,6 +66,11 @@
                         help='Enable debug logging')
     parser.add_argument('-j', '--json', action='store_true',
                         help='Enable json logging (default is YAML)')
+    parser.add_argument('-f', '--files', action='append',
+                        help='File or directory to copy from test system ')
+    parser.add_argument('-o', '--outdir', 
+                        help=('Directory to store locally copied files '
+                         '(Default is ' + config.logpath + '/machine-name)'))
     parser.add_argument('--diskbus', metavar='DISKBUS',
                         choices=('virtio', 'sata', 'ide'),
                         help=('Disk bus to use for customvm installation '

=== modified file 'utah/provisioning/provisioning.py'
--- utah/provisioning/provisioning.py	2012-11-14 18:12:54 +0000
+++ utah/provisioning/provisioning.py	2012-11-16 20:52:18 +0000
@@ -17,7 +17,11 @@
 import paramiko
 import re
 import apt.cache
+
 from glob import glob
+from stat import S_ISDIR
+
+import utah.timeout
 
 from utah.commandstr import commandstr
 from utah.iso import ISO
@@ -28,7 +32,6 @@
 from utah.preseed import Preseed
 from utah.provisioning.exceptions import UTAHProvisioningException
 from utah.retry import retry
-import utah.timeout
 from utah import config
 
 
@@ -152,9 +155,7 @@
 
         if image is None:
             image = config.image
-        if image is None:
-            self.image = None
-        elif image.endswith('.iso'):
+        if image is None or image.endswith('.iso'):
             self.image = ISO(arch=arch,
                     dlpercentincrement=self.dlpercentincrement,
                     image=image, installtype=installtype, logger=self.logger,
@@ -742,19 +743,39 @@
         finally:
             sftp_client.close()
 
-    def downloadfilesindir(self, dirname, target=os.path.normpath('/tmp/')):
-        """
-        Copy all files in dirname on the machine to target on the host.
-        Recursion is not yet implemented.
-        """
-        # TODO: Implement recursion
+    def downloadfilesrecursive(self, files, target=os.path.normpath('/tmp/')):
+        """
+        Recursively copy all files in files to the target directory target.
+        """
         self.activecheck()
         self.ssh_client.connect(self.name,
                                 key_filename=config.sshprivatekey)
         sftp_client = self.ssh_client.open_sftp()
-        files = [os.path.join(dirname, myfile)
-                 for myfile in sftp_client.listdir(dirname)]
-        self.downloadfiles(files, target)
+        myfiles = []
+        
+        if isinstance(files, basestring):
+            files = [files]
+        
+        for myfile in files:
+            newtarget = os.path.join(target, os.path.basename(myfile))
+            if S_ISDIR(sftp_client.stat(myfile).st_mode):
+                self.logger.debug(myfile + ' is a directory, recursing')
+                if not os.path.isdir(newtarget):
+                     self.logger.debug('Attempting to create ' + newtarget)
+                     os.makedirs(newtarget)
+                myfiles = [os.path.join(myfile, x) 
+                           for x in sftp_client.listdir(myfile)]
+#                for basename in sftp_client.listdir(dirname):
+#                    myfile = os.path.join(dirname, basename)
+#                    if S_ISDIR(sftp_client.stat(myfile).st_mode):
+#                        if not os.path.isdir(newtarget):
+#                            os.makedirs(newtarget)
+#                        self.downloadfilesrecursive(myfile, newtarget)
+#                    else:
+#                        myfiles.append(myfile)
+                self.downloadfilesrecursive(myfiles, newtarget)
+            else:
+                self.downloadfiles(myfile, newtarget)
 
     def destroy(self, *args, **kw):
         """

=== modified file 'utah/run.py'
--- utah/run.py	2012-10-25 11:54:11 +0000
+++ utah/run.py	2012-11-16 20:52:18 +0000
@@ -59,4 +59,31 @@
         machine.logger.info('Test log copied to ' + locallog)
         locallogs.append(locallog)
 
+    if args.files is not None:
+        try:
+            locallogs += getfiles(args, machine)
+        except Exception as err:
+            machine.logger.warning('Failed to download files: ' + str(err))
+
     return exitstatus, locallogs
+    
+    
+def getfiles(args, machine):
+    if args.outdir is None:
+        outdir = os.path.join(config.logpath, machine.name)
+    else:
+        outdir = args.outdir
+    if not os.path.isdir(outdir):
+        os.makedirs(outdir)
+
+    machine.downloadfilesrecursive(args.files, outdir)
+    
+    localfiles = []
+    
+    for mydir, dirs, files in os.walk(outdir):
+        myfiles = [os.path.join(mydir, x) for x in files]
+        for myfile in myfiles:
+            localfiles.append(myfile)
+
+    return localfiles
+