Merge lp:~deeptik/linaro-license-protection/publish-to-snapshots into lp:~linaro-automation/linaro-license-protection/trunk

[Deepti B. Kalakeri]

Adding changes to create manifest file and a symlink called lastSuccessful.
Adding tests to address the changes to create manifest file and a symlink called lastSuccessful

[Paul Sokolovsky (pfalcon)]

symlink creation logic is not exactly correct. It should not scan for "latest" dir, it should use current build it was passed as such, it obviously the latest in time, and that's what we want to be linked.

review: Needs Fixing

[Deepti B. Kalakeri (deeptik)]

Ah! correct, I also had the same thing in mind. But not sure why I changed it to search for latest dir. Thanks I will submit the changes.

Regards,
Deepti.

[lp:~deeptik/linaro-license-protection/publish-to-snapshots]

51. By Deepti B. Kalakeri on 2012-03-26

Removed get_latest_dir, symlink now points to the build that was just copied

[Paul Sokolovsky (pfalcon)]

Rev 51 looks good, thanks.

review: Approve

[Deepti B. Kalakeri (deeptik)]

On Mon, Mar 26, 2012 at 4:36 PM, Paul Sokolovsky <<email address hidden>
> wrote:

> Review: Approve
>
> Rev 51 looks good, thanks.
>

Thanks a lot for the review

> --
>
> https://code.launchpad.net/~deeptik/linaro-license-protection/publish-to-snapshots/+merge/99221<https://code.launchpad.net/%7Edeeptik/linaro-license-protection/publish-to-snapshots/+merge/99221>
> You are the owner of
> lp:~deeptik/linaro-license-protection/publish-to-snapshots.
>

--
Thanks and Regards,
Deepti

=== modified file 'scripts/publish_to_snapshots.py'
--- scripts/publish_to_snapshots.py	2012-03-16 10:55:32 +0000
+++ scripts/publish_to_snapshots.py	2012-03-26 10:49:17 +0000
@@ -16,6 +16,8 @@
                     kernel-hwpack ${KERNEL_JOB_NAME}")
 parser.add_argument("-n", "--build-num", dest="build_num", type=int,
                     help="Specify the job build number for android builds only")
+parser.add_argument("-m", "--manifest", dest="manifest", action='store_true',
+                    help="Optional parameter to generate MANIFEST file")
 
 uploads_path = '/srv3/snapshots.linaro.org/uploads/'
 target_path = '/srv3/snapshots.linaro.org/www/'
@@ -83,29 +85,87 @@
 
         return build_dir_path, target_dir_path
 
+    def create_symlink(self, target_dir_path):
+        target_parent_dir = os.path.dirname(target_dir_path)
+        symlink_path = os.path.join(target_parent_dir, "lastSuccessful")
+        try:
+            if os.path.islink(symlink_path):
+                os.unlink(symlink_path)
+
+            os.symlink(target_dir_path, symlink_path)
+            print "The lastSuccessful build is now linked to ", target_dir_path
+            return PASS
+        except Exception, details:
+            print "Failed to create symlink", symlink_path, ":", details
+            return FAIL
+
+    def create_manifest_file(self, target_dir):
+        orig_dir=os.getcwd()
+        os.chdir(target_dir)
+        fn = os.path.join(target_dir, "MANIFEST")
+        lines = []
+
+        try:
+            for path, subdirs, files in os.walk("."):
+                for name in files:
+                    lines.append(os.path.join(path, name).split("./")[1] + "\n")
+            
+            if len(lines) != 0:
+                fd = open(fn, "w+")
+                for line in lines:
+                    if not "MANIFEST" in line:
+                        fd.write(line)
+                fd.close()
+            else:
+                raise Exception("Uploads directory was empty, nothing got moved to destination")
+
+            os.chdir(orig_dir)
+
+            if os.path.isfile(fn):
+                print "Manifest file", fn, "generated"
+                return PASS
+        except Exception, details:
+            print "Got Exception in create_manifest_file: ", details
+            os.chdir(orig_dir)
+            return FAIL
+
+
+    def move_dir_content(self, src_dir, dest_dir):
+        filelist = os.listdir(src_dir)
+        try:
+            for file in filelist:
+                src = os.path.join(src_dir, file)
+                dest = os.path.join(dest_dir, file)
+                if os.path.exists(dest):
+                    if os.path.isdir(dest):
+                        self.move_dir_content(src, dest)
+                        continue
+                    else:
+                        os.remove(dest)
+                shutil.move(src, dest)
+        except shutil.Error:
+            print "Error while moving the content"
 
     def move_artifacts(self, args, build_dir_path, target_dir_path):
         try:
-            # Make a note of the contents of src dir so that 
-            # it can be used to validate the move to destination
-            uploads_dir_list = os.listdir(build_dir_path)
-
             if not os.path.isdir(target_dir_path):
                 os.makedirs(target_dir_path)
                 if not os.path.isdir(target_dir_path):
                     raise OSError
 
-            for fname in uploads_dir_list:
-                fname = os.path.join(build_dir_path, fname)
-                shutil.copy2(fname, target_dir_path)
-
-            target_dir_list = os.listdir(target_dir_path)
-            for fname in uploads_dir_list:
-                if not fname in target_dir_list:
-                    print "Destination missing file", fname
-                    return FAIL
-
-            shutil.rmtree(build_dir_path)
+            self.move_dir_content(build_dir_path, target_dir_path)
+
+            if args.job_type != "kernel-hwpack":
+                ret = self.create_symlink(target_dir_path)
+                if ret != PASS:
+                    return ret
+
+            if args.manifest:
+                ret = self.create_manifest_file(target_dir_path)
+                if ret != PASS:
+                    print "Failed to create manifest file"
+                    return ret
+
             print "Moved the files from '",build_dir_path, "' to '",\
                   target_dir_path, "'"
             return PASS
@@ -113,9 +173,9 @@
         except OSError, details:
             print "Failed to create the target path", target_dir_path, ":" , details 
             return FAIL
+
         except shutil.Error:
             print "Failed to move files destination path", target_dir_path
-            print "Target already exists, move failed"
             return FAIL
 
 def main():

=== modified file 'testing/test_publish_to_snapshots.py'
--- testing/test_publish_to_snapshots.py	2012-03-16 10:55:32 +0000
+++ testing/test_publish_to_snapshots.py	2012-03-26 10:49:17 +0000
@@ -12,14 +12,16 @@
 class TestSnapshotsPublisher(TestCase):
     '''Tests for publishing files to the snapshots.l.o www are.'''
 
-    uploads_path = "./uploads/"
-    target_path = "./www/"
+    uploads_path = "uploads/"
+    target_path = "www/"
+    orig_dir = os.getcwd()
 
     def setUp(self):
         self.parser =  argparse.ArgumentParser()
         self.parser.add_argument("-t", "--job-type", dest="job_type")
         self.parser.add_argument("-j", "--job-name", dest="job_name")
         self.parser.add_argument("-n", "--build-num", dest="build_num", type=int)
+        self.parser.add_argument("-m", "--manifest",  dest="manifest", action='store_true')
         if not os.path.isdir(self.uploads_path):
             os.mkdir(self.uploads_path)
         
@@ -28,6 +30,7 @@
         super(TestSnapshotsPublisher, self).setUp()
 
     def tearDown(self):
+        os.chdir(self.orig_dir)
         if os.path.isdir(self.uploads_path):
             shutil.rmtree(self.uploads_path)
 
@@ -112,9 +115,9 @@
                                         '-n', '1'])
 
         self.publisher.validate_args(param)
-        self.uploads_path = "./dummy_uploads_path"
+        uploads_path = "./dummy_uploads_path"
         try:
-            self.publisher.validate_paths(param, self.uploads_path, self.target_path)
+            self.publisher.validate_paths(param, uploads_path, self.target_path)
         finally:
             sys.stdout = orig_stdout
         stdout.seek(0)
@@ -150,6 +153,8 @@
         try:
             uploads_dir_path, target_dir_path = self.publisher.validate_paths(param, 
                                                 self.uploads_path, self.target_path)
+            uploads_dir_path = os.path.join(self.orig_dir, uploads_dir_path)
+            target_dir_path = os.path.join(self.orig_dir, target_dir_path)
             self.publisher.move_artifacts(param, uploads_dir_path, target_dir_path)
         finally:
             sys.stdout = orig_stdout
@@ -172,6 +177,8 @@
         try:
             uploads_dir_path, target_dir_path = self.publisher.validate_paths(param, 
                                                 self.uploads_path, self.target_path)
+            uploads_dir_path = os.path.join(self.orig_dir, uploads_dir_path)
+            target_dir_path = os.path.join(self.orig_dir, target_dir_path)
             self.publisher.move_artifacts(param, uploads_dir_path, target_dir_path)
         finally:
             sys.stdout = orig_stdout
@@ -179,3 +186,73 @@
          
         stdout.seek(0)
         self.assertIn("Moved the files from", stdout.read())
+
+    def test_create_symlink(self):
+        orig_stdout = sys.stdout
+        stdout = sys.stdout = StringIO()
+        self.publisher = SnapshotsPublisher()
+        param = self.parser.parse_args(['-t', 'android', '-j', 'dummy_job_name', 
+                                       '-n', '1'])
+        self.publisher.validate_args(param)
+        build_dir = '/'.join([param.job_type, param.job_name, str(param.build_num)])
+        build_path = os.path.join(self.uploads_path, build_dir)
+        os.makedirs(build_path)
+        tempfiles = tempfile.mkstemp(dir=build_path)
+        try:
+            uploads_dir_path, target_dir_path = self.publisher.validate_paths(param, 
+                                                self.uploads_path, self.target_path)
+            uploads_dir_path = os.path.join(self.orig_dir, uploads_dir_path)
+            target_dir_path = os.path.join(self.orig_dir, target_dir_path)
+            self.publisher.move_artifacts(param, uploads_dir_path, target_dir_path)
+        finally:
+            sys.stdout = orig_stdout
+            pass
+        
+        stdout.seek(0)
+        msg = "The lastSuccessful build is now linked to  " +  target_dir_path
+        self.assertIn(msg, stdout.read())
+
+    def test_create_manifest_file_option(self):
+        orig_stdout = sys.stdout
+        stdout = sys.stdout = StringIO()
+        self.publisher = SnapshotsPublisher()
+        param = self.parser.parse_args(['-t', 'android', '-j', 'dummy_job_name', 
+                                        '-n', '1', '-m'])
+        self.publisher.validate_args(param)
+        build_dir = '/'.join([param.job_type, param.job_name, str(param.build_num)])
+        build_path = os.path.join(self.uploads_path, build_dir)
+        os.makedirs(build_path)
+        tempfiles = tempfile.mkstemp(dir=build_path)
+        lines = []
+        try:
+            uploads_dir_path, target_dir_path = self.publisher.validate_paths(param, 
+                                                self.uploads_path, self.target_path)
+            uploads_dir_path = os.path.join(self.orig_dir, uploads_dir_path)
+            target_dir_path = os.path.join(self.orig_dir, target_dir_path)
+            os.chdir(uploads_dir_path)
+            for path, subdirs, files in os.walk("."):
+                for name in files:
+                    lines.append(os.path.join(path, name).split("./")[1] + "\n")
+            os.chdir(self.orig_dir)
+            self.publisher.move_artifacts(param, uploads_dir_path, target_dir_path)
+
+            manifest_file=os.path.join(target_dir_path, "MANIFEST")
+            dest = open(manifest_file, "r").read()
+
+            if len(lines) != 0:
+                tempfiles = tempfile.mkstemp(dir=target_dir_path)
+                fd = open(tempfiles[1], "w+")
+                for line in lines:
+                    fd.write(line)
+                fd.close()
+                orig = open(tempfiles[1], "r").read()
+             
+        except Exception, details:
+            pass
+
+        finally:
+            sys.stdout = orig_stdout
+        
+        stdout.seek(0)
+        self.assertIn("Moved the files from", stdout.read())
+        self.assertTrue(orig == dest)