Merge lp:~gesha/linaro-license-protection/fix-prebuilt-publishing into lp:~linaro-automation/linaro-license-protection/trunk

[Georgy Redkozubov]

This branch fixes handling of already existed dirs in dest.
Checking if source being sanitized is not a directory.

[Deepti B. Kalakeri (deeptik)]

Looks good. I would like you to please add some more tests with --staging for android/kernel-hwpack/prebuilt/images/hwpacks. I see there is only one test for 'binaries' type only.

review: Needs Fixing

[lp:~gesha/linaro-license-protection/fix-prebuilt-publishing]

128. By Georgy Redkozubov on 2012-09-04

Added tests for moving artifacts where destination subdirs exist.

[Georgy Redkozubov (gesha)]

> Looks good. I would like you to please add some more tests with --staging for
> android/kernel-hwpack/prebuilt/images/hwpacks. I see there is only one test
> for 'binaries' type only.

Actually we don't need tests for each type of job with --staging. We have params validation, single artifact moving test with and w/o --staging.
I've added tests for moving artifacts with subdirs in the path.

[Deepti B. Kalakeri (deeptik)]

Thanks for adding the test. Actually kernel-hwpack/android/prebuilt/images/hwpacks all of them have different way of storing the data. Hence it would have be good to test regressions in future.
Anyways for now this looks good.
+1.

review: Approve

=== modified file 'scripts/publish_to_snapshots.py'
--- scripts/publish_to_snapshots.py	2012-08-28 20:29:05 +0000
+++ scripts/publish_to_snapshots.py	2012-09-04 06:15:22 +0000
@@ -87,15 +87,16 @@
     def sanitize_file(cls, file_path):
         """This truncates the file and fills it with its own filename."""
         assert not cls.is_accepted_for_staging(file_path)
-        base_file_name = os.path.basename(file_path)
-        protected_file = open(file_path, "w")
-        # Nice property of this is that we are also saving on disk space
-        # needed.
-        protected_file.truncate()
-        # To help distinguish files more easily when they are downloaded,
-        # we write out the base file name as the contents.
-        protected_file.write(base_file_name)
-        protected_file.close()
+        if not os.path.isdir(file_path):
+            base_file_name = os.path.basename(file_path)
+            protected_file = open(file_path, "w")
+            # Nice property of this is that we are also saving on disk space
+            # needed.
+            protected_file.truncate()
+            # To help distinguish files more easily when they are downloaded,
+            # we write out the base file name as the contents.
+            protected_file.write(base_file_name)
+            protected_file.close()
 
     def validate_args(self, args):
         # Validate that all the required information
@@ -263,21 +264,22 @@
             for file in filelist:
                 src = os.path.join(src_dir, file)
                 dest = os.path.join(dest_dir, file)
+                if os.path.isdir(src):
+                    if not os.path.exists(dest):
+                        os.makedirs(dest)
+                    self.move_dir_content(src, dest, sanitize)
+                    continue
                 if os.path.exists(dest):
                     if os.path.isdir(dest):
                         continue
                     else:
                         os.remove(dest)
-                if os.path.isdir(src):
-                    os.mkdir(dest)
-                    self.move_dir_content(src, dest, sanitize)
-                else:
-                    if sanitize and not self.is_accepted_for_staging(src):
-                        # Perform the sanitization before moving the file
-                        # into place.
-                        print "Sanitizing contents of '%s'." % src
-                        self.sanitize_file(src)
-                    shutil.move(src, dest)
+                if sanitize and not self.is_accepted_for_staging(src):
+                    # Perform the sanitization before moving the file
+                    # into place.
+                    print "Sanitizing contents of '%s'." % src
+                    self.sanitize_file(src)
+                shutil.move(src, dest)
         except shutil.Error:
             print "Error while moving the content"
 
 
=== modified file 'tests/test_publish_to_snapshots.py'
--- tests/test_publish_to_snapshots.py	2012-08-22 19:42:28 +0000
+++ tests/test_publish_to_snapshots.py	2012-09-04 06:15:22 +0000
@@ -620,3 +620,78 @@
         msg = "Manifest file " + manifest_file + " generated"
         self.assertIn(msg, res_output)
         self.assertTrue(orig == dest)
+
+    def test_move_artifacts_with_subdirs_successful_move(self):
+        job_dir = 'precise/pre-built/lt-panda/2'
+        orig_stdout = sys.stdout
+        stdout = sys.stdout = StringIO()
+        self.publisher = SnapshotsPublisher()
+        param = self.parser.parse_args(
+            ['-t', 'prebuilt', '-j', 'precise-armhf-ubuntu-desktop',
+             '-n', '1'])
+        self.publisher.validate_args(param)
+        build_dir = '/'.join([param.job_name, str(param.build_num)])
+        build_path = os.path.join(self.uploads_path, build_dir)
+        os.makedirs(build_path)
+        new_subdirs = os.path.join(build_dir, job_dir)
+        new_subdirs_path = os.path.join(self.uploads_path, new_subdirs)
+        os.makedirs(new_subdirs_path)
+        artifact = self.make_temporary_file("Content", root=new_subdirs_path)
+        try:
+            uploads_dir_path, target_dir_path = self.publisher.validate_paths(
+                param, self.uploads_path, self.target_path)
+            uploads_dir_path = os.path.join(self.orig_dir, uploads_dir_path)
+            target_dir_path = os.path.join(self.orig_dir, target_dir_path)
+            subdirs = 'precise/pre-built/lt-panda/1'
+            subdirs_path = os.path.join(target_dir_path, subdirs)
+            os.makedirs(subdirs_path)
+            self.publisher.move_artifacts(param, uploads_dir_path,
+                                          target_dir_path)
+
+            moved_artifact = os.path.join(target_dir_path, job_dir,
+                                      os.path.basename(artifact))
+            print moved_artifact
+            self.assertEqual("Content", open(moved_artifact).read())
+        finally:
+            sys.stdout = orig_stdout
+            pass
+
+        stdout.seek(0)
+        self.assertIn("Moved the files from", stdout.read())
+
+    def test_move_artifacts_with_subdirs_sanitizing_successful_move(self):
+        job_dir = 'precise/pre-built/lt-panda/2'
+        orig_stdout = sys.stdout
+        stdout = sys.stdout = StringIO()
+        self.publisher = SnapshotsPublisher()
+        param = self.parser.parse_args(
+            ['-t', 'prebuilt', '-j', 'precise-armhf-ubuntu-desktop',
+             '-n', '1', '-s'])
+        self.publisher.validate_args(param)
+        build_dir = '/'.join([param.job_name, str(param.build_num)])
+        build_path = os.path.join(self.uploads_path, build_dir)
+        os.makedirs(build_path)
+        new_subdirs = os.path.join(build_dir, job_dir)
+        new_subdirs_path = os.path.join(self.uploads_path, new_subdirs)
+        os.makedirs(new_subdirs_path)
+        artifact = self.make_temporary_file("Content", root=new_subdirs_path)
+        try:
+            uploads_dir_path, target_dir_path = self.publisher.validate_paths(
+                param, self.uploads_path, self.target_path)
+            uploads_dir_path = os.path.join(self.orig_dir, uploads_dir_path)
+            target_dir_path = os.path.join(self.orig_dir, target_dir_path)
+            subdirs = 'precise/pre-built/lt-panda/1'
+            subdirs_path = os.path.join(target_dir_path, subdirs)
+            os.makedirs(subdirs_path)
+            self.publisher.move_artifacts(param, uploads_dir_path,
+                                          target_dir_path)
+
+            moved_artifact = os.path.join(target_dir_path, job_dir,
+                                      os.path.basename(artifact))
+        finally:
+            sys.stdout = orig_stdout
+            pass
+
+        stdout.seek(0)
+        self.assertEqual(os.path.basename(artifact), open(moved_artifact).read())
+        self.assertIn("Moved the files from", stdout.read())