Merge lp:~pfalcon/linaro-android-build-tools/new-pub-staging into lp:linaro-android-build-tools

[Paul Sokolovsky]

Add support to publish to staging with new publish interface. For consistency with existing script --staging (-s) option is used.

[James Tunnicliffe (dooferlad)]

Looks good to me. Shame we don't have tests for this though.

review: Approve

[Georgy Redkozubov (gesha)]

+1

review: Approve

[Paul Sokolovsky (pfalcon)]

Well, this codebase has only unit tests, not high-level tests for this stuff, and as it's just a prototype, doesn't make much sense to invest much into that (we already have projects with broken integration tests).

[lp:~pfalcon/linaro-android-build-tools/new-pub-staging]

597. By Paul Sokolovsky on 2013-04-18

Unset option is None, not False.

=== modified file 'utils/new-publish/propagate.py'
--- utils/new-publish/propagate.py	2013-04-18 04:01:55 +0000
+++ utils/new-publish/propagate.py	2013-04-18 18:55:34 +0000
@@ -10,7 +10,8 @@
 import publib
 
 
-REMOTE_HOST = "mombin.canonical.com"
+REMOTE_HOST_PRODUCTION = "mombin.canonical.com"
+REMOTE_HOST_STAGING = "staging.snapshots.linaro.org"
 PUBLISH_USER_NAME = "linaro-android-build-publish"
 TRIGGER_USER_NAME = "linaro-android-build-publish-trigger"
 PUBLISH_KEY_FILE = "/home/ubuntu/snapshots-sync2.new/linaro-android-build-publish"
@@ -21,17 +22,25 @@
 
 if __name__ == "__main__":
     optparser = optparse.OptionParser(usage="%prog")
+    optparser.add_option("-s", "--staging", action="store_true", help="Publish to staging server")
     optparser.add_option("--identity-publish", metavar="KEY", default=PUBLISH_KEY_FILE, help="Publish SSH key file")
     optparser.add_option("--identity-trigger", metavar="KEY", default=TRIGGER_KEY_FILE, help="Trigger SSH key file")
     optparser.add_option("-n", "--dry-run", action="store_true", help="Don't actually publish files, log commands")
-    optparser.add_option("--stage", default="all", help="Run only specific stage")
+    optparser.add_option("--step", default="all", help="Run only specific step")
     options, args = optparser.parse_args(sys.argv[1:])
     if len(args) != 1:
         optparser.error("Wrong number of arguments")
 
     publib.validate_build_id(args[0])
 
-    if options.stage in ("all", "1"):
+    if options.staging:
+        remote_host = REMOTE_HOST_STAGING
+        opt_staging = "-s"
+    else:
+        remote_host = REMOTE_HOST_PRODUCTION
+        opt_staging = ""
+
+    if options.step in ("all", "1"):
         file_list = []
         for root, dirs, files in os.walk(os.path.join(LOCAL_UPLOAD_DIR, args[0])):
             file_list.extend([os.path.join(root, f) for f in files])
@@ -40,18 +49,18 @@
         dir_list = publib.make_dir_struct(file_list, REMOTE_UPLOAD_DIR, strip=strip)
         print "Dirs:", dir_list
         if not options.dry_run:
-            publib.create_dir_struct(dir_list, REMOTE_HOST, PUBLISH_USER_NAME,
+            publib.create_dir_struct(dir_list, remote_host, PUBLISH_USER_NAME,
                                       options.identity_publish)
         upload_script = publib.make_upload_script(file_list, REMOTE_UPLOAD_DIR, strip=strip)
-        publib.upload_files(upload_script, REMOTE_HOST, PUBLISH_USER_NAME,
+        publib.upload_files(upload_script, remote_host, PUBLISH_USER_NAME,
                              options.identity_publish, options)
 
-    if options.stage in ("all", "2"):
+    if options.step in ("all", "2"):
         job, build = args[0].split("/")
         client = paramiko.SSHClient()
         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        client.connect(REMOTE_HOST, username=TRIGGER_USER_NAME, key_filename=TRIGGER_KEY_FILE)
-        stdin, stdout, stderr = client.exec_command("reshuffle-files -t android -j %s -n %s -m" % (job, build))
+        client.connect(remote_host, username=TRIGGER_USER_NAME, key_filename=TRIGGER_KEY_FILE)
+        stdin, stdout, stderr = client.exec_command("reshuffle-files -t android -j %s -n %s -m %s" % (job, build, opt_staging))
         stdin.close()
         print "=== stdout ==="
         print stdout.read()

=== modified file 'utils/new-publish/publish'
--- utils/new-publish/publish	2013-04-18 12:47:51 +0000
+++ utils/new-publish/publish	2013-04-18 18:55:34 +0000
@@ -19,6 +19,7 @@
     optparser.add_option("-t", "--type", help="Build type")
     optparser.add_option("-p", "--strip", metavar="NUM", type=int, default=0,
                          help="Strip leading components from dest file names.")
+    optparser.add_option("-s", "--staging", action="store_true", help="Publish to staging")
     optparser.add_option("--identity-copy", metavar="KEY", default=COPY_KEY_FILE, help="SSH key file")
     optparser.add_option("--identity-trigger", metavar="KEY", default=TRIGGER_KEY_FILE, help="SSH key file")
     optparser.add_option("-n", "--dry-run", action="store_true", help="Don't actually publish files, log commands")
@@ -48,7 +49,10 @@
     upload_script = publib.make_upload_script(file_list, UPLOAD_DIR, build_id, options.strip)
     publib.upload_files(upload_script, REMOTE_HOST, COPY_USER_NAME, options.identity_copy, options)
 
-    rc = os.system("ssh -i %s %s@%s propagate.py %s" % (options.identity_trigger, TRIGGER_USER_NAME, REMOTE_HOST, build_id))
+    rc = os.system("ssh -i %s %s@%s propagate.py %s %s" % (options.identity_trigger,
+        TRIGGER_USER_NAME, REMOTE_HOST,
+        "-s" if options.staging else "",
+        build_id))
     if rc != 0:
         print "Publishing failed"
         sys.exit(1)