Merge lp:~pfalcon/linaro-android-gerrit-support/manual-create into lp:linaro-android-gerrit-support

[Paul Sokolovsky]

This is follow up to mails I sent to infra@ regarding issues with automatically creating new AOSP projects in our mirror and what to do about them.

With these changes, myself or another Gerrit admin can just run locally:

android.git.linaro.org/create-new-projects.sh --dest=admin

to create needed projects, without requiring to temporarily patch Gerrit group membership or git-gerrit-mirror config files.

[James Tunnicliffe (dooferlad)]

Looks fine to me.

review: Approve

=== modified file 'android.git.linaro.org/create-new-projects.sh'
--- android.git.linaro.org/create-new-projects.sh	2012-03-15 13:49:14 +0000
+++ android.git.linaro.org/create-new-projects.sh	2012-03-23 16:08:38 +0000
@@ -15,10 +15,20 @@
     exit 1
 }
 
+# Require explicit flag to clone new projects, to avoid spurious clones when
+# when run on local system to just create new projects. Should be specified
+# first, if any other options are given.
+if [ "$1" == "--clone" ]; then
+    clone=1
+    shift
+fi
+
 wget "$MANIFEST" -q -O manifest-for-create.xml || fatal "Error getting $MANIFEST"
 
 ./git-gerrit-mirror --mirror-dir=$MIRROR_DIR \
     create --upstream=google --manifest=manifest-for-create.xml $*
 
-./git-gerrit-mirror --mirror-dir=$MIRROR_DIR \
-    clone  --upstream=google --manifest=manifest-for-create.xml $*
+if [ -n "$clone" ]; then
+    ./git-gerrit-mirror --mirror-dir=$MIRROR_DIR \
+        clone  --upstream=google --manifest=manifest-for-create.xml $*
+fi

=== modified file 'android.git.linaro.org/crontab'
--- android.git.linaro.org/crontab	2012-03-16 15:50:36 +0000
+++ android.git.linaro.org/crontab	2012-03-23 16:08:38 +0000
@@ -4,4 +4,4 @@
 
 0  2,14 * * *	(cd ~/linaro-android-gerrit-support; android.git.linaro.org/cron-runner.sh fetch) || echo "Fetch error"
 30 2,14 * * *	(cd ~/linaro-android-gerrit-support; android.git.linaro.org/cron-runner.sh push --force) || echo "Push error"
-0 15 * * * *	(cd ~/linaro-android-gerrit-support; android.git.linaro.org/create-new-projects.sh)
+0 15 * * * *	(cd ~/linaro-android-gerrit-support; android.git.linaro.org/create-new-projects.sh --clone)

=== modified file 'git-gerrit-mirror'
--- git-gerrit-mirror	2012-03-15 13:47:45 +0000
+++ git-gerrit-mirror	2012-03-23 16:08:38 +0000
@@ -9,7 +9,7 @@
 
 
 # Magic name for mirror destination host
-DEST = "DESTINATION"
+DEST_KEYWORD = "DESTINATION"
 
 log = logging.getLogger(__file__)
 
@@ -46,7 +46,7 @@
             if l[0] == "[":
                 self.store_host_exceptions(host, host_vars, host_exceptions)
                 host = l[1:-1]
-                if host != DEST:
+                if not host.startswith(DEST_KEYWORD):
                     assert "://" in host, "URL schema is required in " + l
                     assert "/" not in host.split("://")[1], "Only hostname (no path) allowed in " + l
                 host_exceptions = {}
@@ -69,11 +69,14 @@
         in_file.close()
 
     def get_hosts(self, substr_match=""):
-        return sorted([h for h in self.host_map.keys() if substr_match in h and h != DEST])
+        return sorted([h for h in self.host_map.keys() if substr_match in h and not h.startswith(DEST_KEYWORD)])
 
     def get_var(self, host, var):
         return self.host_map[host][0].get(var)
 
+    def has_host(self, host):
+        return host in self.host_map
+
     def get_mirror_repo(self, host, source_repo):
         """Get mirror path for a repo. By default it is equal
         to source path, but if exception was defined, it is different."""
@@ -148,7 +151,7 @@
     
     for p in projects:
         run_command("ssh -p %s %s gerrit create-project --name %s" % 
-          (conf.get_var(DEST, "gerrit_port"), conf.get_var(DEST, "gerrit_host"), p))
+          (conf.get_var(options.dest, "gerrit_port"), conf.get_var(options.dest, "gerrit_host"), p))
 
 def clone_repos(host, basedir, projects):
     "Clone-mirror repos from remote server"
@@ -186,7 +189,8 @@
         mirror_projname = conf.get_mirror_repo(host, projname)
         os.chdir(abspath)
         force_opt = " --force" if force else ""
-        cmdline = "git push%s %s/%s.git 'refs/heads/*' 'refs/tags/*'" % (force_opt, conf.get_var(DEST, "repo_root"), mirror_projname)
+        repo_root = conf.get_var(options.dest, "repo_root")
+        cmdline = "git push%s %s/%s.git 'refs/heads/*' 'refs/tags/*'" % (force_opt, repo_root, mirror_projname)
         run_command(cmdline)
 
 def check_args(optparser, args, expected):
@@ -212,6 +216,8 @@
                      help="Use manifest for list of upstream projects (overrides $gerrit in config)")
 optparser.add_option("--upstream", metavar="SUBSTR", default="",
                      help="Process only upstreams matching SUBSTR")
+optparser.add_option("--dest", default="",
+                     help="Use alternative destination as specified in mirror.conf")
 optparser.add_option("--force", action="store_true",
                      help="Pass --force to git commands where applicable")
 optparser.add_option("--dry-run", action="store_true",
@@ -227,10 +233,16 @@
 
 options.mirror_dir = os.path.abspath(options.mirror_dir)
 conf = MirrorConfig(options.config)
+dest = options.dest
+if dest:
+    dest = "-" + dest
+options.dest = DEST_KEYWORD + dest
 
 if args[0] == "create":
     check_args(optparser, args, 1)
-    existing_projects = get_gerrit_projects("%s:%s" % (conf.get_var(DEST, "gerrit_host"), conf.get_var(DEST, "gerrit_port")))
+    gerrit_host = conf.get_var(options.dest, "gerrit_host")
+    gerrit_port = conf.get_var(options.dest, "gerrit_port")
+    existing_projects = get_gerrit_projects("%s:%s" % (gerrit_host, gerrit_port))
     for host in conf.get_hosts(options.upstream):
         log.debug("=== Processing: %s ===", host)
         projects = get_project_map_for_a_host(host).values()

=== modified file 'mirror.conf'
--- mirror.conf	2012-03-13 16:19:21 +0000
+++ mirror.conf	2012-03-23 16:08:38 +0000
@@ -1,16 +1,23 @@
 [DESTINATION]
-# Usecase 1: target is Gerrit
-#$repo_root = ssh://[e-mail]:29418
-#$gerrit_host = [e-mail]
+# Usecase 1: target is Gerrit, run as automatic cronjob
 $repo_root = ssh://linaro-aosp-sync-bot@gerrit-git:29418
 $gerrit_host = linaro-aosp-sync-bot@gerrit-git
 $gerrit_port = 29418
 
+[DESTINATION-admin]
+# Usecase 2: target is Gerrit, run locally by Gerrit admin
+# (e.g. to create projects)
+$repo_root = ssh://android.git.linaro.org:29418
+$gerrit_host = android.git.linaro.org
+$gerrit_port = 29418
+
+[DESTINATION-git]
 # Usecase 2: target is a remote tree
-#$repo_root = ssh://somewhere.org/git
+$repo_root = ssh://somewhere.org/git
 
+[DESTINATION-local]
 # Usecase 3: target is a local tree (e.g. for testing)
-#$repo_root = /mnt/gerrit-git-snapshot/git
+$repo_root = /mnt/gerrit-git-snapshot/git
 
 [git://android.git.kernel.org]
 $active = false