Merge lp:~maxb/udd/local-branches into lp:udd

[Max Bowsher]

When investigating a failure like the "NoSuchTag" failures, a key problem is that it is impossible to experiment with added tags to validate that they will fix the conversion.

Therefore, I've written a new option --local-branches - when used, a special BranchStore (thanks for having that factored out! :-) ) is used, which pushes and pulls from BASE_DIR/localbranches/PACKAGENAME/DISTRO-SERIES by default, but if that doesn't exist, it first populates it from Launchpad.

This allows convenient testing of changes to the existing import branches.

[James Westby (james-w)]

On Sun, 03 Oct 2010 20:35:54 -0000, Max Bowsher <email address hidden> wrote:
> Max Bowsher has proposed merging lp:~maxb/udd/local-branches into lp:udd.
>
> Requested reviews:
> Ubuntu Distributed Development Developers (udd)
>
>
> When investigating a failure like the "NoSuchTag" failures, a key
> problem is that it is impossible to experiment with added tags to
> validate that they will fix the conversion.

>
> Therefore, I've written a new option --local-branches - when used, a
> special BranchStore (thanks for having that factored out! :-) ) is
> used, which pushes and pulls from
> BASE_DIR/localbranches/PACKAGENAME/DISTRO-SERIES by default, but if
> that doesn't exist, it first populates it from Launchpad.

Nice idea. I'll review tomorrow, thanks.

Just to note that I've been looking at NoSuchTag today, and think I have
a handle on it, check the referenced bug.

Sorry if I've caused duplicate effort.

Thanks,

James

=== modified file 'icommon.py'
--- icommon.py	2010-10-03 17:10:32 +0000
+++ icommon.py	2010-10-03 20:35:53 +0000
@@ -20,7 +20,7 @@
 
 from debian_bundle import changelog
 
-from bzrlib import branch, diff, errors, merge, tag, testament, transport, ui
+from bzrlib import branch, bzrdir, diff, errors, merge, tag, testament, transport, ui
 from bzrlib.trace import mutter
 
 
@@ -47,6 +47,7 @@
 max_threads_file = os.path.join(base_dir, "max_threads")
 locks_dir = os.path.join(base_dir, "locks")
 updates_dir = os.path.join(base_dir, "updates")
+localbranches_dir = os.path.join(base_dir, "localbranches")
 sqlite_file = os.path.join(base_dir, "meta.db")
 sqlite_package_file = os.path.join(base_dir, "packages.db")
 sqlite_old_failures_file = os.path.join(base_dir, "failures.db")
@@ -1348,6 +1349,53 @@
                 pocket=self._lp_pocket(pocket))
 
 
+class LocalShadowBranchStore(BranchStore):
+
+    def __init__(self, package, lp, suffix):
+        BranchStore.__init__(self, package, lp, suffix)
+        self.branches_dir = os.path.join(localbranches_dir, package)
+        if not os.path.exists(self.branches_dir):
+            os.makedirs(self.branches_dir)
+            format = bzrdir.format_registry.make_bzrdir('2a')
+            to_transport = transport.get_transport(self.branches_dir)
+            newdir = format.initialize_on_transport(to_transport)
+            repo = newdir.create_repository(shared=True)
+            repo.set_make_working_trees(False)
+
+    def side_branch_location(self, distro, release, suite):
+        distro, suite = self._translate_suite(distro, suite)
+        now = datetime.datetime.utcnow()
+        now_str = now.strftime("%Y%m%d%H%M")
+        return os.path.join(self.branches_dir, "%s-%s-%s" % (distro, suite, now_str))
+
+    def get_branch_parts(self, distro, release, pocket, possible_transports=None, readonly=False):
+        local_location = self.push_location(distro, release, pocket)
+        try:
+            return branch.Branch.open(local_location, possible_transports=possible_transports)
+        except errors.NotBranchError:
+            try:
+                br_from = branch.Branch.open(self._branch_location(distro, release, pocket, readonly=readonly),
+                        possible_transports=possible_transports)
+                br_from.lock_read()
+                try:
+                    dir = br_from.bzrdir.sprout(local_location)
+                    new_br = dir.open_branch()
+                    tag._merge_tags_if_possible(br_from, new_br)
+                finally:
+                    br_from.unlock()
+                return new_br
+            except errors.NotBranchError:
+                return None
+
+    def push_location(self, distro, release, pocket):
+        distro, suite = self._translate_suite(distro, make_suite(release, pocket))
+        return os.path.join(self.branches_dir, "%s-%s%s" % (distro,
+            suite, self.suffix))
+
+    def set_official(self, distro, release, pocket, set_status=False,
+            set_reviewer=False):
+        pass # No-op in this implementation
+
 class MemoryBranchStore(BranchStore):
 
     def __init__(self, package, lp, suffix):

=== modified file 'import_package.py'
--- import_package.py	2010-10-03 17:33:40 +0000
+++ import_package.py	2010-10-03 20:35:53 +0000
@@ -53,6 +53,12 @@
                   help="Set debug flags, see 'bzr help debug-flags'")
 parser.add_option("--lp-cache", dest="lp_cache", default=None,
                   help="Set the location of a Launchpadlib cache")
+parser.add_option("--local-branches", dest="local_branches", action="store_true",
+                  help="Debugging mode in which branches in "
+                  "BASE_DIR/localbranches/PACKAGE are used instead of the "
+                  "real Launchpad branches. A local branch will be branched "
+                  "from the Launchpad branch on first access if it does not "
+                  "exist locally")
 
 options, args = parser.parse_args()
 
@@ -647,7 +653,7 @@
     return unimported_versions
 
 
-def push_branches_back(package, temp_dir, bstore, possible_transports):
+def push_branches_back(package, temp_dir, bstore, possible_transports, local_branches):
     for d in distros:
         pushed_devel = False
         for known_release in icommon.lp_distro_releases[d]:
@@ -662,7 +668,7 @@
                 if os.path.exists(source_branch_path + ".overwrite"):
                     overwrite = True
                 to_loc = bstore.push_location(d, known_release, known_pocket)
-                if pushed_devel:
+                if pushed_devel and not local_branches:
                     mutter("sleeping to stack")
                     time.sleep(300)
                     pushed_devel = False
@@ -684,7 +690,7 @@
                     except HTTPError, e:
                         print e.content.decode("utf-8", "replace")
                         raise
-                    if os.path.exists(source_branch_path + ".collisions"):
+                    if os.path.exists(source_branch_path + ".collisions") and not local_branches:
                         mutter("sleeping to file merge proposal")
                         time.sleep(120)
                         f = open(source_branch_path + ".collisions")
@@ -905,7 +911,7 @@
 
 
 def main(package, push=True, check=False, extra_debian=None, no_existing=False,
-        keep_temp=False):
+        keep_temp=False, local_branches=False):
     if not os.path.exists(icommon.logs_dir):
         os.makedirs(icommon.logs_dir)
     log_name = os.path.join(icommon.logs_dir, package)
@@ -920,7 +926,10 @@
         if not no_existing:
             revid_db = icommon.RevidDatabase(icommon.sqlite_file, package)
             suffix = revid_db.get_suffix()
-            bstore = icommon.BranchStore(package, lp, suffix)
+            if local_branches:
+                bstore = icommon.LocalShadowBranchStore(package, lp, suffix)
+            else:
+                bstore = icommon.BranchStore(package, lp, suffix)
             commit_db = icommon.CommitDatabase(icommon.sqlite_file, package)
         else:
             revid_db = icommon.MemoryRevidDatabase(package)
@@ -932,7 +941,7 @@
         def push_back():
             possible_transports = []
             push_branches_back(package, temp_dir, bstore,
-                    possible_transports)
+                    possible_transports, local_branches)
             icommon.generate_debian_diffs(lp, temp_dir, package, bstore,
                     possible_transports=possible_transports)
             icommon.generate_ubuntu_merges(lp, temp_dir, package, bstore,
@@ -990,7 +999,7 @@
             check_dwim(temp_dir, False, do_repo=True)
         if push:
             commit_db.start_commit()
-            push_branches_back(package, temp_dir, bstore, possible_transports)
+            push_branches_back(package, temp_dir, bstore, possible_transports, local_branches)
             icommon.generate_debian_diffs(lp, temp_dir, package, bstore,
                     possible_transports=possible_transports)
             icommon.generate_ubuntu_merges(lp, temp_dir, package, bstore,
@@ -1035,7 +1044,7 @@
             sys.exit(1)
         bstore = icommon.BranchStore(args[0], lp)
         try:
-            sys.exit(push_branches_back(args[0], args[1], bstore, []))
+            sys.exit(push_branches_back(args[0], args[1], bstore, [], False))
         except HTTPError, e:
             print e.content.decode("utf-8", "replace")
             raise
@@ -1051,7 +1060,8 @@
             sys.exit(main(args[0], push=not options.no_push,
                         extra_debian=options.extra_debian, check=options.check,
                         no_existing=options.no_existing,
-                        keep_temp=options.keep_temp))
+                        keep_temp=options.keep_temp,
+                        local_branches=options.local_branches))
         except HTTPError, e:
             print e.content.decode("utf-8", "replace")
             raise