Merge lp:~jelmer/bzr-fastimport/setup.py-fixes into lp:~bzr/bzr-fastimport/fastimport.dev

[Jelmer Vernooij]

[Jelmer Vernooij (jelmer)]

This adds a subvertpy-based svn-fast-export as well, and adds support for the executable bit and symbolic links.

[Ian Clatworthy (ian-clatworthy)]

After an IRC discussion, we've agreed to leave Jelmer's fast-export code in subvertpy. I'll tweak fast-export-from-svn to look for it in preference to the current one.

review: Disapprove

Unmerged revisions

257. By Jelmer Vernooij on 2009-10-18

Support symbolic links in svn-fast-export.py.

256. By Jelmer Vernooij on 2009-10-18

svn-fast-export: Add support for executable bit.

255. By Jelmer Vernooij on 2009-10-18

Backport Ted's patch.

254. By Jelmer Vernooij on 2009-10-18

Use subvertpy bindings in svn-fast-export.py.

=== renamed file 'exporters/svn-fast-export.py' => 'exporters/svn-fast-export-old.py'
=== added file 'exporters/svn-fast-export.py'
--- exporters/svn-fast-export.py	1970-01-01 00:00:00 +0000
+++ exporters/svn-fast-export.py	2009-10-19 09:40:31 +0000
@@ -0,0 +1,220 @@
+#!/usr/bin/python
+#
+# svn-fast-export.py
+# ----------
+#  Walk through each revision of a local Subversion repository and export it
+#  in a stream that git-fast-import can consume.
+#
+# Author: Chris Lee <[e-mail]>
+# License: MIT <http://www.opensource.org/licenses/mit-license.php>
+#
+# Adapted for subvertpy by Jelmer Vernooij <[e-mail]>
+
+trunk_path = '/trunk/'
+branches_path = '/branches/'
+tags_path = '/tags/'
+address = 'localhost'
+
+from cStringIO import StringIO
+import sys, os.path
+from optparse import OptionParser
+import stat
+from time import mktime, strptime
+
+from subvertpy.repos import PATH_CHANGE_DELETE, Repository
+
+ct_short = ['M', 'A', 'D', 'R', 'X']
+
+def dump_file_blob(root, stream, stream_length):
+    sys.stdout.write("data %s\n" % stream_length)
+    sys.stdout.flush()
+    sys.stdout.write(stream.read())
+    sys.stdout.write("\n")
+
+
+class Matcher(object):
+
+    branch = None
+
+    def __init__(self, trunk_path):
+        self.trunk_path = trunk_path
+
+    def branchname(self):
+        return self.branch
+
+    def __str__(self):
+        return super(Matcher, self).__str__() + ":" + self.trunk_path
+
+    @staticmethod
+    def getMatcher(trunk_path):
+        if trunk_path.startswith("regex:"):
+            return RegexStringMatcher(trunk_path)
+        else:
+            return StaticStringMatcher(trunk_path)
+
+
+class StaticStringMatcher(Matcher):
+
+    branch = "master"
+
+    def matches(self, path):
+        return path.startswith(trunk_path)
+
+    def replace(self, path):
+        return path.replace(self.trunk_path, '')
+
+
+class RegexStringMatcher(Matcher):
+
+    def __init__(self, trunk_path):
+        super(RegexStringMatcher, self).__init__(trunk_path)
+        import re
+        self.matcher = re.compile(self.trunk_path[len("regex:"):])
+
+    def matches(self, path):
+        match = self.matcher.match(path)
+        if match:
+            self.branch = match.group(1)
+            return True
+        else:
+            return False
+
+    def replace(self, path):
+        return self.matcher.sub("\g<2>", path)
+
+MATCHER = None
+
+def export_revision(rev, fs):
+    sys.stderr.write("Exporting revision %s... " % rev)
+
+    # Open a root object representing the youngest (HEAD) revision.
+    root = fs.revision_root(rev)
+
+    # And the list of what changed in this revision.
+    changes = root.paths_changed()
+
+    i = 1
+    marks = {}
+    file_changes = []
+
+    for path, (node_id, change_type, text_changed, prop_changed) in changes.iteritems():
+        if root.is_dir(path):
+            continue
+        if not MATCHER.matches(path):
+            # We don't handle branches. Or tags. Yet.
+            pass
+        else:
+            if change_type == PATH_CHANGE_DELETE:
+                file_changes.append("D %s" % MATCHER.replace(path))
+            else:
+                props = root.proplist(path)
+                marks[i] = MATCHER.replace(path)
+                if props.get("svn:special", ""):
+                    contents = root.file_content(path).read()
+                    if not contents.startswith("link "):
+                        sys.stderr.write("special file '%s' is not a symlink, ignoring...\n" % path)
+                        continue
+                    mode = stat.S_IFLNK
+                    stream = StringIO(contents[len("link "):])
+                    stream_length = len(stream.getvalue())
+                else:
+                    if props.get("svn:executable", ""):
+                        mode = 0755
+                    else:
+                        mode = 0644
+                    stream_length = root.file_length(path)
+                    stream = root.file_content(path)
+                file_changes.append("M %o :%s %s" % (mode, i, marks[i]))
+                sys.stdout.write("blob\nmark :%s\n" % i)
+                dump_file_blob(root, stream, stream_length)
+                stream.close()
+                i += 1
+
+    # Get the commit author and message
+    props = fs.revision_proplist(rev)
+
+    # Do the recursive crawl.
+    if props.has_key('svn:author'):
+        author = "%s <%s@%s>" % (props['svn:author'], props['svn:author'], address)
+    else:
+        author = 'nobody <nobody@localhost>'
+
+    if len(file_changes) == 0:
+        sys.stderr.write("skipping.\n")
+        return
+
+    svndate = props['svn:date'][0:-8]
+    commit_time = mktime(strptime(svndate, '%Y-%m-%dT%H:%M:%S'))
+    sys.stdout.write("commit refs/heads/%s\n" % MATCHER.branchname())
+    sys.stdout.write("committer %s %s -0000\n" % (author, int(commit_time)))
+    sys.stdout.write("data %s\n" % len(props['svn:log']))
+    sys.stdout.write(props['svn:log'])
+    sys.stdout.write("\n")
+    sys.stdout.write('\n'.join(file_changes))
+    sys.stdout.write("\n\n")
+    sys.stderr.write("done!\n")
+
+
+def crawl_revisions(repos_path, first_rev=None, final_rev=None):
+    """Open the repository at REPOS_PATH, and recursively crawl all its
+    revisions."""
+
+    # Open the repository at REPOS_PATH, and get a reference to its
+    # versioning filesystem.
+    fs_obj = Repository(repos_path).fs()
+
+    # Query the current youngest revision.
+    if first_rev is None:
+        first_rev = 1
+    if final_rev is None:
+        final_rev = fs_obj.youngest_revision()
+    for rev in xrange(first_rev, final_rev + 1):
+        export_revision(rev, fs_obj)
+
+
+if __name__ == '__main__':
+    usage = '%prog [options] REPOS_PATH'
+    parser = OptionParser()
+    parser.set_usage(usage)
+    parser.add_option('-f', '--final-rev', help='Final revision to import', 
+                      dest='final_rev', metavar='FINAL_REV', type='int')
+    parser.add_option('-r', '--first-rev', help='First revision to import', 
+                      dest='first_rev', metavar='FIRST_REV', type='int')
+    parser.add_option('-t', '--trunk-path', help="Path in repo to /trunk, may be `regex:/cvs/(trunk)/proj1/(.*)`\nFirst group is used as branchname, second to match files",
+                      dest='trunk_path', metavar='TRUNK_PATH')
+    parser.add_option('-b', '--branches-path', help='Path in repo to /branches',
+                      dest='branches_path', metavar='BRANCHES_PATH')
+    parser.add_option('-T', '--tags-path', help='Path in repo to /tags',
+                      dest='tags_path', metavar='TAGS_PATH')
+    parser.add_option('-a', '--address', help='Domain to put on users for their mail address', 
+                      dest='address', metavar='hostname', type='string')
+    (options, args) = parser.parse_args()
+
+    if options.trunk_path != None:
+        trunk_path = options.trunk_path
+    if options.branches_path != None:
+        branches_path = options.branches_path
+    if options.tags_path != None:
+        tags_path = options.tags_path
+    if options.address != None:
+        address = options.address
+
+    MATCHER = Matcher.getMatcher(trunk_path)
+    sys.stderr.write("%s\n" % MATCHER)
+    if len(args) != 1:
+        parser.print_help()
+        sys.exit(2)
+
+    # Canonicalize (enough for Subversion, at least) the repository path.
+    repos_path = os.path.normpath(args[0])
+    if repos_path == '.': 
+        repos_path = ''
+
+    try:
+        import msvcrt
+        msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    except ImportError:
+        pass
+
+    crawl_revisions(repos_path, first_rev=options.first_rev,
+                    final_rev=options.final_rev)