Merge lp:~vmiklos/bzr-fastimport/darcs into lp:~bzr/bzr-fastimport/fastimport.dev
- darcs
- Merge into fastimport.dev
Proposed by
Miklos Vajna
Status: | Merged | ||||||||
---|---|---|---|---|---|---|---|---|---|
Merged at revision: | not available | ||||||||
Proposed branch: | lp:~vmiklos/bzr-fastimport/darcs | ||||||||
Merge into: | lp:~bzr/bzr-fastimport/fastimport.dev | ||||||||
Diff against target: |
797 lines 4 files modified
exporters/darcs/darcs-fast-export (+359/-305) exporters/darcs/darcs-fast-export.txt (+4/-0) exporters/darcs/t/lib-httpd.sh (+67/-0) exporters/darcs/t/test2-git-http.sh (+22/-0) |
||||||||
To merge this branch: | bzr merge lp:~vmiklos/bzr-fastimport/darcs | ||||||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Ian Clatworthy | Approve | ||
Review via email:
|
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Miklos Vajna (vmiklos) wrote : | # |
lp:~vmiklos/bzr-fastimport/darcs
updated
- 257. By Ian Clatworthy
-
Get fastimport working on non-chk repositories again for bzr versions after 2.0.0
- 258. By Ian Clatworthy
-
Handle multi-level branches
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Ian Clatworthy (ian-clatworthy) : | # |
review:
Approve
lp:~vmiklos/bzr-fastimport/darcs
updated
- 259. By Ian Clatworthy
-
http read support for darcs-fast-export
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'exporters/darcs/darcs-fast-export' | |||
2 | --- exporters/darcs/darcs-fast-export 2009-09-23 22:09:43 +0000 | |||
3 | +++ exporters/darcs/darcs-fast-export 2009-10-22 10:35:17 +0000 | |||
4 | @@ -4,7 +4,7 @@ | |||
5 | 4 | 4 | ||
6 | 5 | darcs-fast-export - darcs backend for fast data importers | 5 | darcs-fast-export - darcs backend for fast data importers |
7 | 6 | 6 | ||
9 | 7 | Copyright (c) 2008 Miklos Vajna <vmiklos@frugalware.org> | 7 | Copyright (c) 2008, 2009 Miklos Vajna <vmiklos@frugalware.org> |
10 | 8 | Copyright (c) 2008 Matthias Andree <matthias.andree@gmx.de> | 8 | Copyright (c) 2008 Matthias Andree <matthias.andree@gmx.de> |
11 | 9 | 9 | ||
12 | 10 | This program is free software; you can redistribute it and/or modify | 10 | This program is free software; you can redistribute it and/or modify |
13 | @@ -33,312 +33,366 @@ | |||
14 | 33 | import subprocess | 33 | import subprocess |
15 | 34 | import optparse | 34 | import optparse |
16 | 35 | import re | 35 | import re |
17 | 36 | import urllib | ||
18 | 37 | import StringIO | ||
19 | 36 | 38 | ||
20 | 37 | sys = reload(sys) | 39 | sys = reload(sys) |
21 | 38 | sys.setdefaultencoding("utf-8") | 40 | sys.setdefaultencoding("utf-8") |
22 | 39 | 41 | ||
199 | 40 | def __get_zone(): | 42 | class Handler: |
200 | 41 | now = time.localtime() | 43 | def __init__(self): |
201 | 42 | if time.daylight and now[-1]: | 44 | self.hashes = [] |
202 | 43 | offset = time.altzone | 45 | self.authormap = {} |
203 | 44 | else: | 46 | self.export_marks = [] |
204 | 45 | offset = time.timezone | 47 | self.import_marks = [] |
205 | 46 | hours, minutes = divmod(abs(offset), 3600) | 48 | |
206 | 47 | if offset > 0: | 49 | def __get_zone(self): |
207 | 48 | sign = "-" | 50 | now = time.localtime() |
208 | 49 | else: | 51 | if time.daylight and now[-1]: |
209 | 50 | sign = "+" | 52 | offset = time.altzone |
210 | 51 | return sign, hours, minutes | 53 | else: |
211 | 52 | 54 | offset = time.timezone | |
212 | 53 | def get_zone_str(): | 55 | hours, minutes = divmod(abs(offset), 3600) |
213 | 54 | sign, hours, minutes = __get_zone() | 56 | if offset > 0: |
214 | 55 | return "%s%02d%02d" % (sign, hours, minutes // 60) | 57 | sign = "-" |
215 | 56 | 58 | else: | |
216 | 57 | def get_zone_int(): | 59 | sign = "+" |
217 | 58 | sign, hours, minutes = __get_zone() | 60 | return sign, hours, minutes |
218 | 59 | ret = hours*3600+minutes*60 | 61 | |
219 | 60 | if sign == "-": | 62 | def get_zone_str(self): |
220 | 61 | ret *= -1 | 63 | sign, hours, minutes = self.__get_zone() |
221 | 62 | return ret | 64 | return "%s%02d%02d" % (sign, hours, minutes // 60) |
222 | 63 | 65 | ||
223 | 64 | def get_patchname(patch): | 66 | def get_zone_int(self): |
224 | 65 | ret = [] | 67 | sign, hours, minutes = self.__get_zone() |
225 | 66 | s = "" | 68 | ret = hours*3600+minutes*60 |
226 | 67 | if patch.attributes['inverted'].value == 'True': | 69 | if sign == "-": |
227 | 68 | s = "UNDO: " | 70 | ret *= -1 |
228 | 69 | cs = patch.getElementsByTagName("name")[0].childNodes | 71 | return ret |
229 | 70 | if cs.length > 0: | 72 | |
230 | 71 | ret.append(s + cs[0].data) | 73 | def get_patchname(self, patch): |
231 | 72 | lines = patch.getElementsByTagName("comment") | 74 | ret = [] |
232 | 73 | if lines: | 75 | s = "" |
233 | 74 | for i in lines[0].childNodes[0].data.split('\n'): | 76 | if patch.attributes['inverted'].value == 'True': |
234 | 75 | if not i.startswith("Ignore-this: "): | 77 | s = "UNDO: " |
235 | 76 | ret.append(i) | 78 | cs = patch.getElementsByTagName("name")[0].childNodes |
236 | 77 | return "\n".join(ret).encode('utf-8') | 79 | if cs.length > 0: |
237 | 78 | 80 | ret.append(s + cs[0].data) | |
238 | 79 | def get_author(patch): | 81 | lines = patch.getElementsByTagName("comment") |
239 | 80 | """darcs allows any freeform string, but fast-import has a more | 82 | if lines: |
240 | 81 | strict format, so fix up broken author names here.""" | 83 | for i in lines[0].childNodes[0].data.split('\n'): |
241 | 82 | 84 | if not i.startswith("Ignore-this: "): | |
242 | 83 | author = patch.attributes['author'].value | 85 | ret.append(i) |
243 | 84 | if author in authormap: | 86 | return "\n".join(ret).encode('utf-8') |
244 | 85 | author = authormap[author] | 87 | |
245 | 86 | if not len(author): | 88 | def get_author(self, patch): |
246 | 87 | author = "darcs-fast-export <darcs-fast-export>" | 89 | """darcs allows any freeform string, but fast-import has a more |
247 | 88 | # add missing name | 90 | strict format, so fix up broken author names here.""" |
248 | 89 | elif not ">" in author: | 91 | |
249 | 90 | author = "%s <%s>" % (author.split('@')[0], author) | 92 | author = patch.attributes['author'].value |
250 | 91 | # avoid double quoting | 93 | if author in self.authormap: |
251 | 92 | elif author[0] == '"' and author[-1] == '"': | 94 | author = self.authormap[author] |
252 | 93 | author = author[1:-1] | 95 | if not len(author): |
253 | 94 | # name after email | 96 | author = "darcs-fast-export <darcs-fast-export>" |
254 | 95 | elif author[-1] != '>': | 97 | # add missing name |
255 | 96 | author = author[author.index('>')+2:] + ' ' + author[:author.index('>')+1] | 98 | elif not ">" in author: |
256 | 97 | return author.encode('utf-8') | 99 | author = "%s <%s>" % (author.split('@')[0], author) |
257 | 98 | 100 | # avoid double quoting | |
258 | 99 | def get_date(patch): | 101 | elif author[0] == '"' and author[-1] == '"': |
259 | 100 | try: | 102 | author = author[1:-1] |
260 | 101 | date = time.strptime(patch, "%Y%m%d%H%M%S") | 103 | # name after email |
261 | 102 | except ValueError: | 104 | elif author[-1] != '>': |
262 | 103 | date = time.strptime(patch[:19] + patch[-5:], '%a %b %d %H:%M:%S %Y') | 105 | author = author[author.index('>')+2:] + ' ' + author[:author.index('>')+1] |
263 | 104 | return int(time.mktime(date)) + get_zone_int() | 106 | return author.encode('utf-8') |
264 | 105 | 107 | ||
265 | 106 | def progress(s): | 108 | def get_date(self, patch): |
90 | 107 | print "progress [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s) | ||
91 | 108 | sys.stdout.flush() | ||
92 | 109 | |||
93 | 110 | def log(s): | ||
94 | 111 | logsock.write("[%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s)) | ||
95 | 112 | logsock.flush() | ||
96 | 113 | |||
97 | 114 | hashes = [] | ||
98 | 115 | def parse_inventory(sock=None): | ||
99 | 116 | prev = None | ||
100 | 117 | nextprev = False | ||
101 | 118 | buf = [] | ||
102 | 119 | if not sock: | ||
103 | 120 | sock = open(os.path.join("_darcs", "hashed_inventory")) | ||
104 | 121 | for i in sock.readlines(): | ||
105 | 122 | if i.startswith("hash"): | ||
106 | 123 | buf.insert(0, i[6:-1]) | ||
107 | 124 | if i.startswith("Starting with inventory:"): | ||
108 | 125 | nextprev = True | ||
109 | 126 | elif nextprev: | ||
110 | 127 | prev = i[:-1] | ||
111 | 128 | nextprev = False | ||
112 | 129 | sock.close() | ||
113 | 130 | for i in buf: | ||
114 | 131 | hashes.insert(0, i) | ||
115 | 132 | if prev: | ||
116 | 133 | sock = gzip.open(os.path.join("_darcs", "inventories", prev)) | ||
117 | 134 | parse_inventory(sock) | ||
118 | 135 | |||
119 | 136 | # Option Parser | ||
120 | 137 | usage="%prog [options] darcsrepo" | ||
121 | 138 | opp = optparse.OptionParser(usage=usage) | ||
122 | 139 | opp.add_option("--import-marks", metavar="IFILE", | ||
123 | 140 | help="read state for incremental imports from IFILE") | ||
124 | 141 | opp.add_option("--export-marks", metavar="OFILE", | ||
125 | 142 | help="write state for incremental imports from OFILE") | ||
126 | 143 | opp.add_option("--encoding", | ||
127 | 144 | help="encoding of log [default: %default], if unspecified and input isn't utf-8, guess") | ||
128 | 145 | opp.add_option("--authors-file", metavar="F", | ||
129 | 146 | help="read author transformations in old=new format from F") | ||
130 | 147 | opp.add_option("--working", metavar="W", | ||
131 | 148 | help="working directory which is removed at the end of non-incremental conversions") | ||
132 | 149 | opp.add_option("--logfile", metavar="L", | ||
133 | 150 | help="log file which contains the output of external programs invoked during the conversion") | ||
134 | 151 | opp.add_option("--git-branch", metavar="B", | ||
135 | 152 | help="git branch [default: refs/heads/master]") | ||
136 | 153 | opp.add_option("--progress", metavar="P", | ||
137 | 154 | help="insert progress statements after every n commit [default: 100]") | ||
138 | 155 | (options, args) = opp.parse_args() | ||
139 | 156 | if len(args) < 1: | ||
140 | 157 | opp.error("darcsrepo required") | ||
141 | 158 | |||
142 | 159 | export_marks = [] | ||
143 | 160 | import_marks = [] | ||
144 | 161 | if options.import_marks: | ||
145 | 162 | sock = open(options.import_marks) | ||
146 | 163 | for i in sock.readlines(): | ||
147 | 164 | line = i.strip() | ||
148 | 165 | if not len(line): | ||
149 | 166 | continue | ||
150 | 167 | import_marks.append(line.split(' ')[1]) | ||
151 | 168 | export_marks.append(line) | ||
152 | 169 | sock.close() | ||
153 | 170 | |||
154 | 171 | # read author mapping file in gitauthors format, | ||
155 | 172 | # i. e. in=out (one per # line) | ||
156 | 173 | authormap = {} | ||
157 | 174 | if options.authors_file: | ||
158 | 175 | sock = open(options.authors_file) | ||
159 | 176 | authormap = dict([i.strip().split('=',1) for i in sock]) | ||
160 | 177 | sock.close() | ||
161 | 178 | |||
162 | 179 | origin = os.path.abspath(args[0]) | ||
163 | 180 | if options.working: | ||
164 | 181 | working = os.path.abspath(options.working) | ||
165 | 182 | else: | ||
166 | 183 | working = "%s.darcs" % origin | ||
167 | 184 | patchfile = "%s.patch" % origin | ||
168 | 185 | if options.logfile: | ||
169 | 186 | logfile = os.path.abspath(options.logfile) | ||
170 | 187 | else: | ||
171 | 188 | logfile = "%s.log" % origin | ||
172 | 189 | logsock = open(logfile, "a") | ||
173 | 190 | if options.git_branch: | ||
174 | 191 | git_branch = options.git_branch | ||
175 | 192 | else: | ||
176 | 193 | git_branch = "refs/heads/master" | ||
177 | 194 | |||
178 | 195 | if options.progress: | ||
179 | 196 | prognum = int(options.progress) | ||
180 | 197 | else: | ||
181 | 198 | prognum = 100 | ||
182 | 199 | |||
183 | 200 | progress("getting list of patches") | ||
184 | 201 | if not len(import_marks): | ||
185 | 202 | sock = os.popen("darcs changes --xml --reverse --repo %s" % origin) | ||
186 | 203 | else: | ||
187 | 204 | sock = os.popen("darcs changes --xml --reverse --repo %s --from-match 'hash %s'" % (origin, import_marks[-1])) | ||
188 | 205 | buf = sock.read() | ||
189 | 206 | sock.close() | ||
190 | 207 | # this is hackish. we need to escape some bad chars, otherwise the xml | ||
191 | 208 | # will not be valid | ||
192 | 209 | buf = buf.replace('\x1b', '^[') | ||
193 | 210 | if options.encoding: | ||
194 | 211 | xmldoc = xml.dom.minidom.parseString(unicode(buf, options.encoding).encode('utf-8')) | ||
195 | 212 | else: | ||
196 | 213 | try: | ||
197 | 214 | xmldoc = xml.dom.minidom.parseString(buf) | ||
198 | 215 | except xml.parsers.expat.ExpatError: | ||
266 | 216 | try: | 109 | try: |
395 | 217 | import chardet | 110 | date = time.strptime(patch, "%Y%m%d%H%M%S") |
396 | 218 | except ImportError: | 111 | except ValueError: |
397 | 219 | sys.exit("Error, encoding is not utf-8. Please " + | 112 | date = time.strptime(patch[:19] + patch[-5:], '%a %b %d %H:%M:%S %Y') |
398 | 220 | "either specify it with the --encoding " + | 113 | return int(time.mktime(date)) + self.get_zone_int() |
399 | 221 | "option or install chardet.") | 114 | |
400 | 222 | progress("encoding is not utf8, guessing charset") | 115 | def progress(self, s): |
401 | 223 | encoding = chardet.detect(buf)['encoding'] | 116 | print "progress [%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s) |
402 | 224 | progress("detected encoding is %s" % encoding) | 117 | sys.stdout.flush() |
403 | 225 | xmldoc = xml.dom.minidom.parseString(unicode(buf, encoding).encode('utf-8')) | 118 | |
404 | 226 | sys.stdout.flush() | 119 | def log(self, s): |
405 | 227 | 120 | self.logsock.write("[%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), s)) | |
406 | 228 | darcs2 = False | 121 | self.logsock.flush() |
407 | 229 | oldfashionedpatch = True | 122 | |
408 | 230 | cwd = os.getcwd() | 123 | def parse_inventory(self, sock=None): |
409 | 231 | if os.path.exists(os.path.join(origin, "_darcs", "format")): | 124 | prev = None |
410 | 232 | sock = open(os.path.join(origin, "_darcs", "format")) | 125 | nextprev = False |
411 | 233 | format = [x.strip() for x in sock] | 126 | buf = [] |
412 | 234 | sock.close() | 127 | if not sock: |
413 | 235 | darcs2 = 'darcs-2' in format | 128 | sock = self.open(os.path.join(self.origin, "_darcs", "hashed_inventory")) |
414 | 236 | oldfashionedpatch = not 'hashed' in format | 129 | for i in sock.readlines(): |
415 | 237 | if not oldfashionedpatch: | 130 | if i.startswith("hash"): |
416 | 238 | progress("parsing the inventory") | 131 | buf.insert(0, i[6:-1]) |
417 | 239 | os.chdir(origin) | 132 | if i.startswith("Starting with inventory:"): |
418 | 240 | parse_inventory() | 133 | nextprev = True |
419 | 241 | if not options.import_marks or not os.path.exists(working): | 134 | elif nextprev: |
420 | 242 | # init the tmp darcs repo | 135 | prev = i[:-1] |
421 | 243 | os.mkdir(working) | 136 | nextprev = False |
422 | 244 | os.chdir(working) | 137 | sock.close() |
423 | 245 | if darcs2: | 138 | for i in buf: |
424 | 246 | os.system("darcs init --darcs-2") | 139 | self.hashes.insert(0, i) |
425 | 247 | else: | 140 | if prev: |
426 | 248 | os.system("darcs init --old-fashioned-inventory") | 141 | sock = self.gzip_open(os.path.join(self.origin, "_darcs", "inventories", prev)) |
427 | 249 | else: | 142 | self.parse_inventory(sock) |
428 | 250 | os.chdir(working) | 143 | |
429 | 251 | if options.import_marks: | 144 | # this is like gzip.open but supports urls as well |
430 | 252 | sock = os.popen("darcs pull -a --match 'hash %s' %s" % (import_marks[-1], origin)) | 145 | def gzip_open(self, path): |
431 | 253 | log("Building/updating working directory:\n%s" % sock.read()) | 146 | if os.path.exists(path): |
432 | 254 | sock.close() | 147 | return gzip.open(path) |
433 | 255 | 148 | buf = urllib.urlopen(path).read() | |
434 | 256 | # this is the number of the NEXT patch | 149 | sock = StringIO.StringIO(buf) |
435 | 257 | count = 1 | 150 | return gzip.GzipFile(fileobj=sock) |
436 | 258 | patches = xmldoc.getElementsByTagName('patch') | 151 | |
437 | 259 | if len(import_marks): | 152 | # this is like os.path.exists but supports urls as well |
438 | 260 | patches = patches[1:] | 153 | def path_exists(self, path): |
439 | 261 | count = len(import_marks) + 1 | 154 | if os.path.exists(path): |
440 | 262 | if len(export_marks): | 155 | return True |
441 | 263 | # this is the mark number of the NEXT patch | 156 | else: |
442 | 264 | markcount = int(export_marks[-1].split(' ')[0][1:]) + 1 | 157 | return urllib.urlopen(path).getcode() == 200 |
443 | 265 | else: | 158 | |
444 | 266 | markcount = count | 159 | # this is like open, but supports urls as well |
445 | 267 | # this may be huge and we need it many times | 160 | def open(self, path): |
446 | 268 | patchnum = len(patches) | 161 | if os.path.exists(path): |
447 | 269 | 162 | return open(path) | |
448 | 270 | if not len(import_marks): | 163 | else: |
449 | 271 | progress("starting export, repo has %d patches" % patchnum) | 164 | return urllib.urlopen(path) |
450 | 272 | else: | 165 | |
451 | 273 | progress("continuing export, %d patches to convert" % patchnum) | 166 | def handle_opts(self): |
452 | 274 | paths = [] | 167 | # Option Parser |
453 | 275 | for i in patches: | 168 | usage="%prog [options] darcsrepo" |
454 | 276 | # apply the patch | 169 | opp = optparse.OptionParser(usage=usage) |
455 | 277 | hash = i.attributes['hash'].value | 170 | opp.add_option("--import-marks", metavar="IFILE", |
456 | 278 | buf = ["\nNew patches:\n"] | 171 | help="read state for incremental imports from IFILE") |
457 | 279 | if oldfashionedpatch: | 172 | opp.add_option("--export-marks", metavar="OFILE", |
458 | 280 | sock = gzip.open(os.path.join(origin, "_darcs", "patches", hash)) | 173 | help="write state for incremental imports from OFILE") |
459 | 281 | else: | 174 | opp.add_option("--encoding", |
460 | 282 | sock = gzip.open(os.path.join(origin, "_darcs", "patches", hashes[count-1])) | 175 | help="encoding of log [default: %default], if unspecified and input isn't utf-8, guess") |
461 | 283 | buf.append(sock.read()) | 176 | opp.add_option("--authors-file", metavar="F", |
462 | 284 | sock.close() | 177 | help="read author transformations in old=new format from F") |
463 | 285 | sock = os.popen("darcs changes --context") | 178 | opp.add_option("--working", metavar="W", |
464 | 286 | buf.append(sock.read()) | 179 | help="working directory which is removed at the end of non-incremental conversions") |
465 | 287 | sock.close() | 180 | opp.add_option("--logfile", metavar="L", |
466 | 288 | sock = subprocess.Popen(["darcs", "apply", "--allow-conflicts"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) | 181 | help="log file which contains the output of external programs invoked during the conversion") |
467 | 289 | sock.stdin.write("".join(buf)) | 182 | opp.add_option("--git-branch", metavar="B", |
468 | 290 | sock.stdin.close() | 183 | help="git branch [default: refs/heads/master]") |
469 | 291 | log("Applying %s:\n%s" % (hash, sock.stdout.read())) | 184 | opp.add_option("--progress", metavar="P", |
470 | 292 | sock.stdout.close() | 185 | help="insert progress statements after every n commit [default: 100]") |
471 | 293 | message = get_patchname(i) | 186 | (self.options, self.args) = opp.parse_args() |
472 | 294 | # export the commit | 187 | if len(self.args) < 1: |
473 | 295 | print "commit %s" % git_branch | 188 | opp.error("darcsrepo required") |
474 | 296 | print "mark :%s" % markcount | 189 | |
475 | 297 | if options.export_marks: | 190 | # read author mapping file in gitauthors format, |
476 | 298 | export_marks.append(":%s %s" % (markcount, hash)) | 191 | # i. e. in=out (one per # line) |
477 | 299 | date = get_date(i.attributes['date'].value) | 192 | if self.options.authors_file: |
478 | 300 | print "committer %s %s %s" % (get_author(i), date, get_zone_str()) | 193 | sock = open(self.options.authors_file) |
479 | 301 | print "data %d\n%s" % (len(message), message) | 194 | self.authormap = dict([i.strip().split('=',1) for i in sock]) |
480 | 302 | if markcount > 1: | 195 | sock.close() |
481 | 303 | print "from :%s" % (markcount-1) | 196 | |
482 | 304 | # export the files | 197 | if "://" not in self.args[0]: |
483 | 305 | for j in paths: | 198 | self.origin = os.path.abspath(self.args[0]) |
484 | 306 | print "D %s" % j | 199 | else: |
485 | 307 | paths = [] | 200 | self.origin = self.args[0].strip('/') |
486 | 308 | for (root, dirs, files) in os.walk ("."): | 201 | if self.options.working: |
487 | 309 | for f in files: | 202 | self.working = os.path.abspath(self.options.working) |
488 | 310 | j = os.path.normpath(os.path.join(root, f)) | 203 | else: |
489 | 311 | if j.startswith("_darcs") or "-darcs-backup" in j: | 204 | if "://" not in self.origin: |
490 | 312 | continue | 205 | self.working = "%s.darcs" % self.origin |
491 | 313 | paths.append(j) | 206 | else: |
492 | 314 | sock = open(j) | 207 | self.working = "%s.darcs" % os.path.split(self.origin)[-1] |
493 | 315 | buf = sock.read() | 208 | if self.options.logfile: |
494 | 316 | sock.close() | 209 | logfile = os.path.abspath(self.options.logfile) |
495 | 317 | # darcs does not track the executable bit :/ | 210 | else: |
496 | 318 | print "M 644 inline %s" % j | 211 | if "://" not in self.origin: |
497 | 319 | print "data %s\n%s" % (len(buf), buf) | 212 | logfile = "%s.log" % self.origin |
498 | 320 | if message[:4] == "TAG ": | 213 | else: |
499 | 321 | tag = re.sub('[^\xe9-\xf8\w.\-]+', '_', message[4:].strip().split('\n')[0]).strip('_') | 214 | logfile = "%s.log" % os.path.split(self.origin)[-1] |
500 | 322 | print "tag %s" % tag | 215 | self.logsock = open(logfile, "a") |
501 | 323 | print "from :%s" % markcount | 216 | if self.options.git_branch: |
502 | 324 | print "tagger %s %s %s" % (get_author(i), date, get_zone_str()) | 217 | self.git_branch = self.options.git_branch |
503 | 325 | print "data %d\n%s" % (len(message), message) | 218 | else: |
504 | 326 | if count % prognum == 0: | 219 | self.git_branch = "refs/heads/master" |
505 | 327 | progress("%d/%d patches" % (count, patchnum)) | 220 | |
506 | 328 | count += 1 | 221 | if self.options.progress: |
507 | 329 | markcount += 1 | 222 | self.prognum = int(self.options.progress) |
508 | 330 | 223 | else: | |
509 | 331 | os.chdir(cwd) | 224 | self.prognum = 100 |
510 | 332 | 225 | ||
511 | 333 | if not options.export_marks: | 226 | def handle_import_marks(self): |
512 | 334 | shutil.rmtree(working) | 227 | if self.options.import_marks: |
513 | 335 | logsock.close() | 228 | sock = open(self.options.import_marks) |
514 | 336 | 229 | for i in sock.readlines(): | |
515 | 337 | if options.export_marks: | 230 | line = i.strip() |
516 | 338 | progress("writing export marks") | 231 | if not len(line): |
517 | 339 | sock = open(options.export_marks, 'w') | 232 | continue |
518 | 340 | sock.write("\n".join(export_marks)) | 233 | self.import_marks.append(line.split(' ')[1]) |
519 | 341 | sock.write("\n") | 234 | self.export_marks.append(line) |
520 | 342 | sock.close() | 235 | sock.close() |
521 | 343 | 236 | ||
522 | 344 | progress("finished") | 237 | def get_patches(self): |
523 | 238 | self.progress("getting list of patches") | ||
524 | 239 | if not len(self.import_marks): | ||
525 | 240 | sock = os.popen("darcs changes --xml --reverse --repo %s" % self.origin) | ||
526 | 241 | else: | ||
527 | 242 | sock = os.popen("darcs changes --xml --reverse --repo %s --from-match 'hash %s'" % (self.origin, self.import_marks[-1])) | ||
528 | 243 | buf = sock.read() | ||
529 | 244 | sock.close() | ||
530 | 245 | # this is hackish. we need to escape some bad chars, otherwise the xml | ||
531 | 246 | # will not be valid | ||
532 | 247 | buf = buf.replace('\x1b', '^[') | ||
533 | 248 | if self.options.encoding: | ||
534 | 249 | xmldoc = xml.dom.minidom.parseString(unicode(buf, self.options.encoding).encode('utf-8')) | ||
535 | 250 | else: | ||
536 | 251 | try: | ||
537 | 252 | xmldoc = xml.dom.minidom.parseString(buf) | ||
538 | 253 | except xml.parsers.expat.ExpatError: | ||
539 | 254 | try: | ||
540 | 255 | import chardet | ||
541 | 256 | except ImportError: | ||
542 | 257 | sys.exit("Error, encoding is not utf-8. Please " + | ||
543 | 258 | "either specify it with the --encoding " + | ||
544 | 259 | "option or install chardet.") | ||
545 | 260 | self.progress("encoding is not utf8, guessing charset") | ||
546 | 261 | encoding = chardet.detect(buf)['encoding'] | ||
547 | 262 | self.progress("detected encoding is %s" % encoding) | ||
548 | 263 | xmldoc = xml.dom.minidom.parseString(unicode(buf, encoding).encode('utf-8')) | ||
549 | 264 | sys.stdout.flush() | ||
550 | 265 | return xmldoc.getElementsByTagName('patch') | ||
551 | 266 | |||
552 | 267 | def setup_workdir(self): | ||
553 | 268 | darcs2 = False | ||
554 | 269 | self.oldfashionedpatch = True | ||
555 | 270 | self.cwd = os.getcwd() | ||
556 | 271 | if self.path_exists(os.path.join(self.origin, "_darcs", "format")): | ||
557 | 272 | sock = self.open(os.path.join(self.origin, "_darcs", "format")) | ||
558 | 273 | format = [x.strip() for x in sock] | ||
559 | 274 | sock.close() | ||
560 | 275 | darcs2 = 'darcs-2' in format | ||
561 | 276 | self.oldfashionedpatch = not 'hashed' in format | ||
562 | 277 | if not self.oldfashionedpatch: | ||
563 | 278 | self.progress("parsing the inventory") | ||
564 | 279 | if "://" not in self.origin: | ||
565 | 280 | os.chdir(self.origin) | ||
566 | 281 | self.parse_inventory() | ||
567 | 282 | if not self.options.import_marks or not os.path.exists(self.working): | ||
568 | 283 | # init the tmp darcs repo | ||
569 | 284 | os.mkdir(self.working) | ||
570 | 285 | os.chdir(self.working) | ||
571 | 286 | if darcs2: | ||
572 | 287 | os.system("darcs init --darcs-2") | ||
573 | 288 | else: | ||
574 | 289 | os.system("darcs init --old-fashioned-inventory") | ||
575 | 290 | else: | ||
576 | 291 | os.chdir(self.working) | ||
577 | 292 | if self.options.import_marks: | ||
578 | 293 | sock = os.popen("darcs pull -a --match 'hash %s' %s" % (self.import_marks[-1], self.origin)) | ||
579 | 294 | self.log("Building/updating working directory:\n%s" % sock.read()) | ||
580 | 295 | sock.close() | ||
581 | 296 | |||
582 | 297 | def export_patches(self): | ||
583 | 298 | patches = self.get_patches() | ||
584 | 299 | # this is the number of the NEXT patch | ||
585 | 300 | count = 1 | ||
586 | 301 | if len(self.import_marks): | ||
587 | 302 | patches = patches[1:] | ||
588 | 303 | count = len(self.import_marks) + 1 | ||
589 | 304 | if len(self.export_marks): | ||
590 | 305 | # this is the mark number of the NEXT patch | ||
591 | 306 | markcount = int(self.export_marks[-1].split(' ')[0][1:]) + 1 | ||
592 | 307 | else: | ||
593 | 308 | markcount = count | ||
594 | 309 | # this may be huge and we need it many times | ||
595 | 310 | patchnum = len(patches) | ||
596 | 311 | |||
597 | 312 | if not len(self.import_marks): | ||
598 | 313 | self.progress("starting export, repo has %d patches" % patchnum) | ||
599 | 314 | else: | ||
600 | 315 | self.progress("continuing export, %d patches to convert" % patchnum) | ||
601 | 316 | paths = [] | ||
602 | 317 | for i in patches: | ||
603 | 318 | # apply the patch | ||
604 | 319 | hash = i.attributes['hash'].value | ||
605 | 320 | buf = ["\nNew patches:\n"] | ||
606 | 321 | if self.oldfashionedpatch: | ||
607 | 322 | sock = self.gzip_open(os.path.join(self.origin, "_darcs", "patches", hash)) | ||
608 | 323 | else: | ||
609 | 324 | sock = self.gzip_open(os.path.join(self.origin, "_darcs", "patches", self.hashes[count-1])) | ||
610 | 325 | buf.append(sock.read()) | ||
611 | 326 | sock.close() | ||
612 | 327 | sock = os.popen("darcs changes --context") | ||
613 | 328 | buf.append(sock.read()) | ||
614 | 329 | sock.close() | ||
615 | 330 | sock = subprocess.Popen(["darcs", "apply", "--allow-conflicts"], stdin=subprocess.PIPE, stdout=subprocess.PIPE) | ||
616 | 331 | sock.stdin.write("".join(buf)) | ||
617 | 332 | sock.stdin.close() | ||
618 | 333 | self.log("Applying %s:\n%s" % (hash, sock.stdout.read())) | ||
619 | 334 | sock.stdout.close() | ||
620 | 335 | message = self.get_patchname(i) | ||
621 | 336 | # export the commit | ||
622 | 337 | print "commit %s" % self.git_branch | ||
623 | 338 | print "mark :%s" % markcount | ||
624 | 339 | if self.options.export_marks: | ||
625 | 340 | self.export_marks.append(":%s %s" % (markcount, hash)) | ||
626 | 341 | date = self.get_date(i.attributes['date'].value) | ||
627 | 342 | print "committer %s %s %s" % (self.get_author(i), date, self.get_zone_str()) | ||
628 | 343 | print "data %d\n%s" % (len(message), message) | ||
629 | 344 | if markcount > 1: | ||
630 | 345 | print "from :%s" % (markcount-1) | ||
631 | 346 | # export the files | ||
632 | 347 | for j in paths: | ||
633 | 348 | print "D %s" % j | ||
634 | 349 | paths = [] | ||
635 | 350 | for (root, dirs, files) in os.walk ("."): | ||
636 | 351 | for f in files: | ||
637 | 352 | j = os.path.normpath(os.path.join(root, f)) | ||
638 | 353 | if j.startswith("_darcs") or "-darcs-backup" in j: | ||
639 | 354 | continue | ||
640 | 355 | paths.append(j) | ||
641 | 356 | sock = open(j) | ||
642 | 357 | buf = sock.read() | ||
643 | 358 | sock.close() | ||
644 | 359 | # darcs does not track the executable bit :/ | ||
645 | 360 | print "M 644 inline %s" % j | ||
646 | 361 | print "data %s\n%s" % (len(buf), buf) | ||
647 | 362 | if message[:4] == "TAG ": | ||
648 | 363 | tag = re.sub('[^\xe9-\xf8\w.\-]+', '_', message[4:].strip().split('\n')[0]).strip('_') | ||
649 | 364 | print "tag %s" % tag | ||
650 | 365 | print "from :%s" % markcount | ||
651 | 366 | print "tagger %s %s %s" % (self.get_author(i), date, self.get_zone_str()) | ||
652 | 367 | print "data %d\n%s" % (len(message), message) | ||
653 | 368 | if count % self.prognum == 0: | ||
654 | 369 | self.progress("%d/%d patches" % (count, patchnum)) | ||
655 | 370 | count += 1 | ||
656 | 371 | markcount += 1 | ||
657 | 372 | |||
658 | 373 | os.chdir(self.cwd) | ||
659 | 374 | |||
660 | 375 | if not self.options.export_marks: | ||
661 | 376 | shutil.rmtree(self.working) | ||
662 | 377 | self.logsock.close() | ||
663 | 378 | |||
664 | 379 | def handle_export_marks(self): | ||
665 | 380 | if self.options.export_marks: | ||
666 | 381 | self.progress("writing export marks") | ||
667 | 382 | sock = open(self.options.export_marks, 'w') | ||
668 | 383 | sock.write("\n".join(self.export_marks)) | ||
669 | 384 | sock.write("\n") | ||
670 | 385 | sock.close() | ||
671 | 386 | |||
672 | 387 | self.progress("finished") | ||
673 | 388 | |||
674 | 389 | def handle(self): | ||
675 | 390 | self.handle_opts() | ||
676 | 391 | self.handle_import_marks() | ||
677 | 392 | self.setup_workdir() | ||
678 | 393 | self.export_patches() | ||
679 | 394 | self.handle_export_marks() | ||
680 | 395 | |||
681 | 396 | if __name__ == "__main__": | ||
682 | 397 | h = Handler() | ||
683 | 398 | h.handle() | ||
684 | 345 | 399 | ||
685 | === modified file 'exporters/darcs/darcs-fast-export.txt' | |||
686 | --- exporters/darcs/darcs-fast-export.txt 2009-08-10 22:20:43 +0000 | |||
687 | +++ exporters/darcs/darcs-fast-export.txt 2009-10-22 10:35:17 +0000 | |||
688 | @@ -18,6 +18,10 @@ | |||
689 | 18 | repository. It supports incremental conversion as well, via the | 18 | repository. It supports incremental conversion as well, via the |
690 | 19 | --import-marks / --export-marks switches. | 19 | --import-marks / --export-marks switches. |
691 | 20 | 20 | ||
692 | 21 | Optionally the darcsrepo string may be a HTTP repository, in that case | ||
693 | 22 | only the patches are downloaded, not the pristine, speeding up a | ||
694 | 23 | one-time import. | ||
695 | 24 | |||
696 | 21 | == OPTIONS | 25 | == OPTIONS |
697 | 22 | 26 | ||
698 | 23 | -h, --help:: | 27 | -h, --help:: |
699 | 24 | 28 | ||
700 | === added file 'exporters/darcs/t/lib-httpd.sh' | |||
701 | --- exporters/darcs/t/lib-httpd.sh 1970-01-01 00:00:00 +0000 | |||
702 | +++ exporters/darcs/t/lib-httpd.sh 2009-10-22 10:35:18 +0000 | |||
703 | @@ -0,0 +1,67 @@ | |||
704 | 1 | #!/bin/sh | ||
705 | 2 | # | ||
706 | 3 | # This is based on git's t/lib-httpd.sh, which is | ||
707 | 4 | # Copyright (c) 2008 Clemens Buchacher <drizzd@aon.at> | ||
708 | 5 | # | ||
709 | 6 | |||
710 | 7 | if test -n "$DFE_TEST_SKIP_HTTPD" | ||
711 | 8 | then | ||
712 | 9 | echo "skipping test (undef DFE_TEST_SKIP_HTTPD to enable)" | ||
713 | 10 | exit | ||
714 | 11 | fi | ||
715 | 12 | |||
716 | 13 | LIB_HTTPD_PATH=${LIB_HTTPD_PATH-'/usr/sbin/httpd'} | ||
717 | 14 | LIB_HTTPD_PORT=${LIB_HTTPD_PORT-'8111'} | ||
718 | 15 | |||
719 | 16 | HTTPD_ROOT_PATH="$PWD"/httpd | ||
720 | 17 | HTTPD_DOCUMENT_ROOT_PATH=$HTTPD_ROOT_PATH/www | ||
721 | 18 | |||
722 | 19 | if ! test -x "$LIB_HTTPD_PATH" | ||
723 | 20 | then | ||
724 | 21 | echo "skipping test, no web server found at '$LIB_HTTPD_PATH'" | ||
725 | 22 | exit | ||
726 | 23 | fi | ||
727 | 24 | |||
728 | 25 | HTTPD_VERSION=`$LIB_HTTPD_PATH -v | \ | ||
729 | 26 | sed -n 's/^Server version: Apache\/\([0-9]*\)\..*$/\1/p; q'` | ||
730 | 27 | |||
731 | 28 | if test -n "$HTTPD_VERSION" | ||
732 | 29 | then | ||
733 | 30 | if test -z "$LIB_HTTPD_MODULE_PATH" | ||
734 | 31 | then | ||
735 | 32 | if ! test $HTTPD_VERSION -ge 2 | ||
736 | 33 | then | ||
737 | 34 | echo "skipping test, at least Apache version 2 is required" | ||
738 | 35 | exit | ||
739 | 36 | fi | ||
740 | 37 | |||
741 | 38 | LIB_HTTPD_MODULE_PATH='/usr/lib/apache' | ||
742 | 39 | fi | ||
743 | 40 | else | ||
744 | 41 | error "Could not identify web server at '$LIB_HTTPD_PATH'" | ||
745 | 42 | fi | ||
746 | 43 | |||
747 | 44 | HTTPD_PARA="-d $HTTPD_ROOT_PATH -f $HTTPD_ROOT_PATH/apache.conf" | ||
748 | 45 | |||
749 | 46 | prepare_httpd() { | ||
750 | 47 | mkdir -p $HTTPD_DOCUMENT_ROOT_PATH | ||
751 | 48 | |||
752 | 49 | ln -s $LIB_HTTPD_MODULE_PATH $HTTPD_ROOT_PATH/modules | ||
753 | 50 | |||
754 | 51 | echo "PidFile httpd.pid" > $HTTPD_ROOT_PATH/apache.conf | ||
755 | 52 | echo "DocumentRoot www" >> $HTTPD_ROOT_PATH/apache.conf | ||
756 | 53 | echo "ErrorLog error.log" >> $HTTPD_ROOT_PATH/apache.conf | ||
757 | 54 | |||
758 | 55 | HTTPD_URL=http://127.0.0.1:$LIB_HTTPD_PORT | ||
759 | 56 | } | ||
760 | 57 | |||
761 | 58 | start_httpd() { | ||
762 | 59 | prepare_httpd | ||
763 | 60 | |||
764 | 61 | "$LIB_HTTPD_PATH" $HTTPD_PARA \ | ||
765 | 62 | -c "Listen 127.0.0.1:$LIB_HTTPD_PORT" -k start | ||
766 | 63 | } | ||
767 | 64 | |||
768 | 65 | stop_httpd() { | ||
769 | 66 | "$LIB_HTTPD_PATH" $HTTPD_PARA -k stop | ||
770 | 67 | } | ||
771 | 0 | 68 | ||
772 | === added file 'exporters/darcs/t/test2-git-http.sh' | |||
773 | --- exporters/darcs/t/test2-git-http.sh 1970-01-01 00:00:00 +0000 | |||
774 | +++ exporters/darcs/t/test2-git-http.sh 2009-10-22 10:35:18 +0000 | |||
775 | @@ -0,0 +1,22 @@ | |||
776 | 1 | . ./lib.sh | ||
777 | 2 | . ./lib-httpd.sh | ||
778 | 3 | |||
779 | 4 | rm -rf test2.darcs test2.git httpd | ||
780 | 5 | create_darcs test2 --darcs-2 | ||
781 | 6 | mkdir -p $HTTPD_DOCUMENT_ROOT_PATH | ||
782 | 7 | mv -v test2 $HTTPD_DOCUMENT_ROOT_PATH | ||
783 | 8 | ln -s $HTTPD_DOCUMENT_ROOT_PATH/test2 . | ||
784 | 9 | |||
785 | 10 | mkdir test2.git | ||
786 | 11 | cd test2.git | ||
787 | 12 | git --bare init | ||
788 | 13 | cd .. | ||
789 | 14 | start_httpd | ||
790 | 15 | darcs-fast-export $HTTPD_URL/test2 |(cd test2.git; git fast-import) | ||
791 | 16 | ret=$? | ||
792 | 17 | stop_httpd | ||
793 | 18 | if [ $ret != 0 ]; then | ||
794 | 19 | exit $ret | ||
795 | 20 | fi | ||
796 | 21 | diff_git test2 | ||
797 | 22 | exit $? |
The external change is the http read support (the relevant testcase was a real challenge ;) ), but internal one is a massive refactoring to a Python class.