Merge lp:~jelmer/bzr-builddeb/multiple-upstream-tarballs-pt1 into lp:bzr-builddeb
- multiple-upstream-tarballs-pt1
- Merge into trunk
Proposed by
Jelmer Vernooij
Status: | Merged |
---|---|
Approved by: | James Westby |
Approved revision: | 573 |
Merged at revision: | 570 |
Proposed branch: | lp:~jelmer/bzr-builddeb/multiple-upstream-tarballs-pt1 |
Merge into: | lp:bzr-builddeb |
Diff against target: |
601 lines (+141/-105) 8 files modified
cmds.py (+30/-24) dh_make.py (+11/-5) errors.py (+4/-3) source_distiller.py (+6/-2) tests/test_upstream.py (+33/-18) upstream/__init__.py (+55/-51) upstream/branch.py (+1/-1) upstream/pristinetar.py (+1/-1) |
To merge this branch: | bzr merge lp:~jelmer/bzr-builddeb/multiple-upstream-tarballs-pt1 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
James Westby | Approve | ||
Review via email:
|
Commit message
Description of the change
Initial work on supporting multiple upstream tarballs in bzr-builddeb.
This updates the UpstreamSource interface to return multiple paths for upstream tarballs and updates the AptSource and GetOrigSource implementations to actually return multiple tarballs.
Merging multiple upstream tarballs and extracting/
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
James Westby (james-w) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'cmds.py' | |||
2 | --- cmds.py 2011-06-03 12:17:42 +0000 | |||
3 | +++ cmds.py 2011-06-14 14:11:23 +0000 | |||
4 | @@ -69,6 +69,7 @@ | |||
5 | 69 | BuildFailedError, | 69 | BuildFailedError, |
6 | 70 | DchError, | 70 | DchError, |
7 | 71 | MissingChangelogError, | 71 | MissingChangelogError, |
8 | 72 | MultipleUpstreamTarballsNotSupported, | ||
9 | 72 | NoPreviousUpload, | 73 | NoPreviousUpload, |
10 | 73 | PackageVersionNotPresent, | 74 | PackageVersionNotPresent, |
11 | 74 | StrictBuildFailed, | 75 | StrictBuildFailed, |
12 | @@ -553,45 +554,50 @@ | |||
13 | 553 | 'merge had completed failed. Add the new changelog ' | 554 | 'merge had completed failed. Add the new changelog ' |
14 | 554 | 'entry yourself, review the merge, and then commit.') | 555 | 'entry yourself, review the merge, and then commit.') |
15 | 555 | 556 | ||
17 | 556 | def _do_merge(self, tree, tarball_filename, package, version, | 557 | def _do_merge(self, tree, tarball_filenames, package, version, |
18 | 557 | current_version, upstream_branch, upstream_revision, merge_type, | 558 | current_version, upstream_branch, upstream_revision, merge_type, |
19 | 558 | force): | 559 | force): |
20 | 559 | db = DistributionBranch(tree.branch, None, tree=tree) | 560 | db = DistributionBranch(tree.branch, None, tree=tree) |
21 | 560 | dbs = DistributionBranchSet() | 561 | dbs = DistributionBranchSet() |
22 | 561 | dbs.add_branch(db) | 562 | dbs.add_branch(db) |
24 | 562 | conflicts = db.merge_upstream(tarball_filename, package, version, | 563 | if len(tarball_filenames) > 1: |
25 | 564 | raise MultipleUpstreamTarballsNotSupported() | ||
26 | 565 | conflicts = db.merge_upstream(tarball_filenames[0], package, version, | ||
27 | 563 | current_version, upstream_branch=upstream_branch, | 566 | current_version, upstream_branch=upstream_branch, |
28 | 564 | upstream_revision=upstream_revision, | 567 | upstream_revision=upstream_revision, |
29 | 565 | merge_type=merge_type, force=force) | 568 | merge_type=merge_type, force=force) |
30 | 566 | return conflicts | 569 | return conflicts |
31 | 567 | 570 | ||
33 | 568 | def _fetch_tarball(self, package, version, orig_dir, location, v3): | 571 | def _fetch_tarball(self, package, version, orig_dir, locations, v3): |
34 | 569 | from bzrlib.plugins.builddeb.repack_tarball import repack_tarball | 572 | from bzrlib.plugins.builddeb.repack_tarball import repack_tarball |
35 | 573 | ret = [] | ||
36 | 570 | format = None | 574 | format = None |
52 | 571 | if v3: | 575 | for location in locations: |
53 | 572 | if location.endswith(".tar.bz2") or location.endswith(".tbz2"): | 576 | if v3: |
54 | 573 | format = "bz2" | 577 | if location.endswith(".tar.bz2") or location.endswith(".tbz2"): |
55 | 574 | dest_name = tarball_name(package, version, format=format) | 578 | format = "bz2" |
56 | 575 | tarball_filename = os.path.join(orig_dir, dest_name) | 579 | dest_name = tarball_name(package, version, format=format) |
57 | 576 | try: | 580 | tarball_filename = os.path.join(orig_dir, dest_name) |
58 | 577 | repack_tarball(location, dest_name, target_dir=orig_dir, | 581 | try: |
59 | 578 | force_gz=not v3) | 582 | repack_tarball(location, dest_name, target_dir=orig_dir, |
60 | 579 | except FileExists: | 583 | force_gz=not v3) |
61 | 580 | raise BzrCommandError("The target file %s already exists, and is either " | 584 | except FileExists: |
62 | 581 | "different to the new upstream tarball, or they " | 585 | raise BzrCommandError("The target file %s already exists, and is either " |
63 | 582 | "are of different formats. Either delete the target " | 586 | "different to the new upstream tarball, or they " |
64 | 583 | "file, or use it as the argument to import." | 587 | "are of different formats. Either delete the target " |
65 | 584 | % dest_name) | 588 | "file, or use it as the argument to import." |
66 | 585 | return tarball_filename | 589 | % dest_name) |
67 | 590 | ret.append(tarball_filename) | ||
68 | 591 | return ret | ||
69 | 586 | 592 | ||
70 | 587 | def _get_tarball(self, config, tree, package, version, upstream_branch, | 593 | def _get_tarball(self, config, tree, package, version, upstream_branch, |
72 | 588 | upstream_revision, v3, location): | 594 | upstream_revision, v3, locations): |
73 | 589 | orig_dir = config.orig_dir or default_orig_dir | 595 | orig_dir = config.orig_dir or default_orig_dir |
74 | 590 | orig_dir = os.path.join(tree.basedir, orig_dir) | 596 | orig_dir = os.path.join(tree.basedir, orig_dir) |
75 | 591 | if not os.path.exists(orig_dir): | 597 | if not os.path.exists(orig_dir): |
76 | 592 | os.makedirs(orig_dir) | 598 | os.makedirs(orig_dir) |
77 | 593 | return self._fetch_tarball(package, version, orig_dir, | 599 | return self._fetch_tarball(package, version, orig_dir, |
79 | 594 | location, v3) | 600 | locations, v3) |
80 | 595 | 601 | ||
81 | 596 | def _get_changelog_info(self, tree, last_version, package, distribution): | 602 | def _get_changelog_info(self, tree, last_version, package, distribution): |
82 | 597 | current_version = last_version | 603 | current_version = last_version |
83 | @@ -738,15 +744,15 @@ | |||
84 | 738 | (version, upstream_branch_source)) | 744 | (version, upstream_branch_source)) |
85 | 739 | if need_upstream_tarball: | 745 | if need_upstream_tarball: |
86 | 740 | target_dir = tempfile.mkdtemp() # FIXME: Cleanup? | 746 | target_dir = tempfile.mkdtemp() # FIXME: Cleanup? |
88 | 741 | location = primary_upstream_source.fetch_tarball( | 747 | locations = primary_upstream_source.fetch_tarball( |
89 | 742 | package, version, target_dir) | 748 | package, version, target_dir) |
90 | 743 | source_format = get_source_format(tree) | 749 | source_format = get_source_format(tree) |
91 | 744 | v3 = (source_format in [ | 750 | v3 = (source_format in [ |
92 | 745 | FORMAT_3_0_QUILT, FORMAT_3_0_NATIVE]) | 751 | FORMAT_3_0_QUILT, FORMAT_3_0_NATIVE]) |
94 | 746 | tarball_filename = self._get_tarball(config, tree, package, | 752 | tarball_filenames = self._get_tarball(config, tree, package, |
95 | 747 | version, upstream_branch, upstream_revision, v3, | 753 | version, upstream_branch, upstream_revision, v3, |
98 | 748 | location) | 754 | locations) |
99 | 749 | conflicts = self._do_merge(tree, tarball_filename, package, | 755 | conflicts = self._do_merge(tree, tarball_filenames, package, |
100 | 750 | version, current_version, upstream_branch, upstream_revision, | 756 | version, current_version, upstream_branch, upstream_revision, |
101 | 751 | merge_type, force) | 757 | merge_type, force) |
102 | 752 | if current_version is not None and Version(current_version) >= Version(version): | 758 | if current_version is not None and Version(current_version) >= Version(version): |
103 | 753 | 759 | ||
104 | === modified file 'dh_make.py' | |||
105 | --- dh_make.py 2011-02-02 18:33:23 +0000 | |||
106 | +++ dh_make.py 2011-06-14 14:11:23 +0000 | |||
107 | @@ -13,6 +13,7 @@ | |||
108 | 13 | 13 | ||
109 | 14 | from bzrlib.plugins.builddeb import ( | 14 | from bzrlib.plugins.builddeb import ( |
110 | 15 | default_orig_dir, | 15 | default_orig_dir, |
111 | 16 | errors, | ||
112 | 16 | import_dsc, | 17 | import_dsc, |
113 | 17 | upstream, | 18 | upstream, |
114 | 18 | util, | 19 | util, |
115 | @@ -50,7 +51,7 @@ | |||
116 | 50 | return tree | 51 | return tree |
117 | 51 | 52 | ||
118 | 52 | 53 | ||
120 | 53 | def _get_tarball(tree, tarball, package_name, version, use_v3=False): | 54 | def _get_tarballs(tree, tarball, package_name, version, use_v3=False): |
121 | 54 | from bzrlib.plugins.builddeb.repack_tarball import repack_tarball | 55 | from bzrlib.plugins.builddeb.repack_tarball import repack_tarball |
122 | 55 | config = util.debuild_config(tree, tree) | 56 | config = util.debuild_config(tree, tree) |
123 | 56 | orig_dir = config.orig_dir or default_orig_dir | 57 | orig_dir = config.orig_dir or default_orig_dir |
124 | @@ -62,14 +63,16 @@ | |||
125 | 62 | if tarball.endswith(".tar.bz2") or tarball.endswith(".tbz2"): | 63 | if tarball.endswith(".tar.bz2") or tarball.endswith(".tbz2"): |
126 | 63 | format = "bz2" | 64 | format = "bz2" |
127 | 64 | dest_name = util.tarball_name(package_name, version, format=format) | 65 | dest_name = util.tarball_name(package_name, version, format=format) |
128 | 65 | tarball_filename = os.path.join(orig_dir, dest_name) | ||
129 | 66 | trace.note("Fetching tarball") | 66 | trace.note("Fetching tarball") |
130 | 67 | repack_tarball(tarball, dest_name, target_dir=orig_dir, | 67 | repack_tarball(tarball, dest_name, target_dir=orig_dir, |
131 | 68 | force_gz=not use_v3) | 68 | force_gz=not use_v3) |
132 | 69 | provider = upstream.UpstreamProvider(package_name, version, | 69 | provider = upstream.UpstreamProvider(package_name, version, |
133 | 70 | orig_dir, []) | 70 | orig_dir, []) |
136 | 71 | provider.provide(os.path.join(tree.basedir, "..")) | 71 | orig_files = provider.provide(os.path.join(tree.basedir, "..")) |
137 | 72 | return tarball_filename, util.md5sum_filename(tarball_filename) | 72 | ret = [] |
138 | 73 | for filename in orig_files: | ||
139 | 74 | ret.append((filename, util.md5sum_filename(filename))) | ||
140 | 75 | return ret | ||
141 | 73 | 76 | ||
142 | 74 | 77 | ||
143 | 75 | def import_upstream(tarball, package_name, version, use_v3=False): | 78 | def import_upstream(tarball, package_name, version, use_v3=False): |
144 | @@ -78,12 +81,15 @@ | |||
145 | 78 | parents = [tree.branch.last_revision()] | 81 | parents = [tree.branch.last_revision()] |
146 | 79 | else: | 82 | else: |
147 | 80 | parents = [] | 83 | parents = [] |
149 | 81 | tarball_filename, md5sum = _get_tarball(tree, tarball, | 84 | tarball_filenames = _get_tarballs(tree, tarball, |
150 | 82 | package_name, version, use_v3=use_v3) | 85 | package_name, version, use_v3=use_v3) |
151 | 83 | db = import_dsc.DistributionBranch(tree.branch, tree.branch, tree=tree, | 86 | db = import_dsc.DistributionBranch(tree.branch, tree.branch, tree=tree, |
152 | 84 | upstream_tree=tree) | 87 | upstream_tree=tree) |
153 | 85 | dbs = import_dsc.DistributionBranchSet() | 88 | dbs = import_dsc.DistributionBranchSet() |
154 | 86 | dbs.add_branch(db) | 89 | dbs.add_branch(db) |
155 | 90 | if len(tarball_filenames) > 1: | ||
156 | 91 | raise errors.MultipleUpstreamTarballsNotSupported() | ||
157 | 92 | (tarball_filename, md5sum) = tarball_filenames[0] | ||
158 | 87 | db.import_upstream_tarball(tarball_filename, version, parents, md5sum=md5sum) | 93 | db.import_upstream_tarball(tarball_filename, version, parents, md5sum=md5sum) |
159 | 88 | return tree | 94 | return tree |
160 | 89 | 95 | ||
161 | 90 | 96 | ||
162 | === modified file 'errors.py' | |||
163 | --- errors.py 2011-06-09 21:43:07 +0000 | |||
164 | +++ errors.py 2011-06-14 14:11:23 +0000 | |||
165 | @@ -34,10 +34,11 @@ | |||
166 | 34 | 34 | ||
167 | 35 | 35 | ||
168 | 36 | class MissingUpstreamTarball(BzrError): | 36 | class MissingUpstreamTarball(BzrError): |
170 | 37 | _fmt = "Unable to find the needed upstream tarball: %(tarball_name)s." | 37 | _fmt = ("Unable to find the needed upstream tarball for package %(package)s, " |
171 | 38 | "version %(version)s.") | ||
172 | 38 | 39 | ||
175 | 39 | def __init__(self, tarball_name): | 40 | def __init__(self, package, version): |
176 | 40 | BzrError.__init__(self, tarball_name=tarball_name) | 41 | BzrError.__init__(self, package=package, version=version) |
177 | 41 | 42 | ||
178 | 42 | 43 | ||
179 | 43 | class TarFailed(BzrError): | 44 | class TarFailed(BzrError): |
180 | 44 | 45 | ||
181 | === modified file 'source_distiller.py' | |||
182 | --- source_distiller.py 2010-03-11 23:44:57 +0000 | |||
183 | +++ source_distiller.py 2011-06-14 14:11:23 +0000 | |||
184 | @@ -26,6 +26,7 @@ | |||
185 | 26 | from bzrlib import errors as bzr_errors | 26 | from bzrlib import errors as bzr_errors |
186 | 27 | 27 | ||
187 | 28 | from bzrlib.plugins.builddeb.errors import ( | 28 | from bzrlib.plugins.builddeb.errors import ( |
188 | 29 | MultipleUpstreamTarballsNotSupported, | ||
189 | 29 | TarFailed, | 30 | TarFailed, |
190 | 30 | ) | 31 | ) |
191 | 31 | from bzrlib.plugins.builddeb.util import ( | 32 | from bzrlib.plugins.builddeb.util import ( |
192 | @@ -121,11 +122,14 @@ | |||
193 | 121 | if parent_dir != '' and not os.path.exists(parent_dir): | 122 | if parent_dir != '' and not os.path.exists(parent_dir): |
194 | 122 | os.makedirs(parent_dir) | 123 | os.makedirs(parent_dir) |
195 | 123 | if not self.use_existing: | 124 | if not self.use_existing: |
197 | 124 | tarball = self.upstream_provider.provide(parent_dir) | 125 | tarballs = self.upstream_provider.provide(parent_dir) |
198 | 126 | if len(tarballs) > 1: | ||
199 | 127 | raise MultipleUpstreamTarballsNotSupported() | ||
200 | 128 | tarball = tarballs[0] | ||
201 | 125 | # Extract it to the right place | 129 | # Extract it to the right place |
202 | 126 | tempdir = tempfile.mkdtemp(prefix='builddeb-merge-') | 130 | tempdir = tempfile.mkdtemp(prefix='builddeb-merge-') |
203 | 127 | try: | 131 | try: |
205 | 128 | ret = subprocess.call(['tar','-C',tempdir,'-xf',tarball], | 132 | ret = subprocess.call(['tar', '-C', tempdir, '-xf', tarball], |
206 | 129 | preexec_fn=subprocess_setup) | 133 | preexec_fn=subprocess_setup) |
207 | 130 | if ret != 0: | 134 | if ret != 0: |
208 | 131 | raise TarFailed("uncompress", tarball) | 135 | raise TarFailed("uncompress", tarball) |
209 | 132 | 136 | ||
210 | === modified file 'tests/test_upstream.py' | |||
211 | --- tests/test_upstream.py 2011-06-14 00:32:04 +0000 | |||
212 | +++ tests/test_upstream.py 2011-06-14 14:11:23 +0000 | |||
213 | @@ -186,6 +186,21 @@ | |||
214 | 186 | self.assertEqual("apackage", sources.lookup_package) | 186 | self.assertEqual("apackage", sources.lookup_package) |
215 | 187 | self.assertEqual(0, caller.called) | 187 | self.assertEqual(0, caller.called) |
216 | 188 | 188 | ||
217 | 189 | def test_apt_provider_multiple_tarballs(self): | ||
218 | 190 | caller = MockAptCaller(work=True) | ||
219 | 191 | sources = MockSources(["0.1-1", "0.2-1"], | ||
220 | 192 | [[("checksum", 0L, "apackage_0.1.orig.tar.gz", "tar")], | ||
221 | 193 | [("checksum", 0L, "apackage_0.2.orig.tar.bz2", "tar"), | ||
222 | 194 | ("checksum", 1L, "apackage_0.2.orig-extra.tar.gz", "tar")]]) | ||
223 | 195 | apt_pkg = MockAptPkg(sources) | ||
224 | 196 | src = AptSource() | ||
225 | 197 | src._run_apt_source = caller.call | ||
226 | 198 | paths = src.fetch_tarball("apackage", "0.2", "target", | ||
227 | 199 | _apt_pkg=apt_pkg) | ||
228 | 200 | self.assertEquals(paths, [ | ||
229 | 201 | "target/apackage_0.2.orig.tar.bz2", | ||
230 | 202 | "target/apackage_0.2.orig-extra.tar.gz"]) | ||
231 | 203 | |||
232 | 189 | def test_apt_provider_right_version_bz2(self): | 204 | def test_apt_provider_right_version_bz2(self): |
233 | 190 | caller = MockAptCaller(work=True) | 205 | caller = MockAptCaller(work=True) |
234 | 191 | sources = MockSources(["0.1-1", "0.2-1"], | 206 | sources = MockSources(["0.1-1", "0.2-1"], |
235 | @@ -194,10 +209,9 @@ | |||
236 | 194 | apt_pkg = MockAptPkg(sources) | 209 | apt_pkg = MockAptPkg(sources) |
237 | 195 | src = AptSource() | 210 | src = AptSource() |
238 | 196 | src._run_apt_source = caller.call | 211 | src._run_apt_source = caller.call |
240 | 197 | path = src.fetch_tarball("apackage", "0.2", "target", | 212 | paths = src.fetch_tarball("apackage", "0.2", "target", |
241 | 198 | _apt_pkg=apt_pkg) | 213 | _apt_pkg=apt_pkg) |
244 | 199 | self.assertEquals(path, | 214 | self.assertEquals(paths, ["target/apackage_0.2.orig.tar.bz2"]) |
243 | 200 | "target/apackage_0.2.orig.tar.bz2") | ||
245 | 201 | 215 | ||
246 | 202 | def test_apt_provider_right_version(self): | 216 | def test_apt_provider_right_version(self): |
247 | 203 | caller = MockAptCaller(work=True) | 217 | caller = MockAptCaller(work=True) |
248 | @@ -207,10 +221,9 @@ | |||
249 | 207 | apt_pkg = MockAptPkg(sources) | 221 | apt_pkg = MockAptPkg(sources) |
250 | 208 | src = AptSource() | 222 | src = AptSource() |
251 | 209 | src._run_apt_source = caller.call | 223 | src._run_apt_source = caller.call |
253 | 210 | path = src.fetch_tarball("apackage", "0.2", "target", | 224 | paths = src.fetch_tarball("apackage", "0.2", "target", |
254 | 211 | _apt_pkg=apt_pkg) | 225 | _apt_pkg=apt_pkg) |
257 | 212 | self.assertEquals(path, | 226 | self.assertEquals(paths, ["target/apackage_0.2.orig.tar.gz"]) |
256 | 213 | "target/apackage_0.2.orig.tar.gz") | ||
258 | 214 | self.assertEqual(1, apt_pkg.init_called_times) | 227 | self.assertEqual(1, apt_pkg.init_called_times) |
259 | 215 | self.assertEqual(1, apt_pkg.get_pkg_source_records_called_times) | 228 | self.assertEqual(1, apt_pkg.get_pkg_source_records_called_times) |
260 | 216 | self.assertEqual(1, sources.restart_called_times) | 229 | self.assertEqual(1, sources.restart_called_times) |
261 | @@ -276,7 +289,7 @@ | |||
262 | 276 | self._specific_versions.append((package, version, target_dir)) | 289 | self._specific_versions.append((package, version, target_dir)) |
263 | 277 | if not self._succeed: | 290 | if not self._succeed: |
264 | 278 | raise PackageVersionNotPresent(package, version, self) | 291 | raise PackageVersionNotPresent(package, version, self) |
266 | 279 | return self._tarball_path(package, version, target_dir) | 292 | return [self._tarball_path(package, version, target_dir)] |
267 | 280 | 293 | ||
268 | 281 | def __repr__(self): | 294 | def __repr__(self): |
269 | 282 | return "%s()" % self.__class__.__name__ | 295 | return "%s()" % self.__class__.__name__ |
270 | @@ -377,7 +390,7 @@ | |||
271 | 377 | source = UpstreamBranchSource(self.tree.branch, | 390 | source = UpstreamBranchSource(self.tree.branch, |
272 | 378 | {"1.0": self.tree.branch.last_revision()}) | 391 | {"1.0": self.tree.branch.last_revision()}) |
273 | 379 | os.mkdir("mydir") | 392 | os.mkdir("mydir") |
275 | 380 | self.assertEquals("mydir/foo_1.0.orig.tar.gz", | 393 | self.assertEquals(["mydir/foo_1.0.orig.tar.gz"], |
276 | 381 | source.fetch_tarball("foo", "1.0", "mydir")) | 394 | source.fetch_tarball("foo", "1.0", "mydir")) |
277 | 382 | self.assertPathExists("mydir/foo_1.0.orig.tar.gz") | 395 | self.assertPathExists("mydir/foo_1.0.orig.tar.gz") |
278 | 383 | 396 | ||
279 | @@ -443,7 +456,7 @@ | |||
280 | 443 | {"1.0": self.tree.branch.last_revision()}) | 456 | {"1.0": self.tree.branch.last_revision()}) |
281 | 444 | self.assertIs(None, source._upstream_branch) | 457 | self.assertIs(None, source._upstream_branch) |
282 | 445 | os.mkdir("mydir") | 458 | os.mkdir("mydir") |
284 | 446 | self.assertEquals("mydir/foo_1.0.orig.tar.gz", | 459 | self.assertEquals(["mydir/foo_1.0.orig.tar.gz"], |
285 | 447 | source.fetch_tarball("foo", "1.0", "mydir")) | 460 | source.fetch_tarball("foo", "1.0", "mydir")) |
286 | 448 | self.assertPathExists("mydir/foo_1.0.orig.tar.gz") | 461 | self.assertPathExists("mydir/foo_1.0.orig.tar.gz") |
287 | 449 | self.assertIsNot(None, source._upstream_branch) | 462 | self.assertIsNot(None, source._upstream_branch) |
288 | @@ -721,7 +734,7 @@ | |||
289 | 721 | def test_fetch_tarball(self): | 734 | def test_fetch_tarball(self): |
290 | 722 | source = TarfileSource("foo-1.0.tar.gz", "1.0") | 735 | source = TarfileSource("foo-1.0.tar.gz", "1.0") |
291 | 723 | os.mkdir("bar") | 736 | os.mkdir("bar") |
293 | 724 | self.assertEquals("bar/foo_1.0.orig.tar.gz", | 737 | self.assertEquals(["bar/foo_1.0.orig.tar.gz"], |
294 | 725 | source.fetch_tarball("foo", "1.0", "bar")) | 738 | source.fetch_tarball("foo", "1.0", "bar")) |
295 | 726 | self.assertPathExists("bar/foo_1.0.orig.tar.gz") | 739 | self.assertPathExists("bar/foo_1.0.orig.tar.gz") |
296 | 727 | 740 | ||
297 | @@ -731,7 +744,7 @@ | |||
298 | 731 | zf.close() | 744 | zf.close() |
299 | 732 | source = TarfileSource("bla-2.0.zip", "2.0") | 745 | source = TarfileSource("bla-2.0.zip", "2.0") |
300 | 733 | os.mkdir("bar") | 746 | os.mkdir("bar") |
302 | 734 | self.assertEquals("bar/foo_2.0.orig.tar.gz", | 747 | self.assertEquals(["bar/foo_2.0.orig.tar.gz"], |
303 | 735 | source.fetch_tarball("foo", "2.0", "bar")) | 748 | source.fetch_tarball("foo", "2.0", "bar")) |
304 | 736 | self.assertPathExists("bar/foo_2.0.orig.tar.gz") | 749 | self.assertPathExists("bar/foo_2.0.orig.tar.gz") |
305 | 737 | 750 | ||
306 | @@ -748,7 +761,7 @@ | |||
307 | 748 | bz2.BZ2File("foo-1.0.tar.bz2").close() | 761 | bz2.BZ2File("foo-1.0.tar.bz2").close() |
308 | 749 | source = TarfileSource("foo-1.0.tar.bz2", "1.0") | 762 | source = TarfileSource("foo-1.0.tar.bz2", "1.0") |
309 | 750 | os.mkdir("bar") | 763 | os.mkdir("bar") |
311 | 751 | self.assertEquals("bar/foo_1.0.orig.tar.gz", | 764 | self.assertEquals(["bar/foo_1.0.orig.tar.gz"], |
312 | 752 | source.fetch_tarball("foo", "1.0", "bar")) | 765 | source.fetch_tarball("foo", "1.0", "bar")) |
313 | 753 | self.assertPathExists("bar/foo_1.0.orig.tar.gz") | 766 | self.assertPathExists("bar/foo_1.0.orig.tar.gz") |
314 | 754 | gzip.open("bar/foo_1.0.orig.tar.gz").close() | 767 | gzip.open("bar/foo_1.0.orig.tar.gz").close() |
315 | @@ -761,7 +774,7 @@ | |||
316 | 761 | pass | 774 | pass |
317 | 762 | 775 | ||
318 | 763 | def provide(self, target_dir): | 776 | def provide(self, target_dir): |
320 | 764 | raise MissingUpstreamTarball("test_tarball") | 777 | raise MissingUpstreamTarball("test_tarball", "1.0") |
321 | 765 | 778 | ||
322 | 766 | 779 | ||
323 | 767 | class _TouchUpstreamProvider(UpstreamProvider): | 780 | class _TouchUpstreamProvider(UpstreamProvider): |
324 | @@ -771,9 +784,11 @@ | |||
325 | 771 | self.desired_tarball_name = desired_tarball_name | 784 | self.desired_tarball_name = desired_tarball_name |
326 | 772 | 785 | ||
327 | 773 | def provide(self, target_dir): | 786 | def provide(self, target_dir): |
329 | 774 | f = open(os.path.join(target_dir, self.desired_tarball_name), "wb") | 787 | path = os.path.join(target_dir, self.desired_tarball_name) |
330 | 788 | f = open(path, "wb") | ||
331 | 775 | f.write("I am a tarball, honest\n") | 789 | f.write("I am a tarball, honest\n") |
332 | 776 | f.close() | 790 | f.close() |
333 | 791 | return [path] | ||
334 | 777 | 792 | ||
335 | 778 | 793 | ||
336 | 779 | class _SimpleUpstreamProvider(UpstreamProvider): | 794 | class _SimpleUpstreamProvider(UpstreamProvider): |
337 | @@ -785,11 +800,11 @@ | |||
338 | 785 | self.store_dir = store_dir | 800 | self.store_dir = store_dir |
339 | 786 | 801 | ||
340 | 787 | def provide(self, target_dir): | 802 | def provide(self, target_dir): |
342 | 788 | path = (self.already_exists_in_target(target_dir) | 803 | paths = (self.already_exists_in_target(target_dir) |
343 | 789 | or self.provide_from_store_dir(target_dir)) | 804 | or self.provide_from_store_dir(target_dir)) |
347 | 790 | if path is not None: | 805 | if paths is not None: |
348 | 791 | return path | 806 | return paths |
349 | 792 | raise MissingUpstreamTarball(self._tarball_names()[0]) | 807 | raise MissingUpstreamTarball(self.package, self.version) |
350 | 793 | 808 | ||
351 | 794 | 809 | ||
352 | 795 | class ExtractTarballVersionTests(TestCase): | 810 | class ExtractTarballVersionTests(TestCase): |
353 | 796 | 811 | ||
354 | === modified file 'upstream/__init__.py' | |||
355 | --- upstream/__init__.py 2011-06-13 22:10:22 +0000 | |||
356 | +++ upstream/__init__.py 2011-06-14 14:11:23 +0000 | |||
357 | @@ -85,7 +85,7 @@ | |||
358 | 85 | :param package: Name of the package | 85 | :param package: Name of the package |
359 | 86 | :param version: Version string of the version to fetch | 86 | :param version: Version string of the version to fetch |
360 | 87 | :param target_dir: Directory in which to store the tarball | 87 | :param target_dir: Directory in which to store the tarball |
362 | 88 | :return: Path of the fetched tarball | 88 | :return: Paths of the fetched tarballs |
363 | 89 | """ | 89 | """ |
364 | 90 | raise NotImplementedError(self.fetch_tarball) | 90 | raise NotImplementedError(self.fetch_tarball) |
365 | 91 | 91 | ||
366 | @@ -126,14 +126,17 @@ | |||
367 | 126 | lookup = get_fn(sources, 'lookup', 'Lookup') | 126 | lookup = get_fn(sources, 'lookup', 'Lookup') |
368 | 127 | while lookup(package): | 127 | while lookup(package): |
369 | 128 | version = get_fn(sources, 'version', 'Version') | 128 | version = get_fn(sources, 'version', 'Version') |
370 | 129 | filenames = [] | ||
371 | 129 | for (checksum, size, filename, filekind) in sources.files: | 130 | for (checksum, size, filename, filekind) in sources.files: |
372 | 130 | if filekind != "tar": | 131 | if filekind != "tar": |
373 | 131 | continue | 132 | continue |
374 | 132 | filename = os.path.basename(filename) | 133 | filename = os.path.basename(filename) |
375 | 133 | if filename.startswith("%s_%s.orig" % (package, upstream_version)): | 134 | if filename.startswith("%s_%s.orig" % (package, upstream_version)): |
379 | 134 | if self._run_apt_source(package, version, target_dir): | 135 | filenames.append(filename) |
380 | 135 | return os.path.join(target_dir, filename) | 136 | if filenames: |
381 | 136 | break | 137 | if self._run_apt_source(package, version, target_dir): |
382 | 138 | return [os.path.join(target_dir, filename) | ||
383 | 139 | for filename in filenames] | ||
384 | 137 | note("apt could not find the needed tarball.") | 140 | note("apt could not find the needed tarball.") |
385 | 138 | raise PackageVersionNotPresent(package, upstream_version, self) | 141 | raise PackageVersionNotPresent(package, upstream_version, self) |
386 | 139 | 142 | ||
387 | @@ -157,8 +160,7 @@ | |||
388 | 157 | self.tree = tree | 160 | self.tree = tree |
389 | 158 | self.larstiq = larstiq | 161 | self.larstiq = larstiq |
390 | 159 | 162 | ||
393 | 160 | def _get_orig_source(self, source_dir, desired_tarball_names, | 163 | def _get_orig_source(self, source_dir, prefix, target_dir): |
392 | 161 | target_dir): | ||
394 | 162 | note("Trying to use get-orig-source to retrieve needed tarball.") | 164 | note("Trying to use get-orig-source to retrieve needed tarball.") |
395 | 163 | command = ["make", "-f", "debian/rules", "get-orig-source"] | 165 | command = ["make", "-f", "debian/rules", "get-orig-source"] |
396 | 164 | proc = subprocess.Popen(command, cwd=source_dir) | 166 | proc = subprocess.Popen(command, cwd=source_dir) |
397 | @@ -166,13 +168,18 @@ | |||
398 | 166 | if ret != 0: | 168 | if ret != 0: |
399 | 167 | note("Trying to run get-orig-source rule failed") | 169 | note("Trying to run get-orig-source rule failed") |
400 | 168 | return None | 170 | return None |
403 | 169 | for desired_tarball_name in desired_tarball_names: | 171 | filenames = [] |
404 | 170 | fetched_tarball = os.path.join(source_dir, desired_tarball_name) | 172 | for filename in os.listdir(source_dir): |
405 | 173 | if not filename.startswith(prefix): | ||
406 | 174 | continue | ||
407 | 175 | fetched_tarball = os.path.join(source_dir, filename) | ||
408 | 171 | if os.path.exists(fetched_tarball): | 176 | if os.path.exists(fetched_tarball): |
410 | 172 | repack_tarball(fetched_tarball, desired_tarball_name, | 177 | repack_tarball(fetched_tarball, filename, |
411 | 173 | target_dir=target_dir, force_gz=False) | 178 | target_dir=target_dir, force_gz=False) |
414 | 174 | return fetched_tarball | 179 | filenames.append(os.path.join(target_dir, filename)) |
415 | 175 | note("get-orig-source did not create %s", desired_tarball_name) | 180 | if filenames: |
416 | 181 | return filenames | ||
417 | 182 | note("get-orig-source did not create file with prefix %s", prefix) | ||
418 | 176 | return None | 183 | return None |
419 | 177 | 184 | ||
420 | 178 | def fetch_tarball(self, package, version, target_dir): | 185 | def fetch_tarball(self, package, version, target_dir): |
421 | @@ -182,9 +189,6 @@ | |||
422 | 182 | rules_name = 'debian/rules' | 189 | rules_name = 'debian/rules' |
423 | 183 | rules_id = self.tree.path2id(rules_name) | 190 | rules_id = self.tree.path2id(rules_name) |
424 | 184 | if rules_id is not None: | 191 | if rules_id is not None: |
425 | 185 | desired_tarball_names = [tarball_name(package, version), | ||
426 | 186 | tarball_name(package, version, 'bz2'), | ||
427 | 187 | tarball_name(package, version, 'lzma')] | ||
428 | 188 | tmpdir = tempfile.mkdtemp(prefix="builddeb-get-orig-source-") | 192 | tmpdir = tempfile.mkdtemp(prefix="builddeb-get-orig-source-") |
429 | 189 | try: | 193 | try: |
430 | 190 | base_export_dir = os.path.join(tmpdir, "export") | 194 | base_export_dir = os.path.join(tmpdir, "export") |
431 | @@ -193,11 +197,11 @@ | |||
432 | 193 | os.mkdir(export_dir) | 197 | os.mkdir(export_dir) |
433 | 194 | export_dir = os.path.join(export_dir, "debian") | 198 | export_dir = os.path.join(export_dir, "debian") |
434 | 195 | export(self.tree, export_dir, format="dir") | 199 | export(self.tree, export_dir, format="dir") |
438 | 196 | tarball_path = self._get_orig_source(base_export_dir, | 200 | tarball_paths = self._get_orig_source(base_export_dir, |
439 | 197 | desired_tarball_names, target_dir) | 201 | "%s_%s.orig" % (package, version), target_dir) |
440 | 198 | if tarball_path is None: | 202 | if tarball_paths is None: |
441 | 199 | raise PackageVersionNotPresent(package, version, self) | 203 | raise PackageVersionNotPresent(package, version, self) |
443 | 200 | return tarball_path | 204 | return tarball_paths |
444 | 201 | finally: | 205 | finally: |
445 | 202 | shutil.rmtree(tmpdir) | 206 | shutil.rmtree(tmpdir) |
446 | 203 | note("No debian/rules file to try and use for a get-orig-source rule") | 207 | note("No debian/rules file to try and use for a get-orig-source rule") |
447 | @@ -279,7 +283,7 @@ | |||
448 | 279 | if r != 0: | 283 | if r != 0: |
449 | 280 | note("uscan could not find the needed tarball.") | 284 | note("uscan could not find the needed tarball.") |
450 | 281 | raise PackageVersionNotPresent(package, version, self) | 285 | raise PackageVersionNotPresent(package, version, self) |
452 | 282 | return self._tarball_path(package, version, target_dir) | 286 | return [self._tarball_path(package, version, target_dir)] |
453 | 283 | 287 | ||
454 | 284 | 288 | ||
455 | 285 | class SelfSplitSource(UpstreamSource): | 289 | class SelfSplitSource(UpstreamSource): |
456 | @@ -307,13 +311,13 @@ | |||
457 | 307 | "to create the tarball") | 311 | "to create the tarball") |
458 | 308 | tarball_path = self._tarball_path(package, version, target_dir) | 312 | tarball_path = self._tarball_path(package, version, target_dir) |
459 | 309 | self._split(package, version, tarball_path) | 313 | self._split(package, version, tarball_path) |
461 | 310 | return tarball_path | 314 | return [tarball_path] |
462 | 311 | 315 | ||
463 | 312 | 316 | ||
464 | 313 | class StackedUpstreamSource(UpstreamSource): | 317 | class StackedUpstreamSource(UpstreamSource): |
465 | 314 | """An upstream source that checks a list of other upstream sources. | 318 | """An upstream source that checks a list of other upstream sources. |
466 | 315 | 319 | ||
468 | 316 | The first source that can provide a tarball, wins. | 320 | The first source that can provide a tarball, wins. |
469 | 317 | """ | 321 | """ |
470 | 318 | 322 | ||
471 | 319 | def __init__(self, sources): | 323 | def __init__(self, sources): |
472 | @@ -325,12 +329,11 @@ | |||
473 | 325 | def fetch_tarball(self, package, version, target_dir): | 329 | def fetch_tarball(self, package, version, target_dir): |
474 | 326 | for source in self._sources: | 330 | for source in self._sources: |
475 | 327 | try: | 331 | try: |
477 | 328 | path = source.fetch_tarball(package, version, target_dir) | 332 | paths = source.fetch_tarball(package, version, target_dir) |
478 | 329 | except PackageVersionNotPresent: | 333 | except PackageVersionNotPresent: |
479 | 330 | pass | 334 | pass |
480 | 331 | else: | 335 | else: |
483 | 332 | assert isinstance(path, basestring) | 336 | return paths |
482 | 333 | return path | ||
484 | 334 | raise PackageVersionNotPresent(package, version, self) | 337 | raise PackageVersionNotPresent(package, version, self) |
485 | 335 | 338 | ||
486 | 336 | def get_latest_version(self, package, version): | 339 | def get_latest_version(self, package, version): |
487 | @@ -364,7 +367,7 @@ | |||
488 | 364 | self.source = StackedUpstreamSource(sources) | 367 | self.source = StackedUpstreamSource(sources) |
489 | 365 | 368 | ||
490 | 366 | def provide(self, target_dir): | 369 | def provide(self, target_dir): |
492 | 367 | """Provide the upstream tarball any way possible. | 370 | """Provide the upstream tarball(s) any way possible. |
493 | 368 | 371 | ||
494 | 369 | Call this to place the correctly named tarball in to target_dir, | 372 | Call this to place the correctly named tarball in to target_dir, |
495 | 370 | through means possible. | 373 | through means possible. |
496 | @@ -396,44 +399,45 @@ | |||
497 | 396 | if not os.path.exists(self.store_dir): | 399 | if not os.path.exists(self.store_dir): |
498 | 397 | os.makedirs(self.store_dir) | 400 | os.makedirs(self.store_dir) |
499 | 398 | try: | 401 | try: |
501 | 399 | path = self.source.fetch_tarball(self.package, | 402 | paths = self.source.fetch_tarball(self.package, |
502 | 400 | self.version, self.store_dir) | 403 | self.version, self.store_dir) |
503 | 401 | except PackageVersionNotPresent: | 404 | except PackageVersionNotPresent: |
506 | 402 | raise MissingUpstreamTarball(self._tarball_names()[0]) | 405 | raise MissingUpstreamTarball(self.package, self.version) |
507 | 403 | assert isinstance(path, basestring) | 406 | assert isinstance(paths, list) |
508 | 404 | else: | 407 | else: |
509 | 405 | note("Using the upstream tarball that is present in %s" % | 408 | note("Using the upstream tarball that is present in %s" % |
510 | 406 | self.store_dir) | 409 | self.store_dir) |
514 | 407 | path = self.provide_from_store_dir(target_dir) | 410 | paths = self.provide_from_store_dir(target_dir) |
515 | 408 | assert path is not None | 411 | assert paths is not None |
516 | 409 | return path | 412 | return paths |
517 | 413 | |||
518 | 414 | def _gather_orig_files(self, path): | ||
519 | 415 | prefix = "%s_%s.orig" % (self.package, self.version) | ||
520 | 416 | ret = [] | ||
521 | 417 | path = os.path.abspath(path) | ||
522 | 418 | if not os.path.isdir(path): | ||
523 | 419 | return None | ||
524 | 420 | for filename in os.listdir(path): | ||
525 | 421 | if filename.startswith(prefix): | ||
526 | 422 | ret.append(os.path.join(path, filename)) | ||
527 | 423 | if ret: | ||
528 | 424 | return ret | ||
529 | 425 | return None | ||
530 | 410 | 426 | ||
531 | 411 | def already_exists_in_target(self, target_dir): | 427 | def already_exists_in_target(self, target_dir): |
537 | 412 | for tarball_name in self._tarball_names(): | 428 | return self._gather_orig_files(target_dir) |
533 | 413 | path = os.path.join(target_dir, tarball_name) | ||
534 | 414 | if os.path.exists(path): | ||
535 | 415 | return path | ||
536 | 416 | return None | ||
538 | 417 | 429 | ||
539 | 418 | def already_exists_in_store(self): | 430 | def already_exists_in_store(self): |
545 | 419 | for tarball_name in self._tarball_names(): | 431 | return self._gather_orig_files(self.store_dir) |
541 | 420 | path = os.path.join(self.store_dir, tarball_name) | ||
542 | 421 | if os.path.exists(path): | ||
543 | 422 | return path | ||
544 | 423 | return None | ||
546 | 424 | 432 | ||
547 | 425 | def provide_from_store_dir(self, target_dir): | 433 | def provide_from_store_dir(self, target_dir): |
550 | 426 | path = self.already_exists_in_store() | 434 | paths = self.already_exists_in_store() |
551 | 427 | if path is not None: | 435 | if paths is None: |
552 | 436 | return None | ||
553 | 437 | for path in paths: | ||
554 | 428 | repack_tarball(path, os.path.basename(path), | 438 | repack_tarball(path, os.path.basename(path), |
555 | 429 | target_dir=target_dir, force_gz=False) | 439 | target_dir=target_dir, force_gz=False) |
563 | 430 | return path | 440 | return paths |
557 | 431 | return path | ||
558 | 432 | |||
559 | 433 | def _tarball_names(self): | ||
560 | 434 | return [tarball_name(self.package, self.version), | ||
561 | 435 | tarball_name(self.package, self.version, format='bz2'), | ||
562 | 436 | tarball_name(self.package, self.version, format='lzma')] | ||
564 | 437 | 441 | ||
565 | 438 | 442 | ||
566 | 439 | def extract_tarball_version(path, packagename): | 443 | def extract_tarball_version(path, packagename): |
567 | @@ -473,7 +477,7 @@ | |||
568 | 473 | raise PackageVersionNotPresent(package, version, self) | 477 | raise PackageVersionNotPresent(package, version, self) |
569 | 474 | dest_name = tarball_name(package, version) | 478 | dest_name = tarball_name(package, version) |
570 | 475 | repack_tarball(self.path, dest_name, target_dir=target_dir, force_gz=True) | 479 | repack_tarball(self.path, dest_name, target_dir=target_dir, force_gz=True) |
572 | 476 | return os.path.join(target_dir, dest_name) | 480 | return [os.path.join(target_dir, dest_name)] |
573 | 477 | 481 | ||
574 | 478 | def get_latest_version(self, package, version): | 482 | def get_latest_version(self, package, version): |
575 | 479 | if self.version is not None: | 483 | if self.version is not None: |
576 | 480 | 484 | ||
577 | === modified file 'upstream/branch.py' | |||
578 | --- upstream/branch.py 2011-04-28 15:46:43 +0000 | |||
579 | +++ upstream/branch.py 2011-06-14 14:11:23 +0000 | |||
580 | @@ -268,7 +268,7 @@ | |||
581 | 268 | export(rev_tree, target_filename, 'tgz', tarball_base) | 268 | export(rev_tree, target_filename, 'tgz', tarball_base) |
582 | 269 | finally: | 269 | finally: |
583 | 270 | self.upstream_branch.unlock() | 270 | self.upstream_branch.unlock() |
585 | 271 | return target_filename | 271 | return [target_filename] |
586 | 272 | 272 | ||
587 | 273 | def __repr__(self): | 273 | def __repr__(self): |
588 | 274 | return "<%s for %r>" % (self.__class__.__name__, | 274 | return "<%s for %r>" % (self.__class__.__name__, |
589 | 275 | 275 | ||
590 | === modified file 'upstream/pristinetar.py' | |||
591 | --- upstream/pristinetar.py 2011-06-03 12:34:57 +0000 | |||
592 | +++ upstream/pristinetar.py 2011-06-14 14:11:23 +0000 | |||
593 | @@ -145,7 +145,7 @@ | |||
594 | 145 | raise PackageVersionNotPresent(package, version, self) | 145 | raise PackageVersionNotPresent(package, version, self) |
595 | 146 | except PerFileTimestampsNotSupported: | 146 | except PerFileTimestampsNotSupported: |
596 | 147 | raise PackageVersionNotPresent(package, version, self) | 147 | raise PackageVersionNotPresent(package, version, self) |
598 | 148 | return target_filename | 148 | return [target_filename] |
599 | 149 | 149 | ||
600 | 150 | def _has_version(self, tag_name, md5=None): | 150 | def _has_version(self, tag_name, md5=None): |
601 | 151 | if not self.branch.tags.has_tag(tag_name): | 151 | if not self.branch.tags.has_tag(tag_name): |