Merge ~cjwatson/launchpad:sourcedeps-codetree into launchpad:master
- Git
- lp:~cjwatson/launchpad
- sourcedeps-codetree
- Merge into master
Proposed by
Colin Watson
Status: | Rejected |
---|---|
Rejected by: | Colin Watson |
Proposed branch: | ~cjwatson/launchpad:sourcedeps-codetree |
Merge into: | launchpad:master |
Diff against target: |
816 lines (+159/-477) 3 files modified
dev/null (+0/-3) lib/devscripts/sourcecode.py (+49/-307) lib/devscripts/tests/test_sourcecode.py (+110/-167) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Launchpad code reviewers | Pending | ||
Review via email: mp+373737@code.launchpad.net |
Commit message
Replace most of devscripts.
Description of the change
This drops some bespoke code and potentially lets us migrate individual sourcedeps to git.
We'll need to install python-codetree in various places first (this is too early for us to be able to rely on a Python dependency rather than a system dependency).
This is essentially the same as https:/
To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) wrote : | # |
Unmerged commits
- 8c94b91... by Colin Watson
-
Replace most of devscripts.
sourcecode with codetree. - dc26ef8... by Colin Watson
-
Drop support for private/optional sourcedeps, unused since r12871.
- c6b5d56... by Colin Watson
-
Remove utilities/
sourcedeps. filter, unused since r8510.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/lib/devscripts/sourcecode.py b/lib/devscripts/sourcecode.py | |||
2 | index 5cb6686..22bc84e 100644 | |||
3 | --- a/lib/devscripts/sourcecode.py | |||
4 | +++ b/lib/devscripts/sourcecode.py | |||
5 | @@ -1,136 +1,37 @@ | |||
7 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
8 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
9 | 3 | 3 | ||
10 | 4 | """Tools for maintaining the Launchpad source code.""" | 4 | """Tools for maintaining the Launchpad source code.""" |
11 | 5 | 5 | ||
12 | 6 | from __future__ import absolute_import, print_function, unicode_literals | ||
13 | 7 | |||
14 | 6 | __metaclass__ = type | 8 | __metaclass__ = type |
15 | 7 | __all__ = [ | 9 | __all__ = [ |
19 | 8 | 'interpret_config', | 10 | 'main', |
17 | 9 | 'parse_config_file', | ||
18 | 10 | 'plan_update', | ||
20 | 11 | ] | 11 | ] |
21 | 12 | 12 | ||
24 | 13 | import errno | 13 | import logging |
23 | 14 | import json | ||
25 | 15 | import optparse | 14 | import optparse |
26 | 16 | import os | 15 | import os |
27 | 17 | import shutil | 16 | import shutil |
28 | 18 | import sys | ||
29 | 19 | 17 | ||
45 | 20 | from bzrlib import ui | 18 | from codetree.config import Config |
46 | 21 | from bzrlib.branch import Branch | 19 | from codetree.handlers.bzr import BzrSourceHandler |
32 | 22 | from bzrlib.errors import ( | ||
33 | 23 | BzrError, | ||
34 | 24 | IncompatibleRepositories, | ||
35 | 25 | NotBranchError, | ||
36 | 26 | ) | ||
37 | 27 | from bzrlib.plugin import load_plugins | ||
38 | 28 | from bzrlib.revisionspec import RevisionSpec | ||
39 | 29 | from bzrlib.trace import ( | ||
40 | 30 | enable_default_logging, | ||
41 | 31 | report_exception, | ||
42 | 32 | ) | ||
43 | 33 | from bzrlib.upgrade import upgrade | ||
44 | 34 | from bzrlib.workingtree import WorkingTree | ||
47 | 35 | 20 | ||
48 | 36 | from devscripts import get_launchpad_root | 21 | from devscripts import get_launchpad_root |
49 | 37 | 22 | ||
50 | 38 | 23 | ||
124 | 39 | def parse_config_file(file_handle): | 24 | def plan_update(existing_branches, config): |
52 | 40 | """Parse the source code config file 'file_handle'. | ||
53 | 41 | |||
54 | 42 | :param file_handle: A file-like object containing sourcecode | ||
55 | 43 | configuration. | ||
56 | 44 | :return: A sequence of lines of either '[key, value]' or | ||
57 | 45 | '[key, value, optional]'. | ||
58 | 46 | """ | ||
59 | 47 | for line in file_handle: | ||
60 | 48 | if line == '\n' or line.startswith('#'): | ||
61 | 49 | continue | ||
62 | 50 | yield line.split() | ||
63 | 51 | |||
64 | 52 | |||
65 | 53 | def interpret_config_entry(entry, use_http=False): | ||
66 | 54 | """Interpret a single parsed line from the config file.""" | ||
67 | 55 | branch_name = entry[0] | ||
68 | 56 | components = entry[1].split(';revno=') | ||
69 | 57 | branch_url = components[0] | ||
70 | 58 | if use_http: | ||
71 | 59 | branch_url = branch_url.replace('lp:', 'http://bazaar.launchpad.net/') | ||
72 | 60 | if len(components) == 1: | ||
73 | 61 | revision = None | ||
74 | 62 | else: | ||
75 | 63 | assert len(components) == 2, 'Bad branch URL: ' + entry[1] | ||
76 | 64 | revision = components[1] or None | ||
77 | 65 | if len(entry) > 2: | ||
78 | 66 | assert len(entry) == 3 and entry[2].lower() == 'optional', ( | ||
79 | 67 | 'Bad configuration line: should be space delimited values of ' | ||
80 | 68 | 'sourcecode directory name, branch URL [, "optional"]\n' + | ||
81 | 69 | ' '.join(entry)) | ||
82 | 70 | optional = True | ||
83 | 71 | else: | ||
84 | 72 | optional = False | ||
85 | 73 | return branch_name, branch_url, revision, optional | ||
86 | 74 | |||
87 | 75 | |||
88 | 76 | def load_cache(cache_filename): | ||
89 | 77 | try: | ||
90 | 78 | cache_file = open(cache_filename, 'rb') | ||
91 | 79 | except IOError as e: | ||
92 | 80 | if e.errno == errno.ENOENT: | ||
93 | 81 | return {} | ||
94 | 82 | else: | ||
95 | 83 | raise | ||
96 | 84 | with cache_file: | ||
97 | 85 | return json.load(cache_file) | ||
98 | 86 | |||
99 | 87 | |||
100 | 88 | def interpret_config(config_entries, public_only, use_http=False): | ||
101 | 89 | """Interpret a configuration stream, as parsed by 'parse_config_file'. | ||
102 | 90 | |||
103 | 91 | :param configuration: A sequence of parsed configuration entries. | ||
104 | 92 | :param public_only: If true, ignore private/optional branches. | ||
105 | 93 | :param use_http: If True, force all branch URLs to use http:// | ||
106 | 94 | :return: A dict mapping the names of the sourcecode dependencies to a | ||
107 | 95 | 2-tuple of their branches and whether or not they are optional. | ||
108 | 96 | """ | ||
109 | 97 | config = {} | ||
110 | 98 | for entry in config_entries: | ||
111 | 99 | branch_name, branch_url, revision, optional = interpret_config_entry( | ||
112 | 100 | entry, use_http) | ||
113 | 101 | if not optional or not public_only: | ||
114 | 102 | config[branch_name] = (branch_url, revision, optional) | ||
115 | 103 | return config | ||
116 | 104 | |||
117 | 105 | |||
118 | 106 | def _subset_dict(d, keys): | ||
119 | 107 | """Return a dict that's a subset of 'd', based on the keys in 'keys'.""" | ||
120 | 108 | return dict((key, d[key]) for key in keys) | ||
121 | 109 | |||
122 | 110 | |||
123 | 111 | def plan_update(existing_branches, configuration): | ||
125 | 112 | """Plan the update to existing branches based on 'configuration'. | 25 | """Plan the update to existing branches based on 'configuration'. |
126 | 113 | 26 | ||
127 | 114 | :param existing_branches: A sequence of branches that already exist. | 27 | :param existing_branches: A sequence of branches that already exist. |
137 | 115 | :param configuration: A dictionary of sourcecode configuration, such as is | 28 | :param config: An instance of `codetree.Config`. |
138 | 116 | returned by `interpret_config`. | 29 | :return: a set of the branches that exist locally but not in the |
139 | 117 | :return: (new_branches, update_branches, removed_branches), where | 30 | configuration. |
131 | 118 | 'new_branches' are the branches in the configuration that don't exist | ||
132 | 119 | yet, 'update_branches' are the branches in the configuration that do | ||
133 | 120 | exist, and 'removed_branches' are the branches that exist locally, but | ||
134 | 121 | not in the configuration. 'new_branches' and 'update_branches' are | ||
135 | 122 | dicts of the same form as 'configuration', 'removed_branches' is a | ||
136 | 123 | set of the same form as 'existing_branches'. | ||
140 | 124 | """ | 31 | """ |
141 | 125 | existing_branches = set(existing_branches) | 32 | existing_branches = set(existing_branches) |
150 | 126 | config_branches = set(configuration.keys()) | 33 | config_branches = set(config.directive_map) |
151 | 127 | new_branches = config_branches - existing_branches | 34 | return existing_branches - config_branches |
144 | 128 | removed_branches = existing_branches - config_branches | ||
145 | 129 | update_branches = config_branches.intersection(existing_branches) | ||
146 | 130 | return ( | ||
147 | 131 | _subset_dict(configuration, new_branches), | ||
148 | 132 | _subset_dict(configuration, update_branches), | ||
149 | 133 | removed_branches) | ||
152 | 134 | 35 | ||
153 | 135 | 36 | ||
154 | 136 | def find_branches(directory): | 37 | def find_branches(directory): |
155 | @@ -139,200 +40,50 @@ def find_branches(directory): | |||
156 | 139 | for name in os.listdir(directory): | 40 | for name in os.listdir(directory): |
157 | 140 | if name in ('.', '..'): | 41 | if name in ('.', '..'): |
158 | 141 | continue | 42 | continue |
164 | 142 | try: | 43 | for subdir in ('.bzr', '.git'): |
165 | 143 | Branch.open(os.path.join(directory, name)) | 44 | if os.path.exists(os.path.join(directory, name, subdir)): |
166 | 144 | branches.append(name) | 45 | branches.append(name) |
162 | 145 | except NotBranchError: | ||
163 | 146 | pass | ||
167 | 147 | return branches | 46 | return branches |
168 | 148 | 47 | ||
169 | 149 | 48 | ||
319 | 150 | def get_revision_id(revision, from_branch, tip=False): | 49 | def remove_branches(sourcecode_directory, removed_branches): |
171 | 151 | """Return revision id for a revision number and a branch. | ||
172 | 152 | |||
173 | 153 | If the revision is empty, the revision_id will be None. | ||
174 | 154 | |||
175 | 155 | If ``tip`` is True, the revision value will be ignored. | ||
176 | 156 | """ | ||
177 | 157 | if not tip and revision: | ||
178 | 158 | spec = RevisionSpec.from_string(revision) | ||
179 | 159 | return spec.as_revision_id(from_branch) | ||
180 | 160 | # else return None | ||
181 | 161 | |||
182 | 162 | |||
183 | 163 | def _format_revision_name(revision, tip=False): | ||
184 | 164 | """Formatting helper to return human-readable identifier for revision. | ||
185 | 165 | |||
186 | 166 | If ``tip`` is True, the revision value will be ignored. | ||
187 | 167 | """ | ||
188 | 168 | if not tip and revision: | ||
189 | 169 | return 'revision %s' % (revision,) | ||
190 | 170 | else: | ||
191 | 171 | return 'tip' | ||
192 | 172 | |||
193 | 173 | |||
194 | 174 | def get_branches(sourcecode_directory, new_branches, | ||
195 | 175 | possible_transports=None, tip=False, quiet=False): | ||
196 | 176 | """Get the new branches into sourcecode.""" | ||
197 | 177 | for project, (branch_url, revision, optional) in new_branches.iteritems(): | ||
198 | 178 | destination = os.path.join(sourcecode_directory, project) | ||
199 | 179 | try: | ||
200 | 180 | remote_branch = Branch.open( | ||
201 | 181 | branch_url, possible_transports=possible_transports) | ||
202 | 182 | except BzrError: | ||
203 | 183 | if optional: | ||
204 | 184 | report_exception(sys.exc_info(), sys.stderr) | ||
205 | 185 | continue | ||
206 | 186 | else: | ||
207 | 187 | raise | ||
208 | 188 | possible_transports.append( | ||
209 | 189 | remote_branch.bzrdir.root_transport) | ||
210 | 190 | if not quiet: | ||
211 | 191 | print 'Getting %s from %s at %s' % ( | ||
212 | 192 | project, branch_url, _format_revision_name(revision, tip)) | ||
213 | 193 | # If the 'optional' flag is set, then it's a branch that shares | ||
214 | 194 | # history with Launchpad, so we should share repositories. Otherwise, | ||
215 | 195 | # we should avoid sharing repositories to avoid format | ||
216 | 196 | # incompatibilities. | ||
217 | 197 | force_new_repo = not optional | ||
218 | 198 | revision_id = get_revision_id(revision, remote_branch, tip) | ||
219 | 199 | remote_branch.bzrdir.sprout( | ||
220 | 200 | destination, revision_id=revision_id, create_tree_if_local=True, | ||
221 | 201 | source_branch=remote_branch, force_new_repo=force_new_repo, | ||
222 | 202 | possible_transports=possible_transports) | ||
223 | 203 | |||
224 | 204 | |||
225 | 205 | def find_stale(updated, cache, sourcecode_directory, quiet): | ||
226 | 206 | """Find branches whose revision info doesn't match the cache.""" | ||
227 | 207 | new_updated = dict(updated) | ||
228 | 208 | for project, (branch_url, revision, optional) in updated.iteritems(): | ||
229 | 209 | cache_revision_info = cache.get(project) | ||
230 | 210 | if cache_revision_info is None: | ||
231 | 211 | continue | ||
232 | 212 | if cache_revision_info[0] != int(revision): | ||
233 | 213 | continue | ||
234 | 214 | destination = os.path.join(sourcecode_directory, project) | ||
235 | 215 | try: | ||
236 | 216 | branch = Branch.open(destination) | ||
237 | 217 | except BzrError: | ||
238 | 218 | continue | ||
239 | 219 | if list(branch.last_revision_info()) != cache_revision_info: | ||
240 | 220 | continue | ||
241 | 221 | if not quiet: | ||
242 | 222 | print '%s is already up to date.' % project | ||
243 | 223 | del new_updated[project] | ||
244 | 224 | return new_updated | ||
245 | 225 | |||
246 | 226 | |||
247 | 227 | def update_cache(cache, cache_filename, changed, sourcecode_directory, quiet): | ||
248 | 228 | """Update the cache with the changed branches.""" | ||
249 | 229 | old_cache = dict(cache) | ||
250 | 230 | for project, (branch_url, revision, optional) in changed.iteritems(): | ||
251 | 231 | destination = os.path.join(sourcecode_directory, project) | ||
252 | 232 | branch = Branch.open(destination) | ||
253 | 233 | cache[project] = list(branch.last_revision_info()) | ||
254 | 234 | if cache == old_cache: | ||
255 | 235 | return | ||
256 | 236 | with open(cache_filename, 'wb') as cache_file: | ||
257 | 237 | json.dump(cache, cache_file, indent=4, sort_keys=True) | ||
258 | 238 | if not quiet: | ||
259 | 239 | print 'Cache updated. Please commit "%s".' % cache_filename | ||
260 | 240 | |||
261 | 241 | |||
262 | 242 | def update_branches(sourcecode_directory, update_branches, | ||
263 | 243 | possible_transports=None, tip=False, quiet=False): | ||
264 | 244 | """Update the existing branches in sourcecode.""" | ||
265 | 245 | if possible_transports is None: | ||
266 | 246 | possible_transports = [] | ||
267 | 247 | # XXX: JonathanLange 2009-11-09: Rather than updating one branch after | ||
268 | 248 | # another, we could instead try to get them in parallel. | ||
269 | 249 | for project, (branch_url, revision, optional) in ( | ||
270 | 250 | update_branches.iteritems()): | ||
271 | 251 | # Update project from branch_url. | ||
272 | 252 | destination = os.path.join(sourcecode_directory, project) | ||
273 | 253 | if not quiet: | ||
274 | 254 | print 'Updating %s to %s' % ( | ||
275 | 255 | project, _format_revision_name(revision, tip)) | ||
276 | 256 | local_tree = WorkingTree.open(destination) | ||
277 | 257 | try: | ||
278 | 258 | remote_branch = Branch.open( | ||
279 | 259 | branch_url, possible_transports=possible_transports) | ||
280 | 260 | except BzrError: | ||
281 | 261 | if optional: | ||
282 | 262 | report_exception(sys.exc_info(), sys.stderr) | ||
283 | 263 | continue | ||
284 | 264 | else: | ||
285 | 265 | raise | ||
286 | 266 | possible_transports.append( | ||
287 | 267 | remote_branch.bzrdir.root_transport) | ||
288 | 268 | revision_id = get_revision_id(revision, remote_branch, tip) | ||
289 | 269 | try: | ||
290 | 270 | result = local_tree.pull( | ||
291 | 271 | remote_branch, stop_revision=revision_id, overwrite=True, | ||
292 | 272 | possible_transports=possible_transports) | ||
293 | 273 | except IncompatibleRepositories: | ||
294 | 274 | # XXX JRV 20100407: Ideally remote_branch.bzrdir._format | ||
295 | 275 | # should be passed into upgrade() to ensure the format is the same | ||
296 | 276 | # locally and remotely. Unfortunately smart server branches | ||
297 | 277 | # have their _format set to RemoteFormat rather than an actual | ||
298 | 278 | # format instance. | ||
299 | 279 | upgrade(destination) | ||
300 | 280 | # Upgraded, repoen working tree | ||
301 | 281 | local_tree = WorkingTree.open(destination) | ||
302 | 282 | result = local_tree.pull( | ||
303 | 283 | remote_branch, stop_revision=revision_id, overwrite=True, | ||
304 | 284 | possible_transports=possible_transports) | ||
305 | 285 | if result.old_revid == result.new_revid: | ||
306 | 286 | if not quiet: | ||
307 | 287 | print ' (No change)' | ||
308 | 288 | else: | ||
309 | 289 | if result.old_revno < result.new_revno: | ||
310 | 290 | change = 'Updated' | ||
311 | 291 | else: | ||
312 | 292 | change = 'Reverted' | ||
313 | 293 | if not quiet: | ||
314 | 294 | print ' (%s from %s to %s)' % ( | ||
315 | 295 | change, result.old_revno, result.new_revno) | ||
316 | 296 | |||
317 | 297 | |||
318 | 298 | def remove_branches(sourcecode_directory, removed_branches, quiet=False): | ||
320 | 299 | """Remove sourcecode that's no longer there.""" | 50 | """Remove sourcecode that's no longer there.""" |
321 | 300 | for project in removed_branches: | 51 | for project in removed_branches: |
322 | 301 | destination = os.path.join(sourcecode_directory, project) | 52 | destination = os.path.join(sourcecode_directory, project) |
325 | 302 | if not quiet: | 53 | logging.info('Removing %s', project) |
324 | 303 | print 'Removing %s' % project | ||
326 | 304 | try: | 54 | try: |
327 | 305 | shutil.rmtree(destination) | 55 | shutil.rmtree(destination) |
328 | 306 | except OSError: | 56 | except OSError: |
329 | 307 | os.unlink(destination) | 57 | os.unlink(destination) |
330 | 308 | 58 | ||
331 | 309 | 59 | ||
340 | 310 | def update_sourcecode(sourcecode_directory, config_filename, cache_filename, | 60 | def mangle_config(sourcecode_directory, config, tip=False, use_http=False): |
341 | 311 | public_only, tip, dry_run, quiet=False, use_http=False): | 61 | for directive in config.directive_map.values(): |
342 | 312 | """Update the sourcecode.""" | 62 | directive.location = os.path.join( |
343 | 313 | config_file = open(config_filename) | 63 | sourcecode_directory, directive.location) |
344 | 314 | config = interpret_config( | 64 | if tip: |
345 | 315 | parse_config_file(config_file), public_only, use_http) | 65 | directive.source_options.pop('revno', None) |
346 | 316 | config_file.close() | 66 | if use_http: |
347 | 317 | cache = load_cache(cache_filename) | 67 | handler = directive.source |
348 | 68 | if (isinstance(handler, BzrSourceHandler) and | ||
349 | 69 | handler.source.startswith('lp:')): | ||
350 | 70 | handler.source = handler.source.replace( | ||
351 | 71 | 'lp:', 'http://bazaar.launchpad.net/') | ||
352 | 72 | |||
353 | 73 | |||
354 | 74 | def update_sourcecode(sourcecode_directory, config_filename, | ||
355 | 75 | tip=False, dry_run=False, use_http=False): | ||
356 | 76 | config = Config([config_filename]) | ||
357 | 77 | mangle_config(sourcecode_directory, config, tip=tip, use_http=use_http) | ||
358 | 318 | branches = find_branches(sourcecode_directory) | 78 | branches = find_branches(sourcecode_directory) |
361 | 319 | new, updated, removed = plan_update(branches, config) | 79 | removed = plan_update(branches, config) |
362 | 320 | possible_transports = [] | 80 | # XXX cjwatson 2017-10-31: If we start pulling sourcedeps from git, then |
363 | 81 | # we need to remove old bzr branches first. | ||
364 | 82 | config.build(dry_run=dry_run) | ||
365 | 321 | if dry_run: | 83 | if dry_run: |
369 | 322 | print 'Branches to fetch:', new.keys() | 84 | logging.info('Branches to remove: %s', list(removed)) |
367 | 323 | print 'Branches to update:', updated.keys() | ||
368 | 324 | print 'Branches to remove:', list(removed) | ||
370 | 325 | else: | 85 | else: |
381 | 326 | get_branches( | 86 | remove_branches(sourcecode_directory, removed) |
372 | 327 | sourcecode_directory, new, possible_transports, tip, quiet) | ||
373 | 328 | updated = find_stale(updated, cache, sourcecode_directory, quiet) | ||
374 | 329 | update_branches( | ||
375 | 330 | sourcecode_directory, updated, possible_transports, tip, quiet) | ||
376 | 331 | changed = dict(updated) | ||
377 | 332 | changed.update(new) | ||
378 | 333 | update_cache( | ||
379 | 334 | cache, cache_filename, changed, sourcecode_directory, quiet) | ||
380 | 335 | remove_branches(sourcecode_directory, removed, quiet) | ||
382 | 336 | 87 | ||
383 | 337 | 88 | ||
384 | 338 | # XXX: JonathanLange 2009-09-11: By default, the script will operate on the | 89 | # XXX: JonathanLange 2009-09-11: By default, the script will operate on the |
385 | @@ -349,9 +100,6 @@ def update_sourcecode(sourcecode_directory, config_filename, cache_filename, | |||
386 | 349 | def main(args): | 100 | def main(args): |
387 | 350 | parser = optparse.OptionParser("usage: %prog [options] [root [conffile]]") | 101 | parser = optparse.OptionParser("usage: %prog [options] [root [conffile]]") |
388 | 351 | parser.add_option( | 102 | parser.add_option( |
389 | 352 | '--public-only', action='store_true', | ||
390 | 353 | help='Only fetch/update the public sourcecode branches.') | ||
391 | 354 | parser.add_option( | ||
392 | 355 | '--tip', action='store_true', | 103 | '--tip', action='store_true', |
393 | 356 | help='Ignore revision constraints for all branches and pull tip') | 104 | help='Ignore revision constraints for all branches and pull tip') |
394 | 357 | parser.add_option( | 105 | parser.add_option( |
395 | @@ -374,20 +122,14 @@ def main(args): | |||
396 | 374 | config_filename = args[2] | 122 | config_filename = args[2] |
397 | 375 | else: | 123 | else: |
398 | 376 | config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf') | 124 | config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf') |
399 | 377 | cache_filename = os.path.join( | ||
400 | 378 | root, 'utilities', 'sourcedeps.cache') | ||
401 | 379 | if len(args) > 3: | 125 | if len(args) > 3: |
402 | 380 | parser.error("Too many arguments.") | 126 | parser.error("Too many arguments.") |
411 | 381 | if not options.quiet: | 127 | logging.basicConfig( |
412 | 382 | print 'Sourcecode: %s' % (sourcecode_directory,) | 128 | format='%(message)s', |
413 | 383 | print 'Config: %s' % (config_filename,) | 129 | level=logging.CRITICAL if options.quiet else logging.INFO) |
414 | 384 | enable_default_logging() | 130 | logging.info('Sourcecode: %s', sourcecode_directory) |
415 | 385 | # Tell bzr to use the terminal (if any) to show progress bars | 131 | logging.info('Config: %s', config_filename) |
408 | 386 | ui.ui_factory = ui.make_ui_for_terminal( | ||
409 | 387 | sys.stdin, sys.stdout, sys.stderr) | ||
410 | 388 | load_plugins() | ||
416 | 389 | update_sourcecode( | 132 | update_sourcecode( |
420 | 390 | sourcecode_directory, config_filename, cache_filename, | 133 | sourcecode_directory, config_filename, |
421 | 391 | options.public_only, options.tip, options.dry_run, options.quiet, | 134 | tip=options.tip, dry_run=options.dry_run, use_http=options.use_http) |
419 | 392 | options.use_http) | ||
422 | 393 | return 0 | 135 | return 0 |
423 | diff --git a/lib/devscripts/tests/test_sourcecode.py b/lib/devscripts/tests/test_sourcecode.py | |||
424 | index a675333..01ab8f9 100644 | |||
425 | --- a/lib/devscripts/tests/test_sourcecode.py | |||
426 | +++ b/lib/devscripts/tests/test_sourcecode.py | |||
427 | @@ -1,221 +1,164 @@ | |||
429 | 1 | # Copyright 2009 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
430 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
431 | 3 | 3 | ||
432 | 4 | """Module docstring goes here.""" | 4 | """Module docstring goes here.""" |
433 | 5 | 5 | ||
434 | 6 | from __future__ import absolute_import, print_function, unicode_literals | ||
435 | 7 | |||
436 | 6 | __metaclass__ = type | 8 | __metaclass__ = type |
437 | 7 | 9 | ||
438 | 8 | import os | 10 | import os |
439 | 9 | import shutil | ||
440 | 10 | from StringIO import StringIO | ||
441 | 11 | import tempfile | 11 | import tempfile |
442 | 12 | import unittest | ||
443 | 13 | 12 | ||
447 | 14 | from bzrlib.bzrdir import BzrDir | 13 | from codetree.config import Config |
448 | 15 | from bzrlib.tests import TestCase | 14 | from fixtures import ( |
449 | 16 | from bzrlib.transport import get_transport | 15 | FakeLogger, |
450 | 16 | MonkeyPatch, | ||
451 | 17 | TempDir, | ||
452 | 18 | ) | ||
453 | 19 | import six | ||
454 | 20 | from testtools import TestCase | ||
455 | 17 | 21 | ||
456 | 18 | from devscripts import get_launchpad_root | 22 | from devscripts import get_launchpad_root |
457 | 19 | from devscripts.sourcecode import ( | 23 | from devscripts.sourcecode import ( |
458 | 20 | find_branches, | 24 | find_branches, |
461 | 21 | interpret_config, | 25 | mangle_config, |
460 | 22 | parse_config_file, | ||
462 | 23 | plan_update, | 26 | plan_update, |
463 | 27 | update_sourcecode, | ||
464 | 24 | ) | 28 | ) |
465 | 25 | 29 | ||
466 | 26 | 30 | ||
469 | 27 | class TestParseConfigFile(unittest.TestCase): | 31 | def make_config(lines): |
470 | 28 | """Tests for the config file parser.""" | 32 | with tempfile.NamedTemporaryFile('w') as config_file: |
471 | 33 | for line in lines: | ||
472 | 34 | print(line, file=config_file) | ||
473 | 35 | config_file.flush() | ||
474 | 36 | return Config([config_file.name]) | ||
475 | 29 | 37 | ||
476 | 30 | def makeFile(self, contents): | ||
477 | 31 | return StringIO(contents) | ||
478 | 32 | 38 | ||
483 | 33 | def test_empty(self): | 39 | class TestMangleConfig(TestCase): |
484 | 34 | # Parsing an empty config file returns an empty sequence. | 40 | """Tests for mangling configuration after codetree has parsed it.""" |
481 | 35 | empty_file = self.makeFile("") | ||
482 | 36 | self.assertEqual([], list(parse_config_file(empty_file))) | ||
485 | 37 | 41 | ||
490 | 38 | def test_single_value(self): | 42 | def setUp(self): |
491 | 39 | # Parsing a file containing a single key=value pair returns a sequence | 43 | super(TestMangleConfig, self).setUp() |
492 | 40 | # containing the (key, value) as a list. | 44 | self.tempdir = self.useFixture(TempDir()).path |
493 | 41 | config_file = self.makeFile("key value") | 45 | |
494 | 46 | def test_location(self): | ||
495 | 47 | # All locations are considered to be relative to the given | ||
496 | 48 | # sourcecode directory. | ||
497 | 49 | config = make_config(['key lp:~sabdfl/foo/trunk;revno=1']) | ||
498 | 50 | mangle_config(self.tempdir, config) | ||
499 | 42 | self.assertEqual( | 51 | self.assertEqual( |
510 | 43 | [['key', 'value']], list(parse_config_file(config_file))) | 52 | os.path.join(self.tempdir, 'key'), |
511 | 44 | 53 | config.directive_map['key'].location) | |
512 | 45 | def test_comment_ignored(self): | 54 | |
513 | 46 | # If a line begins with a '#', then its a comment. | 55 | def test_tip_false(self): |
514 | 47 | comment_only = self.makeFile('# foo') | 56 | # If tip=False is passed to mangle_config, revno options are left |
515 | 48 | self.assertEqual([], list(parse_config_file(comment_only))) | 57 | # untouched. |
516 | 49 | 58 | config = make_config(['key lp:~sabdfl/foo/trunk;revno=1']) | |
517 | 50 | def test_optional_value(self): | 59 | mangle_config(self.tempdir, config, tip=False) |
508 | 51 | # Lines in the config file can have a third optional entry. | ||
509 | 52 | config_file = self.makeFile('key value optional') | ||
518 | 53 | self.assertEqual( | 60 | self.assertEqual( |
526 | 54 | [['key', 'value', 'optional']], | 61 | {'revno': '1'}, config.directive_map['key'].source_options) |
527 | 55 | list(parse_config_file(config_file))) | 62 | |
528 | 56 | 63 | def test_tip_true(self): | |
529 | 57 | def test_whitespace_stripped(self): | 64 | # If tip=True is passed to mangle_config, revno options are removed. |
530 | 58 | # Any whitespace around any of the tokens in the config file are | 65 | config = make_config(['key lp:~sabdfl/foo/trunk;revno=1']) |
531 | 59 | # stripped out. | 66 | mangle_config(self.tempdir, config, tip=True) |
532 | 60 | config_file = self.makeFile(' key value optional ') | 67 | self.assertEqual({}, config.directive_map['key'].source_options) |
533 | 68 | |||
534 | 69 | def test_use_http_false(self): | ||
535 | 70 | # If use_http=False is passed to mangle_config, lp: branch URLs are | ||
536 | 71 | # left untouched. | ||
537 | 72 | url_path = '~sabdfl/foo/trunk' | ||
538 | 73 | config = make_config(['key lp:%s' % url_path]) | ||
539 | 74 | mangle_config(self.tempdir, config, use_http=False) | ||
540 | 61 | self.assertEqual( | 75 | self.assertEqual( |
617 | 62 | [['key', 'value', 'optional']], | 76 | 'lp:%s' % url_path, config.directive_map['key'].source.source) |
618 | 63 | list(parse_config_file(config_file))) | 77 | |
619 | 64 | 78 | def test_use_http_true(self): | |
620 | 65 | 79 | # If use_http=True is passed to mangle_config, lp: branch URLs are | |
621 | 66 | class TestInterpretConfiguration(unittest.TestCase): | 80 | # transformed into http:// URLs. |
622 | 67 | """Tests for the configuration interpreter.""" | 81 | url_path = '~sabdfl/foo/trunk' |
623 | 68 | 82 | config = make_config(['key lp:%s' % url_path]) | |
624 | 69 | def test_empty(self): | 83 | mangle_config(self.tempdir, config, use_http=True) |
625 | 70 | # An empty configuration stream means no configuration. | 84 | self.assertEqual( |
626 | 71 | config = interpret_config([], False) | 85 | 'http://bazaar.launchpad.net/%s' % url_path, |
627 | 72 | self.assertEqual({}, config) | 86 | config.directive_map['key'].source.source) |
628 | 73 | 87 | ||
629 | 74 | def test_key_value(self): | 88 | |
630 | 75 | # A (key, value) pair without a third optional value is returned in | 89 | class TestPlanUpdate(TestCase): |
555 | 76 | # the configuration as a dictionary entry under 'key' with '(value, | ||
556 | 77 | # None, False)' as its value. | ||
557 | 78 | config = interpret_config([['key', 'value']], False) | ||
558 | 79 | self.assertEqual({'key': ('value', None, False)}, config) | ||
559 | 80 | |||
560 | 81 | def test_key_value_public_only(self): | ||
561 | 82 | # A (key, value) pair without a third optional value is returned in | ||
562 | 83 | # the configuration as a dictionary entry under 'key' with '(value, | ||
563 | 84 | # None, False)' as its value when public_only is true. | ||
564 | 85 | config = interpret_config([['key', 'value']], True) | ||
565 | 86 | self.assertEqual({'key': ('value', None, False)}, config) | ||
566 | 87 | |||
567 | 88 | def test_key_value_optional(self): | ||
568 | 89 | # A (key, value, optional) entry is returned in the configuration as a | ||
569 | 90 | # dictionary entry under 'key' with '(value, True)' as its value. | ||
570 | 91 | config = interpret_config([['key', 'value', 'optional']], False) | ||
571 | 92 | self.assertEqual({'key': ('value', None, True)}, config) | ||
572 | 93 | |||
573 | 94 | def test_key_value_optional_public_only(self): | ||
574 | 95 | # A (key, value, optional) entry is not returned in the configuration | ||
575 | 96 | # when public_only is true. | ||
576 | 97 | config = interpret_config([['key', 'value', 'optional']], True) | ||
577 | 98 | self.assertEqual({}, config) | ||
578 | 99 | |||
579 | 100 | def test_key_value_revision(self): | ||
580 | 101 | # A (key, value) pair without a third optional value when the | ||
581 | 102 | # value has a suffix of ``;revno=[REVISION]`` is returned in the | ||
582 | 103 | # configuration as a dictionary entry under 'key' with '(value, | ||
583 | 104 | # None, False)' as its value. | ||
584 | 105 | config = interpret_config([['key', 'value;revno=45']], False) | ||
585 | 106 | self.assertEqual({'key': ('value', '45', False)}, config) | ||
586 | 107 | |||
587 | 108 | def test_key_value_revision(self): | ||
588 | 109 | # A (key, value) pair without a third optional value when the | ||
589 | 110 | # value has multiple suffixes of ``;revno=[REVISION]`` raises an | ||
590 | 111 | # error. | ||
591 | 112 | self.assertRaises( | ||
592 | 113 | AssertionError, | ||
593 | 114 | interpret_config, [['key', 'value;revno=45;revno=47']], False) | ||
594 | 115 | |||
595 | 116 | def test_too_many_values(self): | ||
596 | 117 | # A line with too many values raises an error. | ||
597 | 118 | self.assertRaises( | ||
598 | 119 | AssertionError, | ||
599 | 120 | interpret_config, [['key', 'value', 'optional', 'extra']], False) | ||
600 | 121 | |||
601 | 122 | def test_bad_optional_value(self): | ||
602 | 123 | # A third value that is not the "optional" string raises an error. | ||
603 | 124 | self.assertRaises( | ||
604 | 125 | AssertionError, | ||
605 | 126 | interpret_config, [['key', 'value', 'extra']], False) | ||
606 | 127 | |||
607 | 128 | def test_use_http(self): | ||
608 | 129 | # If use_http=True is passed to interpret_config, all lp: branch | ||
609 | 130 | # URLs will be transformed into http:// URLs. | ||
610 | 131 | config = interpret_config( | ||
611 | 132 | [['key', 'lp:~sabdfl/foo/trunk']], False, use_http=True) | ||
612 | 133 | expected_url = 'http://bazaar.launchpad.net/~sabdfl/foo/trunk' | ||
613 | 134 | self.assertEqual(expected_url, config['key'][0]) | ||
614 | 135 | |||
615 | 136 | |||
616 | 137 | class TestPlanUpdate(unittest.TestCase): | ||
631 | 138 | """Tests for how to plan the update.""" | 90 | """Tests for how to plan the update.""" |
632 | 139 | 91 | ||
633 | 140 | def test_trivial(self): | 92 | def test_trivial(self): |
634 | 141 | # In the trivial case, there are no existing branches and no | 93 | # In the trivial case, there are no existing branches and no |
640 | 142 | # configured branches, so there are no branches to add, none to | 94 | # configured branches, so there are none to remove. |
641 | 143 | # update, and none to remove. | 95 | removed = plan_update([], make_config([])) |
637 | 144 | new, existing, removed = plan_update([], {}) | ||
638 | 145 | self.assertEqual({}, new) | ||
639 | 146 | self.assertEqual({}, existing) | ||
642 | 147 | self.assertEqual(set(), removed) | 96 | self.assertEqual(set(), removed) |
643 | 148 | 97 | ||
644 | 149 | def test_all_new(self): | 98 | def test_all_new(self): |
650 | 150 | # If there are no existing branches, then the all of the configured | 99 | # If there are no existing branches, then none have been removed. |
651 | 151 | # branches are new, none are existing and none have been removed. | 100 | removed = plan_update([], make_config(['a lp:a'])) |
647 | 152 | new, existing, removed = plan_update([], {'a': ('b', False)}) | ||
648 | 153 | self.assertEqual({'a': ('b', False)}, new) | ||
649 | 154 | self.assertEqual({}, existing) | ||
652 | 155 | self.assertEqual(set(), removed) | 101 | self.assertEqual(set(), removed) |
653 | 156 | 102 | ||
654 | 157 | def test_all_old(self): | 103 | def test_all_old(self): |
661 | 158 | # If there configuration is now empty, but there are existing | 104 | # If the configuration is now empty but there are existing branches, |
662 | 159 | # branches, then that means all the branches have been removed from | 105 | # then that means all the branches have been removed from the |
663 | 160 | # the configuration, none are new and none are updated. | 106 | # configuration. |
664 | 161 | new, existing, removed = plan_update(['a', 'b', 'c'], {}) | 107 | removed = plan_update(['a', 'b', 'c'], make_config([])) |
659 | 162 | self.assertEqual({}, new) | ||
660 | 163 | self.assertEqual({}, existing) | ||
665 | 164 | self.assertEqual(set(['a', 'b', 'c']), removed) | 108 | self.assertEqual(set(['a', 'b', 'c']), removed) |
666 | 165 | 109 | ||
667 | 166 | def test_all_same(self): | 110 | def test_all_same(self): |
668 | 167 | # If the set of existing branches is the same as the set of | 111 | # If the set of existing branches is the same as the set of |
674 | 168 | # non-existing branches, then they all need to be updated. | 112 | # non-existing branches, then none have been removed. |
675 | 169 | config = {'a': ('b', False), 'c': ('d', True)} | 113 | config = make_config(['a lp:a', 'b lp:b']) |
676 | 170 | new, existing, removed = plan_update(config.keys(), config) | 114 | removed = plan_update(config.directive_map.keys(), config) |
672 | 171 | self.assertEqual({}, new) | ||
673 | 172 | self.assertEqual(config, existing) | ||
677 | 173 | self.assertEqual(set(), removed) | 115 | self.assertEqual(set(), removed) |
678 | 174 | 116 | ||
688 | 175 | def test_smoke_the_default_config(self): | 117 | def test_some_old(self): |
689 | 176 | # Make sure we can parse, interpret and plan based on the default | 118 | # If there are existing branches not in the configuration, then |
690 | 177 | # config file. | 119 | # those branches have been removed. |
691 | 178 | root = get_launchpad_root() | 120 | removed = plan_update(['a', 'b', 'c'], make_config(['a lp:a'])) |
692 | 179 | config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf') | 121 | self.assertEqual(set(['b', 'c']), removed) |
684 | 180 | config_file = open(config_filename) | ||
685 | 181 | config = interpret_config(parse_config_file(config_file), False) | ||
686 | 182 | config_file.close() | ||
687 | 183 | plan_update([], config) | ||
693 | 184 | 122 | ||
694 | 185 | 123 | ||
695 | 186 | class TestFindBranches(TestCase): | 124 | class TestFindBranches(TestCase): |
696 | 187 | """Tests the way that we find branches.""" | 125 | """Tests the way that we find branches.""" |
697 | 188 | 126 | ||
698 | 189 | def setUp(self): | 127 | def setUp(self): |
712 | 190 | TestCase.setUp(self) | 128 | super(TestFindBranches, self).setUp() |
713 | 191 | self.disable_directory_isolation() | 129 | self.tempdir = self.useFixture(TempDir()).path |
701 | 192 | |||
702 | 193 | def makeBranch(self, path): | ||
703 | 194 | transport = get_transport(path) | ||
704 | 195 | transport.ensure_base() | ||
705 | 196 | BzrDir.create_branch_convenience( | ||
706 | 197 | transport.base, possible_transports=[transport]) | ||
707 | 198 | |||
708 | 199 | def makeDirectory(self): | ||
709 | 200 | directory = tempfile.mkdtemp() | ||
710 | 201 | self.addCleanup(shutil.rmtree, directory) | ||
711 | 202 | return directory | ||
714 | 203 | 130 | ||
715 | 204 | def test_empty_directory_has_no_branches(self): | 131 | def test_empty_directory_has_no_branches(self): |
716 | 205 | # An empty directory has no branches. | 132 | # An empty directory has no branches. |
719 | 206 | empty = self.makeDirectory() | 133 | self.assertEqual(set(), set(find_branches(self.tempdir))) |
718 | 207 | self.assertEqual([], list(find_branches(empty))) | ||
720 | 208 | 134 | ||
721 | 209 | def test_directory_with_branches(self): | 135 | def test_directory_with_branches(self): |
722 | 210 | # find_branches finds branches in the directory. | 136 | # find_branches finds branches in the directory. |
726 | 211 | directory = self.makeDirectory() | 137 | os.makedirs(os.path.join(self.tempdir, 'a', '.bzr')) |
727 | 212 | self.makeBranch('%s/a' % directory) | 138 | os.makedirs(os.path.join(self.tempdir, 'b', '.git')) |
728 | 213 | self.assertEqual(['a'], list(find_branches(directory))) | 139 | self.assertEqual(set(['a', 'b']), set(find_branches(self.tempdir))) |
729 | 140 | |||
730 | 141 | def test_ignores_non_branch_directory(self): | ||
731 | 142 | # find_branches ignores any subdirectories in the directory which | ||
732 | 143 | # are not branches. | ||
733 | 144 | os.mkdir(os.path.join(self.tempdir, 'a')) | ||
734 | 145 | self.assertEqual(set(), set(find_branches(self.tempdir))) | ||
735 | 214 | 146 | ||
736 | 215 | def test_ignores_files(self): | 147 | def test_ignores_files(self): |
737 | 216 | # find_branches ignores any files in the directory. | 148 | # find_branches ignores any files in the directory. |
743 | 217 | directory = self.makeDirectory() | 149 | with open(os.path.join(self.tempdir, 'a'), 'w') as some_file: |
744 | 218 | some_file = open('%s/a' % directory, 'w') | 150 | some_file.write('hello\n') |
745 | 219 | some_file.write('hello\n') | 151 | self.assertEqual(set(), set(find_branches(self.tempdir))) |
746 | 220 | some_file.close() | 152 | |
747 | 221 | self.assertEqual([], list(find_branches(directory))) | 153 | |
748 | 154 | class TestSmoke(TestCase): | ||
749 | 155 | """Smoke tests.""" | ||
750 | 156 | |||
751 | 157 | def test_smoke_the_default_config(self): | ||
752 | 158 | # Make sure we can do a dry run based on the default config file. | ||
753 | 159 | self.useFixture(FakeLogger()) | ||
754 | 160 | self.useFixture(MonkeyPatch('sys.stdout', six.StringIO())) | ||
755 | 161 | root = get_launchpad_root() | ||
756 | 162 | config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf') | ||
757 | 163 | fake_sourcecode = self.useFixture(TempDir()).path | ||
758 | 164 | update_sourcecode(fake_sourcecode, config_filename, dry_run=True) | ||
759 | diff --git a/utilities/sourcedeps.cache b/utilities/sourcedeps.cache | |||
760 | 222 | deleted file mode 100644 | 165 | deleted file mode 100644 |
761 | index 1360489..0000000 | |||
762 | --- a/utilities/sourcedeps.cache | |||
763 | +++ /dev/null | |||
764 | @@ -1,42 +0,0 @@ | |||
765 | 1 | { | ||
766 | 2 | "bzr-builder": [ | ||
767 | 3 | 70, | ||
768 | 4 | "launchpad@pqm.canonical.com-20111114140506-6bmt9isw6lcud7yt" | ||
769 | 5 | ], | ||
770 | 6 | "bzr-git": [ | ||
771 | 7 | 280, | ||
772 | 8 | "launchpad@pqm.canonical.com-20171222005919-u98ut0f5z2g618um" | ||
773 | 9 | ], | ||
774 | 10 | "bzr-loom": [ | ||
775 | 11 | 55, | ||
776 | 12 | "launchpad@pqm.canonical.com-20120830090804-cg49kky93htwax7s" | ||
777 | 13 | ], | ||
778 | 14 | "bzr-svn": [ | ||
779 | 15 | 2725, | ||
780 | 16 | "launchpad@pqm.canonical.com-20130816045016-wzr810hu2z459t4y" | ||
781 | 17 | ], | ||
782 | 18 | "cscvs": [ | ||
783 | 19 | 433, | ||
784 | 20 | "launchpad@pqm.canonical.com-20130816043319-bts3l3bckmx431q1" | ||
785 | 21 | ], | ||
786 | 22 | "difftacular": [ | ||
787 | 23 | 11, | ||
788 | 24 | "cjwatson@canonical.com-20190614154330-091l9edcnubsjmsx" | ||
789 | 25 | ], | ||
790 | 26 | "loggerhead": [ | ||
791 | 27 | 493, | ||
792 | 28 | "cjwatson@canonical.com-20190621112125-3aaxj3hrmty19lr6" | ||
793 | 29 | ], | ||
794 | 30 | "mailman": [ | ||
795 | 31 | 977, | ||
796 | 32 | "launchpad@pqm.canonical.com-20130405041235-9ud0xancja2eefd7" | ||
797 | 33 | ], | ||
798 | 34 | "old_xmlplus": [ | ||
799 | 35 | 4, | ||
800 | 36 | "sinzui-20090526164636-1swugzupwvjgomo4" | ||
801 | 37 | ], | ||
802 | 38 | "pygettextpo": [ | ||
803 | 39 | 25, | ||
804 | 40 | "launchpad@pqm.canonical.com-20140116030912-lqm1dtb6a0y4femq" | ||
805 | 41 | ] | ||
806 | 42 | } | ||
807 | 43 | \ No newline at end of file | 0 | \ No newline at end of file |
808 | diff --git a/utilities/sourcedeps.filter b/utilities/sourcedeps.filter | |||
809 | 44 | deleted file mode 100644 | 1 | deleted file mode 100644 |
810 | index d81ca05..0000000 | |||
811 | --- a/utilities/sourcedeps.filter | |||
812 | +++ /dev/null | |||
813 | @@ -1,3 +0,0 @@ | |||
814 | 1 | P *.o | ||
815 | 2 | P *.pyc | ||
816 | 3 | P *.so |
Superseded by https:/ /code.launchpad .net/~cjwatson/ launchpad/ +git/launchpad/ +merge/ 428729.