Merge lp:~cjwatson/launchpad-buildd/ttb-git into lp:launchpad-buildd

Proposed by Colin Watson on 2018-03-14
Status: Needs review
Proposed branch: lp:~cjwatson/launchpad-buildd/ttb-git
Merge into: lp:launchpad-buildd
Prerequisite: lp:~cjwatson/launchpad-buildd/refactor-vcs
Diff against target: 532 lines (+246/-152)
5 files modified
debian/changelog (+2/-0)
lpbuildd/target/generate_translation_templates.py (+25/-40)
lpbuildd/target/tests/test_generate_translation_templates.py (+202/-107)
lpbuildd/tests/test_translationtemplatesbuildmanager.py (+1/-1)
lpbuildd/translationtemplates.py (+16/-4)
To merge this branch: bzr merge lp:~cjwatson/launchpad-buildd/ttb-git
Reviewer Review Type Date Requested Status
Launchpad code reviewers 2018-03-14 Pending
Review via email: mp+341389@code.launchpad.net

Commit message

Convert translation templates builds to the new VCS mixin, thereby adding git support.

Description of the change

There'll be a fair bit of work needed on the Launchpad side before this is useful, but we might as well get started here.

To post a comment you must log in.

Unmerged revisions

329. By Colin Watson on 2018-03-14

Convert translation templates builds to the new VCS mixin, thereby adding git support.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'debian/changelog'
2--- debian/changelog 2018-03-14 14:15:15 +0000
3+++ debian/changelog 2018-03-14 14:15:15 +0000
4@@ -7,6 +7,8 @@
5 * Allow optionally installing snapcraft as a snap (LP: #1737994).
6 * Refactor VCS operations from lpbuildd.target.build_snap out to a module
7 that can be used by other targets.
8+ * Convert translation templates builds to the new VCS mixin, thereby
9+ adding git support.
10
11 -- Colin Watson <cjwatson@ubuntu.com> Fri, 02 Feb 2018 16:14:46 +0000
12
13
14=== modified file 'lpbuildd/target/generate_translation_templates.py'
15--- lpbuildd/target/generate_translation_templates.py 2017-11-10 22:13:03 +0000
16+++ lpbuildd/target/generate_translation_templates.py 2018-03-14 14:15:15 +0000
17@@ -10,6 +10,7 @@
18
19 from lpbuildd.pottery import intltool
20 from lpbuildd.target.operation import Operation
21+from lpbuildd.target.vcs import VCSOperationMixin
22
23
24 logger = logging.getLogger(__name__)
25@@ -19,7 +20,7 @@
26 RETCODE_FAILURE_BUILD = 201
27
28
29-class GenerateTranslationTemplates(Operation):
30+class GenerateTranslationTemplates(VCSOperationMixin, Operation):
31 """Script to generate translation templates from a branch."""
32
33 description = "Generate templates for a branch."
34@@ -28,42 +29,24 @@
35 def add_arguments(cls, parser):
36 super(GenerateTranslationTemplates, cls).add_arguments(parser)
37 parser.add_argument(
38- "branch_spec", help=(
39- "A branch URL or the path of a local branch. URLs are "
40- "recognised by the occurrence of ':'. In the case of a URL, "
41- "this will make up a path for the branch and check out the "
42- "branch to there."))
43- parser.add_argument(
44 "result_name",
45- help="The name of the result tarball. Should end in '.tar.gz'.")
46+ help="the name of the result tarball; should end in '.tar.gz'")
47
48 def __init__(self, args, parser):
49 super(GenerateTranslationTemplates, self).__init__(args, parser)
50 self.work_dir = os.environ["HOME"]
51-
52- def _getBranch(self):
53- """Set `self.branch_dir`, and check out branch if needed."""
54- if ':' in self.args.branch_spec:
55- # This is a branch URL. Check out the branch.
56- self.branch_dir = os.path.join(self.work_dir, 'source-tree')
57- logger.info("Getting remote branch %s..." % self.args.branch_spec)
58- self._checkout(self.args.branch_spec)
59- else:
60- # This is a local filesystem path. Use the branch in-place.
61- logger.info("Using local branch %s..." % self.args.branch_spec)
62- self.branch_dir = self.args.branch_spec
63-
64- def _checkout(self, branch_url):
65- """Check out a source branch to generate from.
66-
67- The branch is checked out to the location specified by
68- `self.branch_dir`.
69- """
70- logger.info(
71- "Exporting branch %s to %s..." % (branch_url, self.branch_dir))
72- self.backend.run(
73- ["bzr", "export", "-q", "-d", branch_url, self.branch_dir])
74- logger.info("Exporting branch done.")
75+ self.branch_dir = os.path.join(self.work_dir, "source-tree")
76+
77+ def install(self):
78+ logger.info("Installing dependencies...")
79+ deps = ["intltool"]
80+ deps.extend(self.vcs_deps)
81+ self.backend.run(["apt-get", "-y", "install"] + deps)
82+
83+ def fetch(self, quiet=False):
84+ logger.info("Fetching %s...", self.vcs_description)
85+ self.vcs_fetch(
86+ os.path.basename(self.branch_dir), cwd=self.work_dir, quiet=quiet)
87
88 def _makeTarball(self, files):
89 """Put the given files into a tarball in the working directory."""
90@@ -78,21 +61,23 @@
91 self.backend.run(cmd)
92 logger.info("Tarball generated.")
93
94+ def generate(self):
95+ logger.info("Generating templates...")
96+ pots = intltool.generate_pots(self.backend, self.branch_dir)
97+ logger.info("Generated %d templates." % len(pots))
98+ if len(pots) > 0:
99+ self._makeTarball(pots)
100+
101 def run(self):
102 """Do It. Generate templates."""
103 try:
104- self.backend.run(["apt-get", "-y", "install", "bzr", "intltool"])
105+ self.install()
106 except Exception:
107 logger.exception("Install failed")
108 return RETCODE_FAILURE_INSTALL
109 try:
110- logger.info(
111- "Generating templates for %s." % self.args.branch_spec)
112- self._getBranch()
113- pots = intltool.generate_pots(self.backend, self.branch_dir)
114- logger.info("Generated %d templates." % len(pots))
115- if len(pots) > 0:
116- self._makeTarball(pots)
117+ self.fetch()
118+ self.generate()
119 except Exception:
120 logger.exception("Build failed")
121 return RETCODE_FAILURE_BUILD
122
123=== modified file 'lpbuildd/target/tests/test_generate_translation_templates.py'
124--- lpbuildd/target/tests/test_generate_translation_templates.py 2017-10-27 08:08:49 +0000
125+++ lpbuildd/target/tests/test_generate_translation_templates.py 2018-03-14 14:15:15 +0000
126@@ -15,23 +15,37 @@
127 from testtools import TestCase
128 from testtools.matchers import (
129 ContainsDict,
130- EndsWith,
131 Equals,
132+ Is,
133+ MatchesDict,
134 MatchesListwise,
135 MatchesSetwise,
136 )
137
138 from lpbuildd.target.cli import parse_args
139-from lpbuildd.tests.fakeslave import FakeMethod
140-
141-
142-class MatchesCall(MatchesListwise):
143-
144- def __init__(self, *args, **kwargs):
145- super(MatchesCall, self).__init__([
146- Equals(args),
147- ContainsDict(
148- {name: Equals(value) for name, value in kwargs.items()})])
149+
150+
151+class RanCommand(MatchesListwise):
152+
153+ def __init__(self, args, get_output=None, echo=None, cwd=None, **env):
154+ kwargs_matcher = {}
155+ if get_output is not None:
156+ kwargs_matcher["get_output"] = Is(get_output)
157+ if echo is not None:
158+ kwargs_matcher["echo"] = Is(echo)
159+ if cwd:
160+ kwargs_matcher["cwd"] = Equals(cwd)
161+ if env:
162+ kwargs_matcher["env"] = MatchesDict(
163+ {key: Equals(value) for key, value in env.items()})
164+ super(RanCommand, self).__init__(
165+ [Equals((args,)), ContainsDict(kwargs_matcher)])
166+
167+
168+class RanAptGet(RanCommand):
169+
170+ def __init__(self, *args):
171+ super(RanAptGet, self).__init__(["apt-get", "-y"] + list(args))
172
173
174 class TestGenerateTranslationTemplates(TestCase):
175@@ -45,82 +59,125 @@
176 self.useFixture(EnvironmentVariable("HOME", self.home_dir))
177 self.logger = self.useFixture(FakeLogger())
178
179- def test_getBranch_url(self):
180- # If passed a branch URL, the template generation script will
181- # check out that branch into a directory called "source-tree."
182- args = [
183- "generate-translation-templates",
184- "--backend=fake", "--series=xenial", "--arch=amd64", "1",
185- "lp://~my/translation/branch", self.result_name,
186- ]
187- generator = parse_args(args=args).operation
188- generator._checkout = FakeMethod()
189- generator._getBranch()
190-
191- self.assertEqual(1, generator._checkout.call_count)
192- self.assertThat(generator.branch_dir, EndsWith("source-tree"))
193-
194- def test_getBranch_dir(self):
195- # If passed a branch directory, the template generation script
196- # works directly in that directory.
197- branch_dir = "/home/me/branch"
198- args = [
199- "generate-translation-templates",
200- "--backend=fake", "--series=xenial", "--arch=amd64", "1",
201- branch_dir, self.result_name,
202- ]
203- generator = parse_args(args=args).operation
204- generator._checkout = FakeMethod()
205- generator._getBranch()
206-
207- self.assertEqual(0, generator._checkout.call_count)
208- self.assertEqual(branch_dir, generator.branch_dir)
209-
210- def _createBranch(self, content_map=None):
211- """Create a working branch.
212-
213- :param content_map: optional dict mapping file names to file
214- contents. Each of these files with their contents will be
215- written to the branch. Currently only supports writing files at
216- the root directory of the branch.
217-
218- :return: the URL of a fresh bzr branch.
219+ def make_branch_contents(self, content_map):
220+ """Create a directory with the contents of a working branch.
221+
222+ :param content_map: A dict mapping file names to file contents.
223+ Each of these files with their contents will be written to the
224+ branch. Currently only supports writing files at the root
225+ directory of the branch.
226 """
227 branch_path = self.useFixture(TempDir()).path
228- branch_url = 'file://' + branch_path
229- subprocess.check_call(['bzr', 'init', '-q'], cwd=branch_path)
230-
231- if content_map is not None:
232- for name, contents in content_map.items():
233- with open(os.path.join(branch_path, name), 'wb') as f:
234- f.write(contents)
235+ for name, contents in content_map.items():
236+ with open(os.path.join(branch_path, name), 'wb') as f:
237+ f.write(contents)
238+ return branch_path
239+
240+ def make_bzr_branch(self, branch_path):
241+ """Make a bzr branch from an existing directory."""
242+ bzr_home = self.useFixture(TempDir()).path
243+ self.useFixture(EnvironmentVariable("BZR_HOME", bzr_home))
244+ self.useFixture(EnvironmentVariable("BZR_EMAIL"))
245+ self.useFixture(EnvironmentVariable("EMAIL"))
246+
247+ subprocess.check_call(["bzr", "init", "-q"], cwd=branch_path)
248+ subprocess.check_call(["bzr", "add", "-q"], cwd=branch_path)
249+ committer_id = "Committer <committer@example.com>"
250+ with EnvironmentVariable("BZR_EMAIL", committer_id):
251 subprocess.check_call(
252- ['bzr', 'add', '-q'] + list(content_map), cwd=branch_path)
253- committer_id = 'Committer <committer@example.com>'
254- with EnvironmentVariable('BZR_EMAIL', committer_id):
255- subprocess.check_call(
256- ['bzr', 'commit', '-q', '-m', 'Populating branch.'],
257- cwd=branch_path)
258-
259- return branch_url
260-
261- def test_getBranch_bzr(self):
262- # _getBranch can retrieve branch contents from a branch URL.
263- bzr_home = self.useFixture(TempDir()).path
264- self.useFixture(EnvironmentVariable('BZR_HOME', bzr_home))
265- self.useFixture(EnvironmentVariable('BZR_EMAIL'))
266- self.useFixture(EnvironmentVariable('EMAIL'))
267-
268- marker_text = "Ceci n'est pas cet branch."
269- branch_url = self._createBranch({'marker.txt': marker_text})
270-
271- args = [
272- "generate-translation-templates",
273- "--backend=uncontained", "--series=xenial", "--arch=amd64", "1",
274- branch_url, self.result_name,
275- ]
276- generator = parse_args(args=args).operation
277- generator._getBranch()
278+ ["bzr", "commit", "-q", "-m", "Populating branch."],
279+ cwd=branch_path)
280+
281+ def make_git_branch(self, branch_path):
282+ subprocess.check_call(["git", "init", "-q"], cwd=branch_path)
283+ subprocess.check_call(
284+ ["git", "config", "user.name", "Committer"], cwd=branch_path)
285+ subprocess.check_call(
286+ ["git", "config", "user.email", "committer@example.com"],
287+ cwd=branch_path)
288+ subprocess.check_call(["git", "add", "."], cwd=branch_path)
289+ subprocess.check_call(
290+ ["git", "commit", "-q", "--allow-empty",
291+ "-m", "Populating branch"],
292+ cwd=branch_path)
293+
294+ def test_install_bzr(self):
295+ args = [
296+ "generate-translation-templates",
297+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
298+ "--branch", "lp:foo", self.result_name,
299+ ]
300+ generator = parse_args(args=args).operation
301+ generator.install()
302+ self.assertThat(generator.backend.run.calls, MatchesListwise([
303+ RanAptGet("install", "intltool", "bzr"),
304+ ]))
305+
306+ def test_install_git(self):
307+ args = [
308+ "generate-translation-templates",
309+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
310+ "--git-repository", "lp:foo", self.result_name,
311+ ]
312+ generator = parse_args(args=args).operation
313+ generator.install()
314+ self.assertThat(generator.backend.run.calls, MatchesListwise([
315+ RanAptGet("install", "intltool", "git"),
316+ ]))
317+
318+ def test_fetch_bzr(self):
319+ # fetch can retrieve branch contents from a Bazaar branch.
320+ marker_text = "Ceci n'est pas cet branch."
321+ branch_path = self.make_branch_contents({'marker.txt': marker_text})
322+ self.make_bzr_branch(branch_path)
323+
324+ args = [
325+ "generate-translation-templates",
326+ "--backend=uncontained", "--series=xenial", "--arch=amd64", "1",
327+ "--branch", branch_path, self.result_name,
328+ ]
329+ generator = parse_args(args=args).operation
330+ generator.fetch(quiet=True)
331+
332+ marker_path = os.path.join(generator.branch_dir, 'marker.txt')
333+ with open(marker_path) as marker_file:
334+ self.assertEqual(marker_text, marker_file.read())
335+
336+ def test_fetch_git(self):
337+ # fetch can retrieve branch contents from a Git repository.
338+ marker_text = "Ceci n'est pas cet branch."
339+ branch_path = self.make_branch_contents({'marker.txt': marker_text})
340+ self.make_git_branch(branch_path)
341+
342+ args = [
343+ "generate-translation-templates",
344+ "--backend=uncontained", "--series=xenial", "--arch=amd64", "1",
345+ "--git-repository", branch_path, self.result_name,
346+ ]
347+ generator = parse_args(args=args).operation
348+ generator.fetch(quiet=True)
349+
350+ marker_path = os.path.join(generator.branch_dir, 'marker.txt')
351+ with open(marker_path) as marker_file:
352+ self.assertEqual(marker_text, marker_file.read())
353+
354+ def test_fetch_git_with_path(self):
355+ # fetch can retrieve branch contents from a Git repository and
356+ # branch name.
357+ marker_text = "Ceci n'est pas cet branch."
358+ branch_path = self.make_branch_contents({'marker.txt': marker_text})
359+ self.make_git_branch(branch_path)
360+ subprocess.call(
361+ ["git", "branch", "-m", "master", "next"], cwd=branch_path)
362+
363+ args = [
364+ "generate-translation-templates",
365+ "--backend=uncontained", "--series=xenial", "--arch=amd64", "1",
366+ "--git-repository", branch_path, "--git-path", "next",
367+ self.result_name,
368+ ]
369+ generator = parse_args(args=args).operation
370+ generator.fetch(quiet=True)
371
372 marker_path = os.path.join(generator.branch_dir, 'marker.txt')
373 with open(marker_path) as marker_file:
374@@ -136,22 +193,23 @@
375 potnames = [
376 member.name
377 for member in tar.getmembers() if not member.isdir()]
378+ self.make_bzr_branch(branchdir)
379
380 args = [
381 "generate-translation-templates",
382 "--backend=uncontained", "--series=xenial", "--arch=amd64", "1",
383- branchdir, self.result_name,
384+ "--branch", branchdir, self.result_name,
385 ]
386 generator = parse_args(args=args).operation
387- generator._getBranch()
388+ generator.fetch(quiet=True)
389 generator._makeTarball(potnames)
390 result_path = os.path.join(self.home_dir, self.result_name)
391 with tarfile.open(result_path, 'r|*') as tar:
392 tarnames = tar.getnames()
393 self.assertThat(tarnames, MatchesSetwise(*(map(Equals, potnames))))
394
395- def test_run(self):
396- # Install dependencies and generate a templates tarball.
397+ def test_run_bzr(self):
398+ # Install dependencies and generate a templates tarball from Bazaar.
399 branch_url = "lp:~my/branch"
400 branch_dir = os.path.join(self.home_dir, "source-tree")
401 po_dir = os.path.join(branch_dir, "po")
402@@ -160,24 +218,61 @@
403 args = [
404 "generate-translation-templates",
405 "--backend=fake", "--series=xenial", "--arch=amd64", "1",
406- branch_url, self.result_name,
407- ]
408- generator = parse_args(args=args).operation
409- generator.backend.add_file(os.path.join(po_dir, "POTFILES.in"), "")
410- generator.backend.add_file(
411- os.path.join(po_dir, "Makevars"), "DOMAIN = test\n")
412- generator.run()
413- self.assertThat(generator.backend.run.calls, MatchesListwise([
414- MatchesCall(["apt-get", "-y", "install", "bzr", "intltool"]),
415- MatchesCall(
416- ["bzr", "export", "-q", "-d", "lp:~my/branch", branch_dir]),
417- MatchesCall(
418- ["rm", "-f",
419- os.path.join(po_dir, "missing"),
420- os.path.join(po_dir, "notexist")]),
421- MatchesCall(["/usr/bin/intltool-update", "-m"], cwd=po_dir),
422- MatchesCall(
423- ["/usr/bin/intltool-update", "-p", "-g", "test"], cwd=po_dir),
424- MatchesCall(
425+ "--branch", branch_url, self.result_name,
426+ ]
427+ generator = parse_args(args=args).operation
428+ generator.backend.add_file(os.path.join(po_dir, "POTFILES.in"), "")
429+ generator.backend.add_file(
430+ os.path.join(po_dir, "Makevars"), "DOMAIN = test\n")
431+ generator.run()
432+ self.assertThat(generator.backend.run.calls, MatchesListwise([
433+ RanAptGet("install", "intltool", "bzr"),
434+ RanCommand(
435+ ["bzr", "branch", "lp:~my/branch", "source-tree"],
436+ cwd=self.home_dir, LANG="C.UTF-8", SHELL="/bin/sh"),
437+ RanCommand(
438+ ["rm", "-f",
439+ os.path.join(po_dir, "missing"),
440+ os.path.join(po_dir, "notexist")]),
441+ RanCommand(["/usr/bin/intltool-update", "-m"], cwd=po_dir),
442+ RanCommand(
443+ ["/usr/bin/intltool-update", "-p", "-g", "test"], cwd=po_dir),
444+ RanCommand(
445+ ["tar", "-C", branch_dir, "-czf", result_path, "po/test.pot"]),
446+ ]))
447+
448+ def test_run_git(self):
449+ # Install dependencies and generate a templates tarball from Git.
450+ repository_url = "lp:~my/repository"
451+ branch_dir = os.path.join(self.home_dir, "source-tree")
452+ po_dir = os.path.join(branch_dir, "po")
453+ result_path = os.path.join(self.home_dir, self.result_name)
454+
455+ args = [
456+ "generate-translation-templates",
457+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
458+ "--git-repository", repository_url, self.result_name,
459+ ]
460+ generator = parse_args(args=args).operation
461+ generator.backend.add_file(os.path.join(po_dir, "POTFILES.in"), "")
462+ generator.backend.add_file(
463+ os.path.join(po_dir, "Makevars"), "DOMAIN = test\n")
464+ generator.run()
465+ self.assertThat(generator.backend.run.calls, MatchesListwise([
466+ RanAptGet("install", "intltool", "git"),
467+ RanCommand(
468+ ["git", "clone", "lp:~my/repository", "source-tree"],
469+ cwd=self.home_dir, LANG="C.UTF-8", SHELL="/bin/sh"),
470+ RanCommand(
471+ ["git", "submodule", "update", "--init", "--recursive"],
472+ cwd=branch_dir, LANG="C.UTF-8", SHELL="/bin/sh"),
473+ RanCommand(
474+ ["rm", "-f",
475+ os.path.join(po_dir, "missing"),
476+ os.path.join(po_dir, "notexist")]),
477+ RanCommand(["/usr/bin/intltool-update", "-m"], cwd=po_dir),
478+ RanCommand(
479+ ["/usr/bin/intltool-update", "-p", "-g", "test"], cwd=po_dir),
480+ RanCommand(
481 ["tar", "-C", branch_dir, "-czf", result_path, "po/test.pot"]),
482 ]))
483
484=== modified file 'lpbuildd/tests/test_translationtemplatesbuildmanager.py'
485--- lpbuildd/tests/test_translationtemplatesbuildmanager.py 2017-09-08 15:57:18 +0000
486+++ lpbuildd/tests/test_translationtemplatesbuildmanager.py 2018-03-14 14:15:15 +0000
487@@ -82,7 +82,7 @@
488 'generate-translation-templates',
489 '--backend=chroot', '--series=xenial', '--arch=i386',
490 self.buildid,
491- url, 'resultarchive',
492+ '--branch', url, 'resultarchive',
493 ]
494 self.assertEqual(expected_command, self.buildmanager.commands[-1])
495 self.assertEqual(
496
497=== modified file 'lpbuildd/translationtemplates.py'
498--- lpbuildd/translationtemplates.py 2017-09-08 15:57:18 +0000
499+++ lpbuildd/translationtemplates.py 2018-03-14 14:15:15 +0000
500@@ -36,16 +36,28 @@
501
502 def initiate(self, files, chroot, extra_args):
503 """See `BuildManager`."""
504- self._branch_url = extra_args['branch_url']
505+ self.branch = extra_args.get('branch')
506+ # XXX cjwatson 2017-11-10: Backward-compatibility; remove once the
507+ # manager passes branch instead.
508+ if self.branch is None:
509+ self.branch = extra_args['branch_url']
510+ self.git_repository = extra_args.get("git_repository")
511+ self.git_path = extra_args.get("git_path")
512
513 super(TranslationTemplatesBuildManager, self).initiate(
514 files, chroot, extra_args)
515
516 def doGenerate(self):
517 """Generate templates."""
518- self.runTargetSubProcess(
519- "generate-translation-templates",
520- self._branch_url, self._resultname)
521+ args = []
522+ if self.branch is not None:
523+ args.extend(["--branch", self.branch])
524+ if self.git_repository is not None:
525+ args.extend(["--git-repository", self.git_repository])
526+ if self.git_path is not None:
527+ args.extend(["--git-path", self.git_path])
528+ args.append(self._resultname)
529+ self.runTargetSubProcess("generate-translation-templates", *args)
530
531 # Satisfy DebianPackageManager's needs without having a misleading
532 # method name here.

Subscribers

People subscribed via source and target branches

to all changes: