launchpad-buildd

Merge lp:~cjwatson/launchpad-buildd/translation-merge-states into lp:launchpad-buildd

  1. translation-merge-states
  2. Merge into trunk
Proposed by Colin Watson
Status: Merged
Merged at revision: 293
Proposed branch: lp:~cjwatson/launchpad-buildd/translation-merge-states
Merge into: lp:launchpad-buildd
Prerequisite: lp:~cjwatson/launchpad-buildd/translation-refactor-chdir
Diff against target: 266 lines (+62/-67)
4 files modified
bin/generate-translation-templates (+5/-2)
debian/changelog (+2/-0)
lpbuildd/tests/test_translationtemplatesbuildmanager.py (+31/-27)
lpbuildd/translationtemplates.py (+24/-38)
To merge this branch: bzr merge lp:~cjwatson/launchpad-buildd/translation-merge-states
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+330434@code.launchpad.net

Commit message

Merge TranslationTemplatesBuildState.{INSTALL,GENERATE} into a single state.

Description of the change

This will make it easier to turn generate-translation-templates into a subclass of Operation, and brings it more into line with buildlivefs and buildsnap.

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'bin/generate-translation-templates'
--- bin/generate-translation-templates 2017-07-25 22:11:19 +0000
+++ bin/generate-translation-templates 2017-09-08 14:41:25 +0000
@@ -1,6 +1,6 @@
1#!/bin/sh1#!/bin/sh
2#2#
3# Copyright 2010,2011 Canonical Ltd. This software is licensed under the3# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
4# GNU Affero General Public License version 3 (see the file LICENSE).4# GNU Affero General Public License version 3 (see the file LICENSE).
55
6# Buildd Slave tool to generate translation templates. Boiler plate code6# Buildd Slave tool to generate translation templates. Boiler plate code
@@ -53,6 +53,8 @@
53 $1 || echo "Got error $? from '$1'."53 $1 || echo "Got error $? from '$1'."
54}54}
5555
56$SUDO $CHROOT $BUILD_CHROOT apt-get install -y bzr intltool || exit 200
57
56# Copy pottery files to chroot.58# Copy pottery files to chroot.
57debug_exec "$SUDO $MKDIR -vp $BUILD_CHROOT$PYMODULES/$BUILDD_PACKAGE"59debug_exec "$SUDO $MKDIR -vp $BUILD_CHROOT$PYMODULES/$BUILDD_PACKAGE"
58debug_exec "$SUDO $TOUCH $BUILD_CHROOT$PYMODULES/$BUILDD_PACKAGE/__init__.py"60debug_exec "$SUDO $TOUCH $BUILD_CHROOT$PYMODULES/$BUILDD_PACKAGE/__init__.py"
@@ -63,4 +65,5 @@
63# Enter chroot, switch back to unprivileged user, execute the generate script.65# Enter chroot, switch back to unprivileged user, execute the generate script.
64$SUDO $CHROOT $BUILD_CHROOT \66$SUDO $CHROOT $BUILD_CHROOT \
65 $SU - $USER \67 $SU - $USER \
66 -c "PYTHONPATH=$PYMODULES $GENERATE_SCRIPT $BRANCH_URL $RESULT_NAME"68 -c "PYTHONPATH=$PYMODULES $GENERATE_SCRIPT $BRANCH_URL $RESULT_NAME" \
69 || exit 201
6770
=== modified file 'debian/changelog'
--- debian/changelog 2017-09-08 14:41:25 +0000
+++ debian/changelog 2017-09-08 14:41:25 +0000
@@ -1,6 +1,8 @@
1launchpad-buildd (152) UNRELEASED; urgency=medium1launchpad-buildd (152) UNRELEASED; urgency=medium
22
3 * Refactor lpbuildd.pottery.intltool to avoid calling chdir.3 * Refactor lpbuildd.pottery.intltool to avoid calling chdir.
4 * Merge TranslationTemplatesBuildState.{INSTALL,GENERATE} into a single
5 state.
46
5 -- Colin Watson <cjwatson@ubuntu.com> Fri, 08 Sep 2017 13:42:17 +01007 -- Colin Watson <cjwatson@ubuntu.com> Fri, 08 Sep 2017 13:42:17 +0100
68
79
=== modified file 'lpbuildd/tests/test_translationtemplatesbuildmanager.py'
--- lpbuildd/tests/test_translationtemplatesbuildmanager.py 2017-08-22 15:55:44 +0000
+++ lpbuildd/tests/test_translationtemplatesbuildmanager.py 2017-09-08 14:41:25 +0000
@@ -14,6 +14,8 @@
14from lpbuildd.tests.fakeslave import FakeSlave14from lpbuildd.tests.fakeslave import FakeSlave
15from lpbuildd.tests.matchers import HasWaitingFiles15from lpbuildd.tests.matchers import HasWaitingFiles
16from lpbuildd.translationtemplates import (16from lpbuildd.translationtemplates import (
17 RETCODE_FAILURE_BUILD,
18 RETCODE_FAILURE_INSTALL,
17 TranslationTemplatesBuildManager,19 TranslationTemplatesBuildManager,
18 TranslationTemplatesBuildState,20 TranslationTemplatesBuildState,
19 )21 )
@@ -65,23 +67,9 @@
65 self.buildmanager.backend_name = original_backend_name67 self.buildmanager.backend_name = original_backend_name
6668
67 # Skip states that are done in DebianBuildManager to the state69 # Skip states that are done in DebianBuildManager to the state
68 # directly before INSTALL.70 # directly before GENERATE.
69 self.buildmanager._state = TranslationTemplatesBuildState.UPDATE71 self.buildmanager._state = TranslationTemplatesBuildState.UPDATE
7072
71 # INSTALL: Install additional packages needed for this job into
72 # the chroot.
73 self.buildmanager.iterate(0)
74 self.assertEqual(
75 TranslationTemplatesBuildState.INSTALL, self.getState())
76 expected_command = [
77 '/usr/bin/sudo',
78 'sudo', 'chroot', self.chrootdir,
79 'apt-get',
80 ]
81 self.assertEqual(expected_command, self.buildmanager.commands[-1][:5])
82 self.assertEqual(
83 self.buildmanager.iterate, self.buildmanager.iterators[-1])
84
85 # GENERATE: Run the slave's payload, the script that generates73 # GENERATE: Run the slave's payload, the script that generates
86 # templates.74 # templates.
87 self.buildmanager.iterate(0)75 self.buildmanager.iterate(0)
@@ -136,34 +124,50 @@
136 self.buildmanager.iterate, self.buildmanager.iterators[-1])124 self.buildmanager.iterate, self.buildmanager.iterators[-1])
137 self.assertFalse(self.slave.wasCalled('buildFail'))125 self.assertFalse(self.slave.wasCalled('buildFail'))
138126
139 def test_iterate_fail_INSTALL(self):127 def test_iterate_fail_GENERATE_install(self):
140 # See that a failing INSTALL is handled properly.128 # See that a GENERATE that fails at the install step is handled
129 # properly.
141 url = 'lp:~my/branch'130 url = 'lp:~my/branch'
142 # The build manager's iterate() kicks off the consecutive states131 # The build manager's iterate() kicks off the consecutive states
143 # after INIT.132 # after INIT.
144 self.buildmanager.initiate(133 self.buildmanager.initiate(
145 {}, 'chroot.tar.gz', {'series': 'xenial', 'branch_url': url})134 {}, 'chroot.tar.gz', {'series': 'xenial', 'branch_url': url})
146135
147 # Skip states to the INSTALL state.136 # Skip states to the GENERATE state.
148 self.buildmanager._state = TranslationTemplatesBuildState.INSTALL137 self.buildmanager._state = TranslationTemplatesBuildState.GENERATE
149138
150 # The buildmanager fails and iterates to the UMOUNT state.139 # The buildmanager fails and reaps processes.
151 self.buildmanager.iterate(-1)140 self.buildmanager.iterate(RETCODE_FAILURE_INSTALL)
152 self.assertEqual(141 self.assertEqual(
153 TranslationTemplatesBuildState.UMOUNT, self.getState())142 TranslationTemplatesBuildState.GENERATE, self.getState())
154 expected_command = [143 expected_command = [
155 'sharepath/slavebin/in-target', 'in-target',144 'sharepath/slavebin/in-target', 'in-target',
156 'umount-chroot',145 'scan-for-processes',
157 '--backend=chroot', '--series=xenial', '--arch=i386',146 '--backend=chroot', '--series=xenial', '--arch=i386',
158 self.buildid,147 self.buildid,
159 ]148 ]
160 self.assertEqual(expected_command, self.buildmanager.commands[-1])149 self.assertEqual(expected_command, self.buildmanager.commands[-1])
161 self.assertEqual(150 self.assertNotEqual(
162 self.buildmanager.iterate, self.buildmanager.iterators[-1])151 self.buildmanager.iterate, self.buildmanager.iterators[-1])
163 self.assertTrue(self.slave.wasCalled('chrootFail'))152 self.assertTrue(self.slave.wasCalled('chrootFail'))
164153
165 def test_iterate_fail_GENERATE(self):154 # The buildmanager iterates to the UMOUNT state.
166 # See that a failing GENERATE is handled properly.155 self.buildmanager.iterateReap(self.getState(), 0)
156 self.assertEqual(
157 TranslationTemplatesBuildState.UMOUNT, self.getState())
158 expected_command = [
159 'sharepath/slavebin/in-target', 'in-target',
160 'umount-chroot',
161 '--backend=chroot', '--series=xenial', '--arch=i386',
162 self.buildid,
163 ]
164 self.assertEqual(expected_command, self.buildmanager.commands[-1])
165 self.assertEqual(
166 self.buildmanager.iterate, self.buildmanager.iterators[-1])
167
168 def test_iterate_fail_GENERATE_build(self):
169 # See that a GENERATE that fails at the build step is handled
170 # properly.
167 url = 'lp:~my/branch'171 url = 'lp:~my/branch'
168 # The build manager's iterate() kicks off the consecutive states172 # The build manager's iterate() kicks off the consecutive states
169 # after INIT.173 # after INIT.
@@ -174,7 +178,7 @@
174 self.buildmanager._state = TranslationTemplatesBuildState.GENERATE178 self.buildmanager._state = TranslationTemplatesBuildState.GENERATE
175179
176 # The buildmanager fails and reaps processes.180 # The buildmanager fails and reaps processes.
177 self.buildmanager.iterate(-1)181 self.buildmanager.iterate(RETCODE_FAILURE_BUILD)
178 expected_command = [182 expected_command = [
179 'sharepath/slavebin/in-target', 'in-target',183 'sharepath/slavebin/in-target', 'in-target',
180 'scan-for-processes',184 'scan-for-processes',
181185
=== modified file 'lpbuildd/translationtemplates.py'
--- lpbuildd/translationtemplates.py 2017-08-05 09:43:43 +0000
+++ lpbuildd/translationtemplates.py 2017-09-08 14:41:25 +0000
@@ -1,15 +1,21 @@
1# Copyright 2010 Canonical Ltd. This software is licensed under the1# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4__metaclass__ = type4__metaclass__ = type
55
6import os6import os
77
8from lpbuildd.debian import DebianBuildManager, DebianBuildState8from lpbuildd.debian import (
9 DebianBuildManager,
10 DebianBuildState,
11 )
12
13
14RETCODE_FAILURE_INSTALL = 200
15RETCODE_FAILURE_BUILD = 201
916
1017
11class TranslationTemplatesBuildState(DebianBuildState):18class TranslationTemplatesBuildState(DebianBuildState):
12 INSTALL = "INSTALL"
13 GENERATE = "GENERATE"19 GENERATE = "GENERATE"
1420
1521
@@ -21,7 +27,7 @@
21 runs on the build slave.27 runs on the build slave.
22 """28 """
2329
24 initial_build_state = TranslationTemplatesBuildState.INSTALL30 initial_build_state = TranslationTemplatesBuildState.GENERATE
2531
26 def __init__(self, slave, buildid):32 def __init__(self, slave, buildid):
27 super(TranslationTemplatesBuildManager, self).__init__(slave, buildid)33 super(TranslationTemplatesBuildManager, self).__init__(slave, buildid)
@@ -33,26 +39,10 @@
33 def initiate(self, files, chroot, extra_args):39 def initiate(self, files, chroot, extra_args):
34 """See `BuildManager`."""40 """See `BuildManager`."""
35 self._branch_url = extra_args['branch_url']41 self._branch_url = extra_args['branch_url']
36 self._chroot_path = os.path.join(
37 self.home, 'build-' + self._buildid, 'chroot-autobuild')
3842
39 super(TranslationTemplatesBuildManager, self).initiate(43 super(TranslationTemplatesBuildManager, self).initiate(
40 files, chroot, extra_args)44 files, chroot, extra_args)
4145
42 def doInstall(self):
43 """Install packages required."""
44 required_packages = [
45 'bzr',
46 'intltool',
47 ]
48 command = ['apt-get', 'install', '-y'] + required_packages
49 chroot = ['sudo', 'chroot', self._chroot_path]
50 self.runSubProcess('/usr/bin/sudo', chroot + command)
51
52 # To satisfy DebianPackageManagers needs without having a misleading
53 # method name here.
54 doRunBuild = doInstall
55
56 def doGenerate(self):46 def doGenerate(self):
57 """Generate templates."""47 """Generate templates."""
58 command = [48 command = [
@@ -60,37 +50,33 @@
60 self._buildid, self._branch_url, self._resultname]50 self._buildid, self._branch_url, self._resultname]
61 self.runSubProcess(self._generatepath, command)51 self.runSubProcess(self._generatepath, command)
6252
53 # Satisfy DebianPackageManager's needs without having a misleading
54 # method name here.
55 doRunBuild = doGenerate
56
63 def gatherResults(self):57 def gatherResults(self):
64 """Gather the results of the build and add them to the file cache."""58 """Gather the results of the build and add them to the file cache."""
65 # The file is inside the chroot, in the home directory of the buildd59 # The file is inside the target, in the home directory of the buildd
66 # user. Should be safe to assume the home dirs are named identically.60 # user. Should be safe to assume the home dirs are named identically.
67 path = os.path.join(self.home, self._resultname)61 path = os.path.join(self.home, self._resultname)
68 if self.backend.path_exists(path):62 if self.backend.path_exists(path):
69 self.addWaitingFileFromBackend(path)63 self.addWaitingFileFromBackend(path)
7064
71 def iterate_INSTALL(self, success):65 def iterate_GENERATE(self, retcode):
72 """Installation was done."""
73 if success == 0:
74 self._state = TranslationTemplatesBuildState.GENERATE
75 self.doGenerate()
76 else:
77 if not self.alreadyfailed:
78 self._slave.chrootFail()
79 self.alreadyfailed = True
80 self._state = TranslationTemplatesBuildState.UMOUNT
81 self.doUnmounting()
82
83 def iterate_GENERATE(self, success):
84 """Template generation finished."""66 """Template generation finished."""
85 if success == 0:67 if retcode == 0:
86 # It worked! Now let's bring in the harvest.68 # It worked! Now let's bring in the harvest.
87 self.gatherResults()69 self.gatherResults()
88 self.doReapProcesses(self._state)
89 else:70 else:
90 if not self.alreadyfailed:71 if not self.alreadyfailed:
91 self._slave.buildFail()72 if retcode == RETCODE_FAILURE_INSTALL:
73 self._slave.chrootFail()
74 elif retcode == RETCODE_FAILURE_BUILD:
75 self._slave.buildFail()
76 else:
77 self._slave.builderFail()
92 self.alreadyfailed = True78 self.alreadyfailed = True
93 self.doReapProcesses(self._state)79 self.doReapProcesses(self._state)
9480
95 def iterateReap_GENERATE(self, success):81 def iterateReap_GENERATE(self, success):
96 """Finished reaping after template generation."""82 """Finished reaping after template generation."""

Subscribers

People subscribed via source and target branches

to all changes: