Merge lp:~robru/cupstream2distro/parallelize-migration into lp:cupstream2distro
- parallelize-migration
- Merge into trunk
Proposed by
Robert Bruce Park
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Robert Bruce Park | ||||
Approved revision: | 1233 | ||||
Merged at revision: | 1227 | ||||
Proposed branch: | lp:~robru/cupstream2distro/parallelize-migration | ||||
Merge into: | lp:cupstream2distro | ||||
Diff against target: |
400 lines (+108/-134) 6 files modified
citrain/jenkins-templates/status.xml.tmpl (+31/-11) citrain/setup_citrain.py (+6/-2) citrain/status.py (+16/-30) files/config.xml (+22/-0) tests/unit/test_script_setup_citrain.py (+5/-2) tests/unit/test_script_status.py (+28/-89) |
||||
To merge this branch: | bzr merge lp:~robru/cupstream2distro/parallelize-migration | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Robert Bruce Park (community) | Approve | ||
PS Jenkins bot | continuous-integration | Approve | |
Review via email: mp+278121@code.launchpad.net |
Commit message
Parallelize migration script.
Description of the change
To post a comment you must log in.
Revision history for this message
Robert Bruce Park (robru) wrote : | # |
- 1232. By Robert Bruce Park
-
Run with 10 minute frequency.
Revision history for this message
PS Jenkins bot (ps-jenkins) wrote : | # |
PASSED: Continuous integration, rev:1232
http://
Executed test runs:
Click here to trigger a rebuild:
http://
review:
Approve
(continuous-integration)
- 1233. By Robert Bruce Park
-
Keep more logs but fewer artifacts.
Revision history for this message
PS Jenkins bot (ps-jenkins) wrote : | # |
PASSED: Continuous integration, rev:1233
http://
Executed test runs:
Click here to trigger a rebuild:
http://
review:
Approve
(continuous-integration)
Revision history for this message
Robert Bruce Park (robru) wrote : | # |
This loks amazing in staging.
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === renamed file 'citrain/jenkins-templates/check-publication-migration.xml.tmpl' => 'citrain/jenkins-templates/status.xml.tmpl' |
2 | --- citrain/jenkins-templates/check-publication-migration.xml.tmpl 2015-11-18 05:51:13 +0000 |
3 | +++ citrain/jenkins-templates/status.xml.tmpl 2015-11-20 11:31:16 +0000 |
4 | @@ -1,25 +1,25 @@ |
5 | <?xml version='1.0' encoding='UTF-8'?> |
6 | <project> |
7 | <actions/> |
8 | - <description>Automatically check destination migration for packages.</description> |
9 | + <description>Regularly refresh silo status.</description> |
10 | <logRotator> |
11 | - <daysToKeep>1</daysToKeep> |
12 | - <numToKeep>500</numToKeep> |
13 | + <daysToKeep>7</daysToKeep> |
14 | + <numToKeep>-1</numToKeep> |
15 | <artifactDaysToKeep>-1</artifactDaysToKeep> |
16 | - <artifactNumToKeep>-1</artifactNumToKeep> |
17 | + <artifactNumToKeep>5</artifactNumToKeep> |
18 | </logRotator> |
19 | <keepDependencies>false</keepDependencies> |
20 | <properties> |
21 | <hudson.security.AuthorizationMatrixProperty> |
22 | <permission>hudson.model.Item.Cancel:ubuntu-core-dev</permission> |
23 | <permission>hudson.model.Item.Cancel:canonical-ci-eng</permission> |
24 | - <permission>hudson.model.Item.Cancel:ci-train-users</permission> |
25 | + <permission>hudson.model.Item.Cancel:ci-train-ppa-service</permission> |
26 | <permission>hudson.model.Item.Build:ubuntu-core-dev</permission> |
27 | <permission>hudson.model.Item.Build:canonical-ci-eng</permission> |
28 | - <permission>hudson.model.Item.Build:ci-train-users</permission> |
29 | + <permission>hudson.model.Item.Build:ci-train-ppa-service</permission> |
30 | <permission>hudson.model.Item.Read:ubuntu-core-dev</permission> |
31 | <permission>hudson.model.Item.Read:canonical-ci-eng</permission> |
32 | - <permission>hudson.model.Item.Read:ci-train-users</permission> |
33 | + <permission>hudson.model.Item.Read:ci-train-ppa-service</permission> |
34 | </hudson.security.AuthorizationMatrixProperty> |
35 | <hudson.model.ParametersDefinitionProperty> |
36 | <parameterDefinitions> |
37 | @@ -38,7 +38,7 @@ |
38 | <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding> |
39 | <triggers> |
40 | <hudson.triggers.TimerTrigger> |
41 | - <spec>H/30 * * * *</spec> |
42 | + <spec>H/10 * * * *</spec> |
43 | </hudson.triggers.TimerTrigger> |
44 | </triggers> |
45 | <concurrentBuild>false</concurrentBuild> |
46 | @@ -46,15 +46,35 @@ |
47 | <hudson.tasks.Shell> |
48 | <command>#!/bin/sh |
49 | export LANG=en_US.UTF-8 |
50 | +export WORKSPACE="$PWD" |
51 | +export SILONAME="{SILO_NAME}" |
52 | export PYTHONPATH={LIBDIR} |
53 | |
54 | -cd # go to home directory |
55 | - |
56 | -{BINDIR}/migration.py |
57 | +[ "$DEBUG" = "true" ] && set -x |
58 | + |
59 | +[ -e {SILOS_DIR}/{SILO_NAME}/request_id_* ] || exit 0 |
60 | + |
61 | +rm -rf $WORKSPACE/* |
62 | + |
63 | +cd {SILOS_DIR}/{SILO_NAME} |
64 | + |
65 | +{BINDIR}/status.py |
66 | +RETVAL=$? |
67 | + |
68 | +cp *.diff "$WORKSPACE" 2>/dev/null || true |
69 | + |
70 | +exit $RETVAL |
71 | </command> |
72 | </hudson.tasks.Shell> |
73 | </builders> |
74 | <publishers> |
75 | + <hudson.tasks.ArtifactArchiver> |
76 | + <artifacts>*.diff</artifacts> |
77 | + <allowEmptyArchive>true</allowEmptyArchive> |
78 | + <onlyIfSuccessful>false</onlyIfSuccessful> |
79 | + <fingerprint>false</fingerprint> |
80 | + <defaultExcludes>true</defaultExcludes> |
81 | + </hudson.tasks.ArtifactArchiver> |
82 | </publishers> |
83 | <buildWrappers/> |
84 | </project> |
85 | |
86 | === modified file 'citrain/setup_citrain.py' |
87 | --- citrain/setup_citrain.py 2015-11-04 16:32:54 +0000 |
88 | +++ citrain/setup_citrain.py 2015-11-20 11:31:16 +0000 |
89 | @@ -101,7 +101,12 @@ |
90 | """Setup a silo jenkins job.""" |
91 | context = dict(SILOS_DIR=SILOS_DIR, SILO_NAME=siloname) |
92 | siloname = siloname.replace('/', '-') |
93 | - steps = dict(build='1', publish='2', autopkgtests='2.5', merge_clean='3') |
94 | + steps = dict( |
95 | + status='0', |
96 | + build='1', |
97 | + publish='2', |
98 | + autopkgtests='2.5', |
99 | + merge_clean='3') |
100 | for job, i in sorted(steps.items()): |
101 | job = job.replace('_', '-') |
102 | setup_job('{}-{}-{}'.format(siloname, i, job), template(job), context) |
103 | @@ -114,7 +119,6 @@ |
104 | setup_silo(siloname) |
105 | setup_job('cyphermox-test') |
106 | setup_job('prepare-silo') |
107 | - setup_job('check-publication-migration') |
108 | setup_job('staleness-report') |
109 | setup_job('upgrade-chroot') |
110 | setup_job('apt-get-clean') |
111 | |
112 | === renamed file 'citrain/migration.py' => 'citrain/status.py' |
113 | --- citrain/migration.py 2015-11-19 23:08:21 +0000 |
114 | +++ citrain/status.py 2015-11-20 11:31:16 +0000 |
115 | @@ -15,9 +15,9 @@ |
116 | # this program; if not, write to the Free Software Foundation, Inc., |
117 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
118 | |
119 | -"""CI Train Migration Script |
120 | +"""CI Train Status Script |
121 | |
122 | -Check if all silo packages have migrated, and if so, trigger merge & clean. |
123 | +Update silo status, also triggering Merge&Clean if silo is ready. |
124 | |
125 | Environment variables: |
126 | |
127 | @@ -30,36 +30,22 @@ |
128 | |
129 | from citrain.recipes.base import BuildBase |
130 | from citrain.recipes.manager import Manager |
131 | -from cupstream2distro.utils import env, run_script |
132 | -from cupstream2distro.silomanager import SiloState, stock_main |
133 | +from cupstream2distro.utils import run_script |
134 | +from cupstream2distro.silomanager import stock_main |
135 | from citrain.merge_clean import merge |
136 | |
137 | SUCCESS = re.compile(r'^((Release|Updates) pocket( \([^()]+\). ?)?)+$') |
138 | |
139 | |
140 | -def main(): |
141 | - """Execute the migration check, logging & saving any errors. |
142 | - |
143 | - :returns: 0. Always. |
144 | - """ |
145 | - for silo_state in SiloState.iterate(): |
146 | - env.SILONAME = silo_state |
147 | - BuildBase.failures.clear() |
148 | - logging.info('\n\nInspecting silo %s:', silo_state.ppa.web_link) |
149 | - try: |
150 | - silo_state.enforce_lock() |
151 | - silo_state.lock_fd.close() |
152 | - except BlockingIOError: |
153 | - logging.info('Silo %s is in use, skipping.', env.SILONAME) |
154 | - continue |
155 | - |
156 | - status = silo_state.status = Manager(silo_state).get_states() |
157 | - if SUCCESS.match(status): |
158 | - logging.info('Looks good, proceeding with merge & clean.') |
159 | - stock_main(merge) |
160 | - continue |
161 | - silo_state.save_config() |
162 | - return 0 |
163 | - |
164 | - |
165 | -run_script(__name__, __doc__, main) |
166 | +def update_status(silo_state): |
167 | + """Set the status for this silo.""" |
168 | + silo_state.lock_fd.close() |
169 | + BuildBase.failures.clear() |
170 | + logging.info('Inspecting PPA %s:', silo_state.ppa.web_link) |
171 | + status = silo_state.status = Manager(silo_state).get_states() |
172 | + if SUCCESS.match(status): |
173 | + logging.info('Looks good, proceeding with merge & clean.') |
174 | + stock_main(merge) |
175 | + |
176 | + |
177 | +run_script(__name__, __doc__, lambda: stock_main(update_status)) |
178 | |
179 | === modified file 'files/config.xml' |
180 | --- files/config.xml 2015-10-28 23:28:38 +0000 |
181 | +++ files/config.xml 2015-11-20 11:31:16 +0000 |
182 | @@ -202,6 +202,28 @@ |
183 | </hudson.model.AllView> |
184 | <listView> |
185 | <owner class="hudson" reference="../../.."/> |
186 | + <name>0. Status</name> |
187 | + <filterExecutors>false</filterExecutors> |
188 | + <filterQueue>false</filterQueue> |
189 | + <properties class="hudson.model.View$PropertyList"/> |
190 | + <jobNames> |
191 | + <comparator class="hudson.util.CaseInsensitiveComparator"/> |
192 | + </jobNames> |
193 | + <jobFilters/> |
194 | + <columns> |
195 | + <hudson.views.StatusColumn/> |
196 | + <hudson.views.WeatherColumn/> |
197 | + <hudson.views.JobColumn/> |
198 | + <hudson.views.LastSuccessColumn/> |
199 | + <hudson.views.LastFailureColumn/> |
200 | + <hudson.views.LastDurationColumn/> |
201 | + <hudson.views.BuildButtonColumn/> |
202 | + </columns> |
203 | + <includeRegex>.*-0-status</includeRegex> |
204 | + <recurse>false</recurse> |
205 | + </listView> |
206 | + <listView> |
207 | + <owner class="hudson" reference="../../.."/> |
208 | <name>1. Build</name> |
209 | <filterExecutors>false</filterExecutors> |
210 | <filterQueue>false</filterQueue> |
211 | |
212 | === modified file 'tests/unit/test_script_setup_citrain.py' |
213 | --- tests/unit/test_script_setup_citrain.py 2015-11-04 16:32:54 +0000 |
214 | +++ tests/unit/test_script_setup_citrain.py 2015-11-20 11:31:16 +0000 |
215 | @@ -123,6 +123,10 @@ |
216 | 'publish.xml.tmpl', |
217 | {'SILO_NAME': 'ubuntu/landing-000', |
218 | 'SILOS_DIR': os.path.expanduser('~/silos')}), |
219 | + call('ubuntu-landing-000-0-status', |
220 | + 'status.xml.tmpl', |
221 | + {'SILO_NAME': 'ubuntu/landing-000', |
222 | + 'SILOS_DIR': os.path.expanduser('~/silos')}), |
223 | ]) |
224 | |
225 | def test_main(self): |
226 | @@ -137,7 +141,6 @@ |
227 | self.assertEqual(self.script.setup_job.mock_calls, [ |
228 | call('cyphermox-test'), |
229 | call('prepare-silo'), |
230 | - call('check-publication-migration'), |
231 | call('staleness-report'), |
232 | call('upgrade-chroot'), |
233 | call('apt-get-clean'), |
234 | @@ -152,11 +155,11 @@ |
235 | 'abandon/config.xml', |
236 | 'apt-get-clean/config.xml', |
237 | 'cyphermox-test/config.xml', |
238 | - 'check-publication-migration/config.xml', |
239 | 'pbuilder-clean/config.xml', |
240 | 'prepare-silo/config.xml', |
241 | 'revert/config.xml', |
242 | 'staleness-report/config.xml', |
243 | + 'ubuntu-landing-{:03d}-0-status/config.xml', |
244 | 'ubuntu-landing-{:03d}-1-build/config.xml', |
245 | 'ubuntu-landing-{:03d}-2-publish/config.xml', |
246 | 'ubuntu-landing-{:03d}-2.5-autopkgtests/config.xml', |
247 | |
248 | === renamed file 'tests/unit/test_script_migration.py' => 'tests/unit/test_script_status.py' |
249 | --- tests/unit/test_script_migration.py 2015-11-19 23:08:21 +0000 |
250 | +++ tests/unit/test_script_status.py 2015-11-20 11:31:16 +0000 |
251 | @@ -14,31 +14,25 @@ |
252 | # this program; if not, write to the Free Software Foundation, Inc., |
253 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
254 | |
255 | -"""Tests for CI Train Migration script.""" |
256 | +"""Tests for CI Train Status script.""" |
257 | |
258 | -from os.path import join |
259 | from mock import Mock, call |
260 | |
261 | from tests.unit import CITrainScriptTestCase |
262 | |
263 | -from cupstream2distro.settings import SILOS_DIR |
264 | - |
265 | # Unused import necessary for code coverage reporting |
266 | -from citrain import migration |
267 | -COVERAGE = [migration] |
268 | - |
269 | - |
270 | -class MigrationTestCase(CITrainScriptTestCase): |
271 | - """Tests for CI Train Migration script.""" |
272 | - scriptname = 'migration.py' |
273 | +from citrain import status |
274 | +COVERAGE = [status] |
275 | + |
276 | + |
277 | +class StatusTestCase(CITrainScriptTestCase): |
278 | + """Tests for CI Train Status script.""" |
279 | + scriptname = 'status.py' |
280 | |
281 | def setUp(self): |
282 | super().setUp() |
283 | self.script.Manager = Mock() |
284 | self.script.stock_main = Mock() |
285 | - self.script.glob.return_value = [ |
286 | - join(SILOS_DIR, 'ubuntu', 'landing-00{}'.format(x)) |
287 | - for x in range(3)] |
288 | |
289 | def test_success_regex(self): |
290 | """Ensure we can properly recognize silos ready to auto-merge.""" |
291 | @@ -60,89 +54,34 @@ |
292 | results = [bool(self.script.SUCCESS.match(stat)) for stat in bad] |
293 | self.assertEqual(results, [False] * len(bad)) |
294 | |
295 | - def test_main(self): |
296 | + def test_status(self): |
297 | """Trigger merges after successful migrations.""" |
298 | tokenize = lambda: Mock(return_value={'hi'}) |
299 | - states = self.script.SiloState.iterate.return_value = [Mock( |
300 | - tokenize=tokenize(), |
301 | - is_published=True, |
302 | - is_autopkgtesting=False) for rid in range(3)] |
303 | + silo_state = Mock(tokenize=tokenize()) |
304 | mgr = self.script.Manager.return_value |
305 | mgr.get_states.return_value = 'Release pocket (foo).' |
306 | - self.assertEqual(self.script.main(), 0) |
307 | - for state in states: |
308 | - self.assertEqual(state.mock_calls, [ |
309 | - call.enforce_lock(), |
310 | - call.lock_fd.close(), |
311 | - ]) |
312 | + self.assertIsNone(self.script.update_status(silo_state)) |
313 | + self.assertEqual(silo_state.mock_calls, [ |
314 | + call.lock_fd.close(), |
315 | + ]) |
316 | self.assertEqual(self.script.Manager.mock_calls, [ |
317 | - call(states[0]), |
318 | - call().get_states(), |
319 | - call(states[1]), |
320 | - call().get_states(), |
321 | - call(states[2]), |
322 | + call(silo_state), |
323 | call().get_states(), |
324 | ]) |
325 | self.assertEqual(self.script.stock_main.mock_calls, [ |
326 | call(self.script.merge) |
327 | - ] * 3) |
328 | - |
329 | - def test_main_skip_empty_silos(self): |
330 | - """Don't check silos that are empty.""" |
331 | - self.script.SiloState.iterate.return_value = [] |
332 | - self.assertEqual(self.script.main(), 0) |
333 | - self.assertEqual(self.script.Manager.mock_calls, []) |
334 | - self.assertEqual(self.script.stock_main.mock_calls, []) |
335 | - |
336 | - def test_main_skip_locked_silos(self): |
337 | - """Skip over silos that are currently used by other processes.""" |
338 | - states = self.script.SiloState.iterate.return_value = [Mock()] |
339 | - states[0].enforce_lock.side_effect = BlockingIOError |
340 | - self.assertEqual(self.script.main(), 0) |
341 | - self.assertEqual(self.script.Manager.mock_calls, []) |
342 | - self.assertEqual(self.script.stock_main.mock_calls, []) |
343 | - self.assertEqual(states[0].mock_calls, [call.enforce_lock()]) |
344 | - |
345 | - def test_main_skip_unpublished_silos(self): |
346 | + ]) |
347 | + |
348 | + def test_status_skip_unpublished_silos(self): |
349 | """Don't merge silos that are unpublished.""" |
350 | tokenize = Mock(return_value={'hi'}) |
351 | - states = self.script.SiloState.iterate.return_value = [ |
352 | - Mock(tokenize=tokenize, is_published=False) for rid in range(3)] |
353 | - mgr = self.script.Manager.return_value |
354 | - mgr.get_states.return_value = 'Proposed pocket (foo).' |
355 | - self.assertEqual(self.script.main(), 0) |
356 | - self.assertEqual(self.script.Manager.mock_calls, [ |
357 | - call(states[0]), |
358 | - call().get_states(), |
359 | - call(states[1]), |
360 | - call().get_states(), |
361 | - call(states[2]), |
362 | - call().get_states(), |
363 | - ]) |
364 | - self.assertEqual(self.script.stock_main.mock_calls, []) |
365 | - for state in states: |
366 | - self.assertEqual(state.set_migrating.mock_calls, []) |
367 | - self.assertEqual(state.save_config.mock_calls, [call()]) |
368 | - |
369 | - def test_main_dont_merge_migrating_silos(self): |
370 | - """Don't merge silos that are still migrating.""" |
371 | - tokenize = Mock(return_value={'hi'}) |
372 | - states = self.script.SiloState.iterate.return_value = [Mock( |
373 | - tokenize=tokenize, |
374 | - is_published=True, |
375 | - requestid=str(1), |
376 | - )] |
377 | - mgr = self.script.Manager.return_value |
378 | - mgr.get_states.return_value = 'Proposed pocket (foo).' |
379 | - self.assertEqual(self.script.main(), 0) |
380 | - self.assertEqual(states[0].status, mgr.get_states.return_value) |
381 | - self.assertEqual(states[0].mock_calls, [ |
382 | - call.enforce_lock(), |
383 | - call.lock_fd.close(), |
384 | - call.save_config(), |
385 | - ]) |
386 | - self.assertEqual(self.script.Manager.mock_calls, [ |
387 | - call(states[0]), |
388 | - call().get_states(), |
389 | - ]) |
390 | - self.assertEqual(self.script.stock_main.mock_calls, []) |
391 | + silo_state = Mock(tokenize=tokenize) |
392 | + mgr = self.script.Manager.return_value |
393 | + mgr.get_states.return_value = 'Proposed pocket (foo).' |
394 | + self.assertIsNone(self.script.update_status(silo_state)) |
395 | + self.assertEqual(self.script.Manager.mock_calls, [ |
396 | + call(silo_state), |
397 | + call().get_states(), |
398 | + ]) |
399 | + self.assertEqual(self.script.stock_main.mock_calls, []) |
400 | + self.assertEqual(silo_state.save_config.mock_calls, []) |
https:/ /ci-train. staging. ubuntu. com/view/ 0.%20Status/
I'm enjoying this in staging, just need to write tests.