Merge lp:~openerp-dev/openerp-tools/trunk-configured-branches-vmt into lp:openerp-tools
- trunk-configured-branches-vmt
- Merge into trunk
Proposed by
Vo Minh Thu
Status: | Merged |
---|---|
Merged at revision: | 197 |
Proposed branch: | lp:~openerp-dev/openerp-tools/trunk-configured-branches-vmt |
Merge into: | lp:openerp-tools |
Diff against target: |
1776 lines (+684/-606) 13 files modified
openerp-runbot/openerp-runbot (+6/-1) openerp-runbot/openerprunbot/__init__.py (+1/-0) openerp-runbot/openerprunbot/core.py (+154/-578) openerp-runbot/openerprunbot/jobs/__init__.py (+7/-0) openerp-runbot/openerprunbot/jobs/install_all.py (+366/-0) openerp-runbot/openerprunbot/jobs/migrate_script.py (+61/-0) openerp-runbot/openerprunbot/misc.py (+54/-1) openerp-runbot/openerprunbot/templates/branches.html.mako (+2/-1) openerp-runbot/openerprunbot/templates/defs.html.mako (+24/-18) openerp-runbot/openerprunbot/templates/nginx.conf.mako (+1/-1) openerp-runbot/starting.html (+1/-0) openerp-runbot/stopped.html (+1/-0) openerp-runbot/try-template.py (+6/-6) |
To merge this branch: | bzr merge lp:~openerp-dev/openerp-tools/trunk-configured-branches-vmt |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
OpenERP R&D Team | Pending | ||
Review via email: mp+113346@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
- 201. By OpenERP Online
-
[FIX] runbot: the completing branches were used in place of the original ones.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'openerp-runbot/openerp-runbot' |
2 | --- openerp-runbot/openerp-runbot 2012-04-19 15:34:05 +0000 |
3 | +++ openerp-runbot/openerp-runbot 2012-07-04 11:46:17 +0000 |
4 | @@ -57,6 +57,7 @@ |
5 | parser.add_option("--test", metavar="INT", default=1, help="run tests flag (%default)") |
6 | parser.add_option("--workers", metavar="INT", default=4, help="number of workers (%default)") |
7 | parser.add_option("--start-job-id", metavar="INT", default=0, help="initial job id (%default)") |
8 | + parser.add_option("--job-type", metavar="JOB_TYPE", default='install_all', help="name of the default job type (%default)") |
9 | parser.add_option("--debug", action="store_true", help="ease debugging by e.g. limiting branches") |
10 | o, a = parser.parse_args(sys.argv) |
11 | if o.init: |
12 | @@ -64,6 +65,9 @@ |
13 | elif o.clean: |
14 | runbot_clean(o.dir) |
15 | elif o.run: |
16 | + assert o.job_type in openerprunbot.jobs.JOBS |
17 | + path = os.path.dirname(sys.modules['__main__'].__file__) |
18 | + openerprunbot.core.run(["cp", os.path.join(path, "starting.html"), "static/index.html"]) |
19 | openerprunbot.server.read_state() |
20 | openerprunbot.server.start_server(int(o.nginx_port)-1) |
21 | server_net_port = int(o.nginx_port) + int(o.number) * 2 |
22 | @@ -76,10 +80,11 @@ |
23 | r = openerprunbot.core.RunBot(o.dir, o.poll, server_net_port, |
24 | server_xml_port, client_web_port, o.number, o.nginx_port, |
25 | o.nginx_domain, o.test, o.workers, int(o.start_job_id), |
26 | - o.debug, lp) |
27 | + o.debug, lp, o.job_type) |
28 | runbot_kill_msg() |
29 | r.loop() |
30 | runbot_kill_msg() |
31 | + openerprunbot.core.run(["cp", os.path.join(path, "stopped.html"), "static/index.html"]) |
32 | print "Last used job id:", r.current_job_id |
33 | else: |
34 | parser.print_help() |
35 | |
36 | === modified file 'openerp-runbot/openerprunbot/__init__.py' |
37 | --- openerp-runbot/openerprunbot/__init__.py 2011-12-06 23:17:14 +0000 |
38 | +++ openerp-runbot/openerprunbot/__init__.py 2012-07-04 11:46:17 +0000 |
39 | @@ -5,6 +5,7 @@ |
40 | import re |
41 | |
42 | from . import core |
43 | +from . import jobs |
44 | from . import launchpad |
45 | from . import misc |
46 | from . import server |
47 | |
48 | === modified file 'openerp-runbot/openerprunbot/core.py' |
49 | --- openerp-runbot/openerprunbot/core.py 2012-05-07 07:21:32 +0000 |
50 | +++ openerp-runbot/openerprunbot/core.py 2012-07-04 11:46:17 +0000 |
51 | @@ -16,6 +16,7 @@ |
52 | |
53 | import openerprunbot |
54 | from openerprunbot.misc import * |
55 | +from openerprunbot.jobs.install_all import InstallAllJob |
56 | |
57 | # Hard-coded branches we always want, used to complement monitored branches |
58 | # (i.e. to create branch groups). |
59 | @@ -31,67 +32,36 @@ |
60 | 'web_trunk': '~openerp/openerp-web/trunk', |
61 | } |
62 | |
63 | +migration_scripts_branch = None |
64 | +migration_platform_branch = None |
65 | + |
66 | # Number of build 'slots' per branch group. |
67 | POINTS = 5 |
68 | |
69 | +# Job states |
70 | +STATE_ALLOCATED = 'allocated' |
71 | +STATE_BROKEN = 'broken' |
72 | +STATE_PULLING = 'pulling' |
73 | +STATE_RUNNING = 'running' |
74 | +STATE_TESTING = 'testing' |
75 | + |
76 | # This constant matches community_dashboard.py. |
77 | NEW_MERGE_STATUS = ['Needs review', 'Code failed to merge', 'Approved'] |
78 | |
79 | #---------------------------------------------------------- |
80 | -# OpenERP RunBot misc |
81 | -#---------------------------------------------------------- |
82 | - |
83 | -def underscore(s): |
84 | - return s.replace("~","").replace(":","_").replace("/","_").replace(".","_").replace(" ","_") |
85 | - |
86 | -def get_committer_info(repo_path): |
87 | - committer_name = None |
88 | - committer_xgram = None |
89 | - committer_email = None |
90 | - |
91 | - output = run_output(["bzr", "log", "--long", "-r-1"], cwd=repo_path) |
92 | - |
93 | - committer_re = re.compile('committer: *(.+)<(.+)@(.+)>') |
94 | - for i in output.split('\n'): |
95 | - m = committer_re.match(i) |
96 | - if m: |
97 | - committer_name = m.group(1).strip() |
98 | - committer_xgram = m.group(2) |
99 | - committer_email = m.group(2) + '@' + m.group(3) |
100 | - break |
101 | - |
102 | - return committer_name, committer_xgram, committer_email |
103 | - |
104 | -def has_test_enable_flag(server_bin_path): |
105 | - """ |
106 | - Test whether an openerp-server executable has the --test-enable flag. |
107 | - (When the flag is present, testing for success/failure is done in a |
108 | - different way.) |
109 | - """ |
110 | - p1 = subprocess.Popen([server_bin_path, "--help"], |
111 | - stdout=subprocess.PIPE) |
112 | - p2 = subprocess.Popen(["grep", "test-enable"], stdin=p1.stdout, |
113 | - stdout=subprocess.PIPE) |
114 | - p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. |
115 | - output = p2.communicate()[0] |
116 | - return output == " --test-enable Enable YAML and unit tests.\n" |
117 | - |
118 | -#---------------------------------------------------------- |
119 | # OpenERP RunBot Branch |
120 | #---------------------------------------------------------- |
121 | |
122 | class RunBotBranch(object): |
123 | def __init__(self, runbot, branch, group=None, |
124 | - overriden_repo_path=None, overriden_project_name=None): |
125 | + repo_path=None, project_name=None, trigger_build=False): |
126 | """ |
127 | Represent a single Launchpad branch, e.g. |
128 | `~openerp-dev/openobject-server/trunk-foo-bar`. |
129 | - |
130 | - A branch has an overriden_repo_path when it is a trunk or 6.{0,1} branch |
131 | - used to complement another 'real' branch. |
132 | """ |
133 | self.runbot = runbot |
134 | self.group = group |
135 | + self.trigger_build = trigger_build |
136 | |
137 | # e.g. trunk |
138 | self.name = branch.name |
139 | @@ -100,13 +70,10 @@ |
140 | self.unique_name = branch.unique_name |
141 | self.unique_name_underscore = underscore(self.unique_name) |
142 | # server, addons, or web (both for openobject-client-web and openerp-web) |
143 | - if overriden_project_name: |
144 | - self.project_name = overriden_project_name |
145 | + if project_name: |
146 | + self.project_name = project_name |
147 | else: |
148 | - self.project_name = re.search("/(openobject|openerp)-(addons|server|client-web|web)/",self.unique_name).group(2) |
149 | - if self.project_name == 'client-web': |
150 | - self.project_name = 'web' |
151 | - assert self.project_name in ('server', 'addons', 'web') |
152 | + self.project_name = project_name_from_unique_name(self.unique_name) |
153 | |
154 | self.lp_date_last_modified = branch.date_last_modified |
155 | self.lp_revision_count = branch.revision_count |
156 | @@ -116,10 +83,10 @@ |
157 | self.merge_count = 0 |
158 | |
159 | # Repository path <Root>/repo/<branchuniquename> |
160 | - self.repo_path=os.path.join(self.runbot.wd,'repo',self.unique_name_underscore) |
161 | - self.overriden_repo_path = overriden_repo_path |
162 | - if overriden_repo_path: |
163 | - self.repo_path=overriden_repo_path |
164 | + if repo_path: |
165 | + self.repo_path=repo_path |
166 | + else: |
167 | + self.repo_path=os.path.join(self.runbot.wd,'repo',self.unique_name_underscore) |
168 | |
169 | self.committer_name = None |
170 | self.committer_xgram = None |
171 | @@ -134,7 +101,7 @@ |
172 | if self.local_revision_count != self.lp_revision_count: |
173 | self.local_date_last_modified = self.lp_date_last_modified |
174 | self.local_revision_count = self.lp_revision_count |
175 | - if not self.overriden_repo_path or self.group.is_configured(): |
176 | + if self.trigger_build: |
177 | name = self.project_name |
178 | self.group.need_run_reason.append(name) |
179 | |
180 | @@ -142,50 +109,87 @@ |
181 | # OpenERP RunBot Grouped Branch |
182 | #---------------------------------------------------------- |
183 | |
184 | -class RunBotGroupedBranchBase(object): |
185 | - def __init__(self, runbot, sticky): |
186 | +class RunBotGroupedBranch(object): |
187 | + """ |
188 | + A single 'branch group' represents a single branch name for all our |
189 | + projects. E.g. trunk is available in server, addons, web, and client-web. |
190 | + |
191 | + When the corresponding project doesn't have that branch, trunk is used. |
192 | + For instance, a branch group can represent |
193 | + `~openerp-dev/openobject-server/trunk-foo-bar` |
194 | + `~openerp-dev/openobject-addons/trunk-foo-bar` |
195 | + and |
196 | + `~openerp/openobject-web-client/6.0` |
197 | + `~openerp/openerp-web/trunk` |
198 | + will be used to complete the group, building the four branches together. |
199 | + |
200 | + Alternatively a 'branch group' can be configured. In that case, all |
201 | + branches are manually specified; no attempt is done to complete the group. |
202 | + """ |
203 | + def __init__(self, runbot, team_name, name, version, sticky, |
204 | + server_branch=None, client_web_branch=None, |
205 | + web_branch=None, addons_branches=None, modules=None, job_type=None): |
206 | self.runbot = runbot |
207 | self.sticky = sticky |
208 | + self.job_type = job_type |
209 | + |
210 | # Points are build 'slots' |
211 | self.points = [None for x in xrange(POINTS)] |
212 | + |
213 | + self.team_name = team_name |
214 | + self.name = name # group name or manually chosen name. |
215 | + self.name_underscore = underscore(self.name) |
216 | + self.version = version # '6.0', '6.1', or 'trunk' |
217 | + |
218 | # Reason to do a build, e.g. because 'addons' was commited to. |
219 | self.need_run_reason = [] |
220 | - self.version = None |
221 | + |
222 | + self.server_branch = server_branch |
223 | + self.web_branch = client_web_branch or web_branch |
224 | + self.addons_branches = addons_branches |
225 | + |
226 | + self.server = None |
227 | + self.web = None |
228 | + self.addons = [] |
229 | + |
230 | + self.json_path=os.path.join(runbot.wd,'static',"%s-%s.json"%(team_name, self.name.replace('_','-').replace('.','-'))) |
231 | + |
232 | + # List of modules to install instead of `all`. |
233 | + self.modules = modules.split(',') if modules else [] |
234 | + |
235 | + self.configured = False |
236 | + |
237 | + # A normal build_number is maxint, a build_number is assigned when a |
238 | + # build is manually requested or a triggering branch has a new revision. |
239 | + # The build number is used to assign a position in the queue. |
240 | + # The value will be reset to maxint after the build is done. |
241 | + self.build_number = sys.maxint |
242 | + |
243 | # True when it is not possible to provide default branches based on |
244 | # the name. |
245 | self.wrong_matching = False |
246 | |
247 | - # A normal manual_build is maxint, a manual_build is used only when |
248 | - # manually requesting a build with a 'build' command pushed in the |
249 | - # queue. The value will be reset to maxint after the build is done. |
250 | - self.manual_build = sys.maxint |
251 | - |
252 | - # List of modules to install instead of `all`. |
253 | - self.modules = [] |
254 | - |
255 | def add_point(self, j): |
256 | - assert not j.completed |
257 | - p = Point(self, j) |
258 | - self.points = self.points[1:] + [p] # TODO delete db and on-disk data |
259 | + assert j.state == STATE_ALLOCATED |
260 | + self.points = self.points[1:] + [j] # TODO delete db and on-disk data |
261 | |
262 | def complete_point(self, j): |
263 | - assert j.completed |
264 | + assert j.state in (STATE_RUNNING, STATE_BROKEN) |
265 | for p in self.points + [None]: |
266 | if p and p.job_id == j.job_id: |
267 | break |
268 | - if p and p.state != 'running': |
269 | - p.update(j) |
270 | + if p and p.state != j.state: |
271 | p.save_json() |
272 | |
273 | def all_points_completed(self): |
274 | for p in self.points: |
275 | - if p is not None and p.state != 'running': |
276 | + if p is not None and p.state not in (STATE_RUNNING, STATE_BROKEN): |
277 | return False |
278 | return True |
279 | |
280 | def is_sticky_job(self, j): |
281 | """ Return True if the job is the latest running point of a sticky branch. """ |
282 | - runnings = filter(lambda x: x and x.running_t0, self.points) |
283 | + runnings = filter(lambda x: x and x.state == STATE_RUNNING, self.points) |
284 | return self.sticky and runnings and runnings[-1].job_id == j.job_id |
285 | |
286 | def update_state(self, state): |
287 | @@ -198,43 +202,33 @@ |
288 | if self.sticky != previous: |
289 | log("stickiness changed", team=self.team_name, branch=self.name, now=self.sticky) |
290 | |
291 | - def is_configured(self): |
292 | - return isinstance(self, ConfiguredGroup) |
293 | - |
294 | -class ConfiguredGroup(RunBotGroupedBranchBase): |
295 | - def __init__(self, runbot, team_name, name, version, sticky, |
296 | - server_branch=None, client_web_branch=None, |
297 | - web_branch=None, addons_branches=None, modules=None): |
298 | - super(ConfiguredGroup, self).__init__(runbot, sticky) |
299 | - self.team_name = team_name |
300 | - self.name = name # Unique manually-chosen name, not necessarily the group name. |
301 | - self.name_underscore = underscore(self.name) |
302 | - self.version = version # '6.0', '6.1', or 'trunk' |
303 | - |
304 | - self.server_branch = server_branch |
305 | - self.web_branch = client_web_branch or web_branch |
306 | - self.addons_branches = addons_branches |
307 | - |
308 | - self.server = None |
309 | - self.web = None |
310 | - self.addons = [] |
311 | - |
312 | - self.json_path=os.path.join(runbot.wd,'static',"%s-%s.json"%(team_name, self.name.replace('_','-').replace('.','-'))) |
313 | - |
314 | - self.modules = modules.split(',') if modules else [] |
315 | + def repo_updates(self): |
316 | + return [copy.copy(x) for x in self.all_branches()] |
317 | + |
318 | + def all_branches(self): |
319 | + r = [] |
320 | + r.append(self.server) |
321 | + r.extend([x for x in self.addons]) |
322 | + r.append(self.web) |
323 | + if hasattr(self, 'migration_script'): |
324 | + r.extend([self.migration_script, self.migration_platform]) |
325 | + return r |
326 | + |
327 | + def lp_date_last_modified(self): |
328 | + return max(p.lp_date_last_modified for p in self.all_branches() if p and p.trigger_build) |
329 | |
330 | def is_ok(self): |
331 | """ |
332 | - Test whether the group is useable, i.e. if self.populate_branches() |
333 | + Test whether the group is useable, i.e. if self.configure_branches() |
334 | didn't early return. |
335 | """ |
336 | - if self.server and self.web and self.addons: |
337 | - return True |
338 | - else: |
339 | - return False |
340 | + return self.server and self.web and self.addons |
341 | |
342 | - def populate_branches(self, launchpad): |
343 | + # TODO all the xxxx_branch argument in __init__ can be moved here. |
344 | + def configure_branches(self, launchpad): |
345 | + """ Fetch the configured branches. """ |
346 | log("runbot-populate-configured-branches") |
347 | + self.configured = True |
348 | repo = os.path.join(self.runbot.wd, 'repo') |
349 | filters = {'status': NEW_MERGE_STATUS} |
350 | |
351 | @@ -243,7 +237,7 @@ |
352 | if not b: |
353 | log("WARNING:no such unique name", name=self.server_branch) |
354 | return |
355 | - self.server = RunBotBranch(self.runbot, b, group=self, overriden_repo_path=path, overriden_project_name='server') |
356 | + self.server = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='server', trigger_build=True) |
357 | self.server.update_launchpad(b) |
358 | self.server.merge_count = len(list(b.getMergeProposals(**filters))) |
359 | |
360 | @@ -253,7 +247,7 @@ |
361 | if not b: |
362 | log("WARNING:no such unique name", name=self.web_branch) |
363 | return |
364 | - self.web = RunBotBranch(self.runbot, b, group=self, overriden_repo_path=path, overriden_project_name='web') |
365 | + self.web = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='web', trigger_build=True) |
366 | self.web.update_launchpad(b) |
367 | self.web.merge_count = len(list(b.getMergeProposals(**filters))) |
368 | else: |
369 | @@ -267,59 +261,13 @@ |
370 | log("WARNING:no such unique name", name=self.addons_branches[x]) |
371 | self.addons = [] |
372 | return |
373 | - bb = RunBotBranch(self.runbot, b, group=self, overriden_repo_path=path, overriden_project_name='addons') |
374 | + bb = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='addons', trigger_build=True) |
375 | bb.update_launchpad(b) |
376 | bb.merge_count = len(list(b.getMergeProposals(**filters))) |
377 | self.addons.append(bb) |
378 | |
379 | - def repo_updates(self): |
380 | - r = [] |
381 | - r.append(copy.copy(self.server)) |
382 | - for x in self.addons: |
383 | - r.append(copy.copy(x)) |
384 | - r.append(copy.copy(self.web)) |
385 | - return r |
386 | - |
387 | - def lp_date_last_modified(self): |
388 | - lp_date_last_modified = None |
389 | - for p in [self.server, self.web] + self.addons: |
390 | - if p and (not lp_date_last_modified or p.lp_date_last_modified > lp_date_last_modified): |
391 | - lp_date_last_modified = p.lp_date_last_modified |
392 | - assert lp_date_last_modified # at least one project in the group |
393 | - return lp_date_last_modified |
394 | - |
395 | -class RunBotGroupedBranch(RunBotGroupedBranchBase): |
396 | - """ |
397 | - A single 'branch group' represents a single branch name for all our |
398 | - projects. E.g. trunk is available in server, addons, web, and client-web. |
399 | - |
400 | - When the corresponding project doesn't have that branch, trunk is used. |
401 | - For instance, a branch group can represent |
402 | - `~openerp-dev/openobject-server/trunk-foo-bar` |
403 | - `~openerp-dev/openobject-addons/trunk-foo-bar` |
404 | - and |
405 | - `~openerp/openobject-web-client/6.0` |
406 | - `~openerp/openerp-web/trunk` |
407 | - will be used to complete the group, building the four branches together. |
408 | - |
409 | - Alternatively a 'branch group' can be configured. In that case, all |
410 | - branches are manually specified; no attempt is done to complete the group. |
411 | - """ |
412 | - |
413 | - def __init__(self, runbot, b, team_name, sticky): |
414 | - """A grouped branch can be created from any branch.""" |
415 | - super(RunBotGroupedBranch, self).__init__(runbot, sticky) |
416 | - self.team_name = team_name |
417 | - self.name = b.name |
418 | - self.name_underscore = underscore(self.name) |
419 | - self.server = None |
420 | - self.addons = [] # Only one branch in a non-configured group. |
421 | - self.web = None |
422 | - self.json_path=os.path.join(runbot.wd,'static',"%s-%s.json"%(team_name, self.name.replace('_','-').replace('.','-'))) |
423 | - |
424 | - self.add_branch(b) |
425 | - |
426 | def add_branch(self, b): |
427 | + """ Add a single branch (when not using configure_branches(). """ |
428 | assert b.name == self.name |
429 | b.group = self |
430 | if b.project_name == 'addons': |
431 | @@ -363,403 +311,28 @@ |
432 | for p in ('server', 'web'): |
433 | if not getattr(self, p): |
434 | b = self.runbot.main_branches[p+ending] |
435 | - setattr(self, p, RunBotBranch(self.runbot,b,group=self,overriden_repo_path=paths[p])) |
436 | + setattr(self, p, RunBotBranch(self.runbot,b,group=self,repo_path=paths[p])) |
437 | getattr(self, p).update_launchpad(b) |
438 | - elif getattr(self, p).overriden_repo_path: |
439 | + elif not getattr(self, p).trigger_build: |
440 | b = self.runbot.main_branches[p+ending] |
441 | getattr(self, p).update_launchpad(b) |
442 | if not self.addons: |
443 | b = self.runbot.main_branches['addons'+ending] |
444 | - self.addons = [RunBotBranch(self.runbot,b,group=self,overriden_repo_path=paths['addons'])] |
445 | + self.addons = [RunBotBranch(self.runbot,b,group=self,repo_path=paths['addons'])] |
446 | self.addons[0].update_launchpad(b) |
447 | - elif self.addons[0].overriden_repo_path: |
448 | + elif not self.addons[0].trigger_build: |
449 | b = self.runbot.main_branches['addons'+ending] |
450 | self.addons[0].update_launchpad(b) |
451 | |
452 | - def repo_updates(self): |
453 | - r = [] |
454 | - r.append(copy.copy(self.server)) |
455 | - r.append(copy.copy(self.addons[0])) |
456 | - r.append(copy.copy(self.web)) |
457 | - return r |
458 | - |
459 | - def lp_date_last_modified(self): |
460 | - lp_date_last_modified = None |
461 | - for p in ('server', 'web'): |
462 | - if getattr(self, p) and not getattr(self, p).overriden_repo_path \ |
463 | - and (not lp_date_last_modified or getattr(self, p).lp_date_last_modified > lp_date_last_modified): |
464 | - lp_date_last_modified = getattr(self, p).lp_date_last_modified |
465 | - if self.addons and not self.addons[0].overriden_repo_path \ |
466 | - and (not lp_date_last_modified or self.addons[0].lp_date_last_modified > lp_date_last_modified): |
467 | - lp_date_last_modified = self.addons[0].lp_date_last_modified |
468 | - assert lp_date_last_modified # at least one project in the group |
469 | - return lp_date_last_modified |
470 | - |
471 | -#---------------------------------------------------------- |
472 | -# OpenERP RunBot Worker |
473 | -#---------------------------------------------------------- |
474 | - |
475 | -class Job(object): |
476 | - """ |
477 | - A Job encapsulates all the necessary data to build a branch group for a |
478 | - given slot. The build is done in its own worker thread. |
479 | - """ |
480 | - |
481 | - def __init__(self, g, port, test, job_id, debug): |
482 | - self.job_id = job_id |
483 | - self.completed = False |
484 | - self.team_name = g.team_name |
485 | - self.name = g.name |
486 | - self.name_underscore = g.name_underscore |
487 | - self.version = g.version |
488 | - self.debug = debug |
489 | - |
490 | - self.repo_updates = g.repo_updates() |
491 | - self.port = port |
492 | - self.test = test |
493 | - self.running_server_pid = 0 |
494 | - self.client_web_pid = 0 |
495 | - self.running_t0=0 |
496 | - |
497 | - repo = os.path.join(g.runbot.wd,'repo') |
498 | - self.server_src = g.server.repo_path |
499 | - self.client_web_src = g.web.repo_path if g.version == '6.0' and g.web else None |
500 | - self.web_src = g.web.repo_path if g.web else None |
501 | - |
502 | - # if addons is not the full addons branch use trunk |
503 | - self.addons_src = [a.repo_path for a in g.addons] |
504 | - |
505 | - # Running path <Root>/static/<domain> |
506 | - self.subdomain = "%s-%s-%s"%(self.team_name, self.name.replace('_','-').replace('.','-'),self.job_id) |
507 | - self.running_path = os.path.join(g.runbot.wd,'static',self.subdomain) |
508 | - self.json_path = g.json_path |
509 | - self.log_path = os.path.join(self.running_path,'logs') |
510 | - self.flags_path = os.path.join(self.running_path,'flags') |
511 | - |
512 | - # Server |
513 | - self.server_path=os.path.join(self.running_path,"server") |
514 | - self.server_bin_path=os.path.join(self.server_path,"openerp-server") |
515 | - self.server_log_path=os.path.join(self.log_path,'server.txt') |
516 | - self.server_log_base_path=os.path.join(self.log_path,'test-base.txt') |
517 | - self.server_log_all_path=os.path.join(self.log_path,'test-all.txt') |
518 | - |
519 | - # coverage |
520 | - self.coverage_file_path=os.path.join(self.log_path,'coverage.pickle') |
521 | - self.coverage_base_path=os.path.join(self.log_path,'coverage-base') |
522 | - self.coverage_all_path=os.path.join(self.log_path,'coverage-all') |
523 | - |
524 | - # Web60 |
525 | - self.client_web_pid=None |
526 | - self.client_web_path=os.path.join(self.running_path,"client-web") |
527 | - self.client_web_bin_path=os.path.join(self.client_web_path,"openerp-web.py") |
528 | - self.client_web_doc_path=os.path.join(self.client_web_path,"doc") |
529 | - self.client_web_log_path=os.path.join(self.log_path,'client-web.txt') |
530 | - |
531 | - # test |
532 | - self.test_base_result=None |
533 | - self.test_base_path=os.path.join(self.log_path,'test-base.txt') |
534 | - self.test_all_result=None |
535 | - self.test_all_path=os.path.join(self.log_path,'test-all.txt') |
536 | - |
537 | - self.server_net_port=g.runbot.server_net_port |
538 | - self.server_xml_port=g.runbot.server_xml_port |
539 | - self.client_web_port=g.runbot.client_web_port |
540 | - |
541 | - self.db = self.name_underscore + '_' + str(self.job_id) |
542 | - self.db_all = "%s_all" % self.db |
543 | - |
544 | - for p in ('addons', 'server', 'web'): |
545 | - setattr(self, p + '_committer_name', None) |
546 | - setattr(self, p + '_committer_xgram', None) |
547 | - setattr(self, p + '_committer_email', None) |
548 | - self.start_time = 0 |
549 | - self.completed_time = 0 |
550 | - |
551 | - self.modules = g.modules |
552 | - |
553 | - def spawn(self): |
554 | - log("runbot-spawn-worker-" + str(self.job_id), group=self.name) |
555 | - t = threading.Thread(target=self.work, name=('runbot-group-worker-' + str(self.job_id))) |
556 | - t.daemon = True |
557 | - t.start() |
558 | - |
559 | - def work(self): |
560 | - try: |
561 | - self.pull_branches() |
562 | - self.start_time = time.time() |
563 | - self.start() |
564 | - self.completed_time = time.time() |
565 | - self.completed = True |
566 | - log("runbot-end-worker", job=self.name) |
567 | - except Exception, e: |
568 | - self.completed = True |
569 | - log("runbot-end-worker [with exception]", job=self.name) |
570 | - print traceback.format_exc() |
571 | - |
572 | - def pull_branches(self): |
573 | - for b in self.repo_updates: |
574 | - log("branch-update",branch=b.unique_name) |
575 | - if os.path.exists(b.repo_path): |
576 | - run(["bzr", "pull", "-d", b.repo_path, "--overwrite"]) |
577 | - else: |
578 | - run(["bzr", "branch", "lp:%s"%b.unique_name, b.repo_path]) |
579 | - run(["bzr", "update", "-r", str(b.local_revision_count), b.repo_path]) |
580 | - |
581 | - committer_name, committer_xgram, committer_email = \ |
582 | - get_committer_info(b.repo_path) |
583 | - b.committer_name = committer_name |
584 | - b.committer_xgram = committer_xgram |
585 | - b.committer_email = committer_email |
586 | - |
587 | - def start_rsync(self): |
588 | - log("job-start-rsync",branch=self.name) |
589 | - for i in [self.running_path,self.log_path,self.flags_path,self.client_web_doc_path]: |
590 | - if not os.path.exists(i): |
591 | - os.makedirs(i) |
592 | - # copy server |
593 | - run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.server_src, self.server_path]) |
594 | - |
595 | - # copy addons (6.0 uses bin/addons, 6.1 and trunk use openerp/addons) |
596 | - addons_dest = "openerp/addons" |
597 | - if not os.path.exists(os.path.join(self.server_path,addons_dest)): |
598 | - addons_dest = "bin/addons" |
599 | - addons_path = os.path.join(self.server_path,addons_dest) |
600 | - for a in self.addons_src: |
601 | - run(["rsync","-a","--exclude",".bzr", "%s/"%a, addons_path]) |
602 | - if self.web_src and self.version != '6.0': |
603 | - run(["rsync","-a","--exclude",".bzr", "%s/addons/"%self.web_src, addons_path]) |
604 | - |
605 | - # copy web-client |
606 | - if self.client_web_src: |
607 | - run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.client_web_src, self.client_web_path]) |
608 | - |
609 | - def start_createdb(self): |
610 | - run(["psql","template1","-c","select pg_terminate_backend(procpid) from pg_stat_activity where datname in ('%s','%s')"%(self.db,self.db_all)]) |
611 | - time.sleep(3) |
612 | - run(["dropdb",self.db]) |
613 | - run(["dropdb",self.db_all]) |
614 | - run(["createdb",self.db]) |
615 | - run(["createdb",self.db_all]) |
616 | - |
617 | - def run_log(self, cmd, logfile, env=None): |
618 | - env = dict(os.environ, **env) if env else None |
619 | - log("run", *cmd, logfile=logfile) |
620 | - out=open(logfile,"w") |
621 | - p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True, env=env) |
622 | - self.running_server_pid=p.pid |
623 | - p.communicate() |
624 | - return p |
625 | - |
626 | - def resolve_server_bin_path(self): |
627 | - # This can be done only if the files are present otherwise the if |
628 | - # will always fail. Alternatively, server_bin_path could be a property. |
629 | - if not os.path.exists(self.server_bin_path): # for 6.0 branches |
630 | - self.server_bin_path=os.path.join(self.server_path,"bin","openerp-server.py") |
631 | - |
632 | - def start_test_base(self): |
633 | - log("job-start-server-base") |
634 | - cmd = [self.server_bin_path,"-d",self.db,"-i","base","--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"] |
635 | - _has_test_enable_flag = False |
636 | - if has_test_enable_flag(self.server_bin_path): |
637 | - cmd.append("--test-enable") |
638 | - _has_test_enable_flag = True |
639 | - cmd = ["coverage","run","--branch"] + cmd |
640 | - self.run_log(cmd, logfile=self.test_base_path,env={'COVERAGE_FILE': self.coverage_file_path}) |
641 | - run(["coverage","html","-d",self.coverage_base_path,"--ignore-errors","--include=*.py"],env={'COVERAGE_FILE': self.coverage_file_path}) |
642 | - if _has_test_enable_flag: |
643 | - success_message = "openerp.modules.loading: Modules loaded." |
644 | - rc = not bool(run(["grep",success_message,self.test_base_path])) |
645 | - else: |
646 | - rc = bool(run(["grep","Traceback",self.test_base_path])) |
647 | - self.test_base_result = rc |
648 | - |
649 | - def start_test_all(self): |
650 | - log("job-start-server-all") |
651 | - mods = [] |
652 | - if not os.path.exists(os.path.join(self.server_path,"openerp/addons")): |
653 | - mods = os.listdir(os.path.join(self.server_path,"bin/addons")) |
654 | - elif os.path.exists(os.path.join(self.server_path,"openerp/addons")): |
655 | - mods = os.listdir(os.path.join(self.server_path,"openerp/addons")) |
656 | - bl = [ |
657 | - 'document_ftp', 'l10n_lu', |
658 | - '.bzrignore', '__init__.pyc', '__init__.py', 'base_quality_interrogation.py', |
659 | - ] |
660 | - mods = [m for m in mods if m not in bl] |
661 | - if self.modules: |
662 | - # TODO feedback when a requested modules does'nt exist. |
663 | - mods = [m for m in mods if m in self.modules] |
664 | - mods = ",".join(mods) |
665 | - cmd = [self.server_bin_path,"-d",self.db_all,"-i",mods,"--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"] |
666 | - _has_test_enable_flag = False |
667 | - if has_test_enable_flag(self.server_bin_path): |
668 | - cmd.append("--test-enable") |
669 | - _has_test_enable_flag = True |
670 | - cmd = ["coverage","run","--branch"] + cmd |
671 | - self.run_log(cmd, logfile=self.test_all_path) |
672 | - run(["coverage","html","-d",self.coverage_all_path,"--ignore-errors","--include=*.py"]) |
673 | - if _has_test_enable_flag: |
674 | - success_message = "openerp.modules.loading: Modules loaded." |
675 | - rc = not bool(run(["grep",success_message,self.test_all_path])) |
676 | - else: |
677 | - rc = bool(run(["grep","Traceback",self.test_all_path])) |
678 | - self.test_all_result = rc |
679 | - |
680 | - def start_server(self): |
681 | - port = self.port |
682 | - log("job-start-server",branch=self.name,port=port) |
683 | - cmd=[self.server_bin_path,"--no-xmlrpcs","--netrpc-port=%d"%(self.server_net_port+port),"--xmlrpc-port=%d"%(self.server_xml_port+port)] |
684 | - if os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi.py')) \ |
685 | - or os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi', 'core.py')): |
686 | - cmd+=["--db-filter=%d(_.*)?$","--load=web"] |
687 | - log("run",*cmd,log=self.server_log_path) |
688 | - out=open(self.server_log_path,"w") |
689 | - p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True) |
690 | - self.running_server_pid=p.pid |
691 | - |
692 | - def start_client_web(self): |
693 | - if not self.client_web_src: |
694 | - return |
695 | - port = self.port |
696 | - log("job-start-client-web",branch=self.name,port=port) |
697 | - config=""" |
698 | - [global] |
699 | - server.environment = "development" |
700 | - server.socket_host = "0.0.0.0" |
701 | - server.socket_port = %d |
702 | - server.thread_pool = 10 |
703 | - tools.sessions.on = True |
704 | - log.access_level = "INFO" |
705 | - log.error_level = "INFO" |
706 | - tools.csrf.on = False |
707 | - tools.log_tracebacks.on = False |
708 | - tools.cgitb.on = True |
709 | - openerp.server.host = 'localhost' |
710 | - openerp.server.port = '%d' |
711 | - openerp.server.protocol = 'socket' |
712 | - openerp.server.timeout = 450 |
713 | - [openerp-web] |
714 | - dblist.filter = 'BOTH' |
715 | - dbbutton.visible = True |
716 | - company.url = '' |
717 | - openerp.server.host = 'localhost' |
718 | - openerp.server.port = '%d' |
719 | - openerp.server.protocol = 'socket' |
720 | - openerp.server.timeout = 450 |
721 | - """%(self.client_web_port+port,self.server_net_port+port,self.server_net_port+port) |
722 | - config=config.replace("\n ","\n") |
723 | - cfgs = [os.path.join(self.client_web_path,"doc","openerp-web.cfg"), os.path.join(self.client_web_path,"openerp-web.cfg")] |
724 | - for i in cfgs: |
725 | - f=open(i,"w") |
726 | - f.write(config) |
727 | - f.close() |
728 | - |
729 | - cmd=[self.client_web_bin_path] |
730 | - log("run",*cmd,log=self.client_web_log_path) |
731 | - out=open(self.client_web_log_path,"w") |
732 | - p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True) |
733 | - self.client_web_pid=p.pid |
734 | - |
735 | - def start(self): |
736 | - log("job-start",branch=self.name,port=self.port) |
737 | - self.start_rsync() |
738 | - self.resolve_server_bin_path() |
739 | - self.start_createdb() |
740 | - try: |
741 | - if self.test: |
742 | - self.start_test_base() |
743 | - if not self.debug: |
744 | - self.start_test_all() |
745 | - else: |
746 | - self.test_all_result = True |
747 | - self.start_server() |
748 | - self.start_client_web() |
749 | - except OSError,e: |
750 | - log("branch-start-error",exception=e) |
751 | - except IOError,e: |
752 | - log("branch-start-error",exception=e) |
753 | - self.running_t0=time.time() |
754 | - log("branch-started",branch=self.name,port=self.port) |
755 | - |
756 | - def stop(self): |
757 | - log("Stopping job", id=self.job_id, branch=self.name) |
758 | - if self.running_server_pid: |
759 | - kill(self.running_server_pid) |
760 | - if self.client_web_pid: |
761 | - kill(self.client_web_pid) |
762 | - |
763 | -#---------------------------------------------------------- |
764 | -# OpenERP RunBot Build Slot |
765 | -#---------------------------------------------------------- |
766 | - |
767 | -class Point(object): |
768 | - """A point is a build slot and associated to a worker thread.""" |
769 | - def __init__(self, g, j): |
770 | - """Create a Point from a given group and job.""" |
771 | - self.version = g.version |
772 | - self.port = j.port |
773 | - self.job_id = j.job_id |
774 | - self.team_name = j.team_name |
775 | - self.name_underscore = j.name_underscore |
776 | - self.db = j.db |
777 | - self.running_path = j.running_path |
778 | - self.json_path = j.json_path |
779 | - self.subdomain = j.subdomain |
780 | - self.repo_updates = j.repo_updates # committer date not yet available, available after j.work() is called. |
781 | - self.server_rev = g.server.local_revision_count |
782 | - self.addons_rev = [a.local_revision_count for a in g.addons] |
783 | - self.web_rev = g.web.local_revision_count if g.web else 0 |
784 | - self.need_run_reason = g.need_run_reason |
785 | - for p in ('addons', 'server', 'web'): |
786 | - setattr(self, p + '_committer_name', getattr(j, p + '_committer_name')) |
787 | - setattr(self, p + '_committer_xgram', getattr(j, p + '_committer_xgram')) |
788 | - setattr(self, p + '_committer_email', getattr(j, p + '_committer_email')) |
789 | - self.update(j) |
790 | - self.manual_build = g.manual_build |
791 | - |
792 | - def update(self, j): |
793 | - """ |
794 | - Update the Point from a job. The job should be the same than the one |
795 | - used in `__init__()`. |
796 | - """ |
797 | - self.state = 'running' if j.completed else 'testing' |
798 | - self.running_t0 = j.running_t0 |
799 | - self.test_base_result = j.test_base_result |
800 | - self.test_all_result = j.test_all_result |
801 | - |
802 | - def save_json(self): |
803 | - """Append the point data to a JSON file for posterity.""" |
804 | - # A job must be saved only once. |
805 | - if hasattr(self, 'json_saved'): |
806 | - log("=== save_json() called more than once. ===", job_id=self.job_id) |
807 | - self.json_saved = True |
808 | - |
809 | - path = self.json_path |
810 | - state = {} |
811 | - |
812 | - if os.path.exists(path): |
813 | - with open(path, 'r') as h: |
814 | - state = simplejson.loads(h.read()) |
815 | - |
816 | - value = {} |
817 | - committers = [] |
818 | - for p in ('addons', 'server', 'web'): |
819 | - committers += [ |
820 | - p + '_committer_name', |
821 | - p + '_committer_xgram', |
822 | - # p + '_committer_email', |
823 | - ] |
824 | - for a in [ |
825 | - 'job_id', 'db', 'running_path', 'subdomain', 'server_rev', |
826 | - 'addons_rev', 'web_rev', 'need_run_reason', |
827 | - 'test_base_result', 'test_all_result', |
828 | - ] + committers: |
829 | - value[a] = getattr(self, a) |
830 | - state.setdefault('jobs', []) |
831 | - state['jobs'].append(value) |
832 | - |
833 | - with open(path, 'w') as h: |
834 | - data = simplejson.dumps(state, sort_keys=True, indent=4) |
835 | - h.write(data) |
836 | + if self.job_type == 'migrate_script': |
837 | + if not hasattr(self, 'migration_script'): |
838 | + self.migration_script = RunBotBranch(self.runbot, |
839 | + migration_scripts_branch, group=self, repo_path='repo/openerp_migration_script', project_name='migration-scripts', trigger_build=True) |
840 | + self.migration_platform = RunBotBranch(self.runbot, |
841 | + migration_platform_branch, group=self, repo_path='repo/openerp_migration_platform', project_name='migration-platform', trigger_build=True) |
842 | + self.migration_script.update_launchpad(migration_scripts_branch) |
843 | + self.migration_platform.update_launchpad(migration_platform_branch) |
844 | + print "initialized group with job type <migration_script>" |
845 | |
846 | #---------------------------------------------------------- |
847 | # OpenERP RunBot Engine |
848 | @@ -769,7 +342,7 @@ |
849 | """Used as a singleton, to manage almost all the Runbot state and logic.""" |
850 | def __init__(self, wd, poll, server_net_port, server_xml_port, |
851 | client_web_port, number, nginx_port, domain, test, workers, |
852 | - current_job_id, debug, lp): |
853 | + current_job_id, debug, lp, default_job_type='install_all'): |
854 | self.wd=wd |
855 | self.server_net_port=int(server_net_port) |
856 | self.server_xml_port=int(server_xml_port) |
857 | @@ -788,9 +361,10 @@ |
858 | # realizing a queue of groups to be processed). |
859 | self.current_job_id = current_job_id |
860 | self.debug = debug |
861 | - self.manual_build_count = 0 |
862 | + self.next_build_number = 0 |
863 | self.launchpad = lp |
864 | self.checked_date_last_modified = None |
865 | + self.default_job_type = default_job_type |
866 | |
867 | def registered_teams(self): |
868 | return openerprunbot.state.get('registered-teams', []) + ['openerp-dev'] |
869 | @@ -833,7 +407,7 @@ |
870 | return gs |
871 | |
872 | def nginx_groups_registered(self,team_name): |
873 | - gs = [(g.name,g) for g in self.groups.values() if not any(g.points) and g.team_name==team_name and g.manual_build == sys.maxint] |
874 | + gs = [(g.name,g) for g in self.groups.values() if not any(g.points) and g.team_name==team_name and g.build_number == sys.maxint] |
875 | gs.sort() |
876 | return [g for (x,g) in gs] |
877 | |
878 | @@ -897,26 +471,28 @@ |
879 | |
880 | def process_add(self, b, sticky, team_name): |
881 | log("runbot-process-add", b.unique_name) |
882 | - if not re.search("/(openobject|openerp)-(addons|server|client-web|web)/", b.unique_name): |
883 | + if not project_name_from_unique_name(b.unique_name): |
884 | log("WARNING: can't add branch: project name is not standard.", unique_name=b.unique_name) |
885 | return |
886 | if b.unique_name not in self.branches: |
887 | - self.branches[b.unique_name] = RunBotBranch(self, b) |
888 | + self.branches[b.unique_name] = RunBotBranch(self, b, trigger_build=True) |
889 | bb = self.branches[b.unique_name] |
890 | if (team_name, bb.name) not in self.groups: |
891 | - self.groups[(team_name, bb.name)] = RunBotGroupedBranch(self, bb, team_name, sticky) |
892 | - else: |
893 | - self.groups[(team_name, bb.name)].add_branch(bb) |
894 | + self.groups[(team_name, bb.name)] = RunBotGroupedBranch(self, team_name, bb.name, None, sticky, job_type=self.default_job_type) |
895 | + self.groups[(team_name, bb.name)].add_branch(bb) |
896 | bb.update_launchpad(b) |
897 | filters = {'status': NEW_MERGE_STATUS} |
898 | - bb.merge_count = len(list(b.getMergeProposals(**filters))) |
899 | + try: |
900 | + bb.merge_count = len(list(b.getMergeProposals(**filters))) |
901 | + except Exception, e: |
902 | + bb.merge_count = 0 |
903 | |
904 | def register_configured_branches(self): |
905 | for team, v in openerprunbot.state.get('configured-branches', {}).items(): |
906 | if not team: continue |
907 | for name, c in v.items(): |
908 | if not name: continue |
909 | - g = ConfiguredGroup(self, team, name, c['version'], 0, |
910 | + g = RunBotGroupedBranch(self, team, name, c['version'], 0, |
911 | server_branch=c['server_branch'], |
912 | client_web_branch=c.get('client_web_branch'), |
913 | web_branch=c.get('web_branch'), |
914 | @@ -925,7 +501,7 @@ |
915 | |
916 | # TODO or if it is already there but with different branches. |
917 | if (g.team_name, g.name) not in self.groups: |
918 | - g.populate_branches(self.launchpad) |
919 | + g.configure_branches(self.launchpad) |
920 | if g.is_ok(): |
921 | log("adding configured group", team=g.team_name, name=g.name) |
922 | self.groups[(g.team_name, g.name)] = g |
923 | @@ -936,6 +512,11 @@ |
924 | def populate_branches(self): |
925 | """Return all LP branches matching our teams and projects.""" |
926 | log("runbot-populate-branches") |
927 | + # Get migration branches |
928 | + global migration_scripts_branch |
929 | + global migration_platform_branch |
930 | + migration_scripts_branch = self.launchpad.get_branch(unique_name='~openerp-dev/openerp-int/migration-scripts') |
931 | + migration_platform_branch = self.launchpad.get_branch(unique_name='~openerp-dev/openerp-int/migration-platform') |
932 | # Register main sticky branches |
933 | for k, v in MAIN_BRANCHES.items(): |
934 | b = self.launchpad.get_branch(unique_name=v) |
935 | @@ -944,6 +525,7 @@ |
936 | |
937 | # Register other branches |
938 | for v in openerprunbot.state.get('registered-branches', []): |
939 | + break |
940 | m = openerprunbot.branch_input_re.match(v) |
941 | if m: |
942 | b = self.launchpad.get_branch(unique_name=v) |
943 | @@ -961,26 +543,19 @@ |
944 | if self.debug: break |
945 | team_branches = self.launchpad.get_team_branches(team_name) |
946 | for b in team_branches: |
947 | - if re.search("/(openobject|openerp)-(addons|server|client-web|web)/",b.unique_name): |
948 | + if project_name_from_unique_name(b.unique_name): |
949 | self.process_add(b, 0, team_name) |
950 | |
951 | # Register configured branches |
952 | self.register_configured_branches() |
953 | |
954 | for g in self.groups.values(): |
955 | - if not g.is_configured(): |
956 | + if not g.configured: |
957 | g.add_main_branches() |
958 | |
959 | self.checked_date_last_modified = max(x.lp_date_last_modified() for x in self.groups.values()) |
960 | self.assign_positions() |
961 | return self.get_queue() |
962 | - #return self.sorted_groups() |
963 | - |
964 | - def sorted_groups(self): |
965 | - gs = [(g.sticky, -g.manual_build, g.lp_date_last_modified(), g) for g in self.groups.values()] |
966 | - gs.sort(reverse=1) |
967 | - gs = [g for (x,y,z,g) in gs][:self.number] |
968 | - return gs |
969 | |
970 | def assign_positions(self): |
971 | # Sort by last modification date, then assign a position in the queue. |
972 | @@ -992,13 +567,13 @@ |
973 | continue |
974 | if not g.sticky: |
975 | self.checked_date_last_modified = g.lp_date_last_modified() |
976 | - if g.need_run_reason and g.manual_build == sys.maxint: |
977 | - self.manual_build_count +=1 |
978 | - g.manual_build = self.manual_build_count |
979 | + if g.need_run_reason and g.build_number == sys.maxint: |
980 | + self.next_build_number +=1 |
981 | + g.build_number = self.next_build_number |
982 | |
983 | def get_queue(self): |
984 | - gs = sorted(self.groups.values(), key=lambda x: x.manual_build) |
985 | - return filter(lambda g: g.manual_build != sys.maxint, gs) |
986 | + gs = sorted(self.groups.values(), key=lambda x: x.build_number) |
987 | + return filter(lambda g: g.build_number != sys.maxint, gs) |
988 | |
989 | def available_workers(self): |
990 | return self.workers - len([t for t in threading.enumerate() if t.name.startswith('runbot-group-worker-')]) |
991 | @@ -1008,32 +583,32 @@ |
992 | command, params = openerprunbot.queue.get() |
993 | if command == 'build': |
994 | team_name, group_name = params |
995 | - if (team_name, group_name) in self.groups and self.groups[(team_name, group_name)].manual_build == sys.maxint: |
996 | - self.manual_build_count += 1 |
997 | - self.groups[(team_name, group_name)].manual_build = self.manual_build_count |
998 | + if (team_name, group_name) in self.groups and self.groups[(team_name, group_name)].build_number == sys.maxint: |
999 | + self.next_build_number += 1 |
1000 | + self.groups[(team_name, group_name)].build_number = self.next_build_number |
1001 | self.groups[(team_name, group_name)].need_run_reason.append('build') |
1002 | else: |
1003 | log("WARNING: unknown command", command) |
1004 | |
1005 | def reset_build_numbers(self): |
1006 | - gs = [(g.manual_build, g) for g in self.groups.values()] |
1007 | + gs = [(g.build_number, g) for g in self.groups.values()] |
1008 | gs.sort() |
1009 | gs = [g for (x,g) in gs] |
1010 | - self.manual_build_count = 0 |
1011 | + self.next_build_number = 0 |
1012 | sticky_branches = 0 |
1013 | for g in gs: |
1014 | if g.sticky: |
1015 | sticky_branches += 1 |
1016 | - if g.manual_build == sys.maxint: |
1017 | + if g.build_number == sys.maxint: |
1018 | break |
1019 | - self.manual_build_count += 1 |
1020 | - g.manual_build = self.manual_build_count |
1021 | + self.next_build_number += 1 |
1022 | + g.build_number = self.next_build_number |
1023 | self.number = max(sticky_branches + 1, self.number) |
1024 | |
1025 | def complete_jobs(self): |
1026 | """Update all slots with the completed jobs.""" |
1027 | for job in self.jobs.values(): |
1028 | - if job.completed: |
1029 | + if job.state in (STATE_RUNNING, STATE_BROKEN): |
1030 | for g in self.groups.values(): |
1031 | if g.name == job.name: |
1032 | g.complete_point(job) |
1033 | @@ -1052,7 +627,7 @@ |
1034 | for job in self.jobs.itervalues(): |
1035 | if self.is_sticky_job(job): |
1036 | continue |
1037 | - if not job.running_t0: |
1038 | + if not job.running_since: |
1039 | continue |
1040 | if not victim or job.job_id < victim.job_id: |
1041 | victim = job |
1042 | @@ -1082,10 +657,11 @@ |
1043 | if g.need_run_reason and g.all_points_completed(): |
1044 | port = self.allocate_port() |
1045 | self.current_job_id += 1 |
1046 | - job = Job(g, port, self.test, self.current_job_id, self.debug) |
1047 | + job_class = openerprunbot.jobs.JOBS[g.job_type] if g.job_type else openerprunbot.jobs.JOBS[self.default_job_type] |
1048 | + job = job_class(g, port, self.test, self.current_job_id, self.debug) |
1049 | g.add_point(job) |
1050 | g.need_run_reason = [] |
1051 | - g.manual_build = sys.maxint |
1052 | + g.build_number = sys.maxint |
1053 | self.jobs[job.job_id] = job |
1054 | job.spawn() |
1055 | |
1056 | |
1057 | === added directory 'openerp-runbot/openerprunbot/jobs' |
1058 | === added file 'openerp-runbot/openerprunbot/jobs/__init__.py' |
1059 | --- openerp-runbot/openerprunbot/jobs/__init__.py 1970-01-01 00:00:00 +0000 |
1060 | +++ openerp-runbot/openerprunbot/jobs/__init__.py 2012-07-04 11:46:17 +0000 |
1061 | @@ -0,0 +1,7 @@ |
1062 | +import install_all |
1063 | +import migrate_script |
1064 | + |
1065 | +JOBS = { |
1066 | + 'install_all': install_all.InstallAllJob, |
1067 | + 'migrate_script': migrate_script.MigrateScriptJob, |
1068 | + } |
1069 | |
1070 | === added file 'openerp-runbot/openerprunbot/jobs/install_all.py' |
1071 | --- openerp-runbot/openerprunbot/jobs/install_all.py 1970-01-01 00:00:00 +0000 |
1072 | +++ openerp-runbot/openerprunbot/jobs/install_all.py 2012-07-04 11:46:17 +0000 |
1073 | @@ -0,0 +1,366 @@ |
1074 | +import os |
1075 | +import subprocess |
1076 | +import threading |
1077 | +import time |
1078 | +import traceback |
1079 | + |
1080 | +from ..misc import * |
1081 | + |
1082 | +# Job states |
1083 | +STATE_ALLOCATED = 'allocated' |
1084 | +STATE_BROKEN = 'broken' |
1085 | +STATE_PULLING = 'pulling' |
1086 | +STATE_RUNNING = 'running' |
1087 | +STATE_TESTING = 'testing' |
1088 | + |
1089 | +class InstallAllJob(object): |
1090 | + """ |
1091 | + A Job encapsulates all the necessary data to build a branch group for a |
1092 | + given slot. The build is done in its own worker thread. |
1093 | + """ |
1094 | + |
1095 | + def __init__(self, g, port, test, job_id, debug): |
1096 | + self.job_id = job_id |
1097 | + self.state = STATE_ALLOCATED |
1098 | + self.team_name = g.team_name |
1099 | + self.name = g.name |
1100 | + self.name_underscore = g.name_underscore |
1101 | + self.version = g.version |
1102 | + self.debug = debug |
1103 | + self.need_run_reason = g.need_run_reason |
1104 | + |
1105 | + self.repo_updates = g.repo_updates() |
1106 | + self.port = port |
1107 | + self.test = test |
1108 | + self.running_server_pid = 0 |
1109 | + self.client_web_pid = 0 |
1110 | + self.running_since=0 |
1111 | + |
1112 | + repo = os.path.join(g.runbot.wd,'repo') |
1113 | + job_suffix = '_job-' + str(self.job_id) |
1114 | + self.server_src = g.server.repo_path + job_suffix |
1115 | + self.client_web_src = (g.web.repo_path + job_suffix) if g.version == '6.0' and g.web else None |
1116 | + self.web_src = (g.web.repo_path + job_suffix) if g.web else None |
1117 | + |
1118 | + # if addons is not the full addons branch use trunk |
1119 | + self.addons_src = [a.repo_path + job_suffix for a in g.addons] |
1120 | + |
1121 | + if hasattr(g, 'migration_script'): |
1122 | + self.migration_script_src = g.migration_script.repo_path + job_suffix |
1123 | + self.migration_platform_src = g.migration_platform.repo_path + job_suffix |
1124 | + |
1125 | + # Running path <Root>/static/<domain> |
1126 | + self.subdomain = "%s-%s-%s"%(self.team_name, self.name.replace('_','-').replace('.','-'),self.job_id) |
1127 | + self.running_path = os.path.join(g.runbot.wd,'static',self.subdomain) |
1128 | + self.json_path = g.json_path |
1129 | + self.log_path = os.path.join(self.running_path,'logs') |
1130 | + self.flags_path = os.path.join(self.running_path,'flags') |
1131 | + |
1132 | + # Server |
1133 | + self.server_path=os.path.join(self.running_path,"server") |
1134 | + self.server_bin_path=os.path.join(self.server_path,"openerp-server") |
1135 | + self.server_log_path=os.path.join(self.log_path,'server.txt') |
1136 | + self.server_log_base_path=os.path.join(self.log_path,'test-base.txt') |
1137 | + self.server_log_all_path=os.path.join(self.log_path,'test-all.txt') |
1138 | + |
1139 | + # coverage |
1140 | + self.coverage_file_path=os.path.join(self.log_path,'coverage.pickle') |
1141 | + self.coverage_base_path=os.path.join(self.log_path,'coverage-base') |
1142 | + self.coverage_all_path=os.path.join(self.log_path,'coverage-all') |
1143 | + |
1144 | + # Web60 |
1145 | + self.client_web_pid=None |
1146 | + self.client_web_path=os.path.join(self.running_path,"client-web") |
1147 | + self.client_web_bin_path=os.path.join(self.client_web_path,"openerp-web.py") |
1148 | + self.client_web_doc_path=os.path.join(self.client_web_path,"doc") |
1149 | + self.client_web_log_path=os.path.join(self.log_path,'client-web.txt') |
1150 | + |
1151 | + # test |
1152 | + self.test_base_path=os.path.join(self.log_path,'test-base.txt') |
1153 | + self.test_result=None |
1154 | + self.test_all_path=os.path.join(self.log_path,'test-all.txt') |
1155 | + |
1156 | + self.server_net_port=g.runbot.server_net_port |
1157 | + self.server_xml_port=g.runbot.server_xml_port |
1158 | + self.client_web_port=g.runbot.client_web_port |
1159 | + |
1160 | + self.db = self.name_underscore + '_' + str(self.job_id) |
1161 | + self.db_all = "%s_all" % self.db |
1162 | + |
1163 | + for p in ('addons', 'server', 'web'): |
1164 | + setattr(self, p + '_committer_name', None) |
1165 | + setattr(self, p + '_committer_xgram', None) |
1166 | + setattr(self, p + '_committer_email', None) |
1167 | + self.start_time = 0 |
1168 | + self.completed_time = 0 |
1169 | + |
1170 | + self.modules = g.modules |
1171 | + |
1172 | + |
1173 | + self.server_rev = g.server.local_revision_count |
1174 | + self.addons_rev = [a.local_revision_count for a in g.addons] |
1175 | + self.web_rev = g.web.local_revision_count if g.web else 0 |
1176 | + self.build_number = g.build_number |
1177 | + |
1178 | + def spawn(self): |
1179 | + log("runbot-spawn-worker-" + str(self.job_id), group=self.name) |
1180 | + t = threading.Thread(target=self.work, name=('runbot-group-worker-' + str(self.job_id))) |
1181 | + t.daemon = True |
1182 | + t.start() |
1183 | + |
1184 | + def work(self): |
1185 | + try: |
1186 | + self.state = STATE_PULLING |
1187 | + r = self.pull_branches() |
1188 | + if r: |
1189 | + self.state = STATE_TESTING |
1190 | + self.start_time = time.time() |
1191 | + self.start() |
1192 | + self.completed_time = time.time() |
1193 | + self.state = STATE_RUNNING |
1194 | + log("runbot-end-worker", job=self.name) |
1195 | + else: |
1196 | + self.state = STATE_BROKEN |
1197 | + log("runbot-end-worker [failed pull]", job=self.name) |
1198 | + except Exception, e: |
1199 | + self.state = STATE_BROKEN |
1200 | + log("runbot-end-worker [with exception]", job=self.name) |
1201 | + print traceback.format_exc() |
1202 | + |
1203 | + def pull_branches(self): |
1204 | + job_suffix = '_job-' + str(self.job_id) |
1205 | + for b in self.repo_updates: |
1206 | + log("branch-update",branch=b.unique_name) |
1207 | + repo_path = b.repo_path + job_suffix |
1208 | + if os.path.exists(repo_path): |
1209 | + rc = run(["bzr", "clean-tree", "-d", repo_path, "--force", "--quiet"]) |
1210 | + if rc: return False |
1211 | + rc = run(["bzr", "pull", "-d", repo_path, "--overwrite", "--quiet"]) |
1212 | + if rc: return False |
1213 | + else: |
1214 | + rc = run(["bzr", "branch", "lp:%s" % b.unique_name, repo_path, "--quiet"]) |
1215 | + if rc: return False |
1216 | + rc = run(["bzr", "update", "-r", str(b.local_revision_count), repo_path, "--quiet"]) |
1217 | + if rc: return False |
1218 | + |
1219 | + committer_name, committer_xgram, committer_email = \ |
1220 | + get_committer_info(repo_path) |
1221 | + b.committer_name = committer_name |
1222 | + b.committer_xgram = committer_xgram |
1223 | + b.committer_email = committer_email |
1224 | + log("get-commiter-info", name=committer_name, xgram=committer_xgram, email=committer_email) |
1225 | + return True |
1226 | + |
1227 | + def start_rsync(self): |
1228 | + log("job-start-rsync",branch=self.name) |
1229 | + for i in [self.running_path,self.log_path,self.flags_path,self.client_web_doc_path]: |
1230 | + if not os.path.exists(i): |
1231 | + os.makedirs(i) |
1232 | + # copy server |
1233 | + run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.server_src, self.server_path]) |
1234 | + |
1235 | + # copy addons (6.0 uses bin/addons, 6.1 and trunk use openerp/addons) |
1236 | + addons_dest = "openerp/addons" |
1237 | + if not os.path.exists(os.path.join(self.server_path,addons_dest)): |
1238 | + addons_dest = "bin/addons" |
1239 | + addons_path = os.path.join(self.server_path,addons_dest) |
1240 | + for a in self.addons_src: |
1241 | + run(["rsync","-a","--exclude",".bzr", "%s/"%a, addons_path]) |
1242 | + if self.web_src and self.version != '6.0': |
1243 | + run(["rsync","-a","--exclude",".bzr", "%s/addons/"%self.web_src, addons_path]) |
1244 | + |
1245 | + # copy web-client |
1246 | + if self.client_web_src: |
1247 | + run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.client_web_src, self.client_web_path]) |
1248 | + |
1249 | + def start_createdb(self): |
1250 | + run(["psql","template1","-c","select pg_terminate_backend(procpid) from pg_stat_activity where datname in ('%s','%s')"%(self.db,self.db_all)]) |
1251 | + time.sleep(3) |
1252 | + run(["dropdb",self.db]) |
1253 | + run(["dropdb",self.db_all]) |
1254 | + run(["createdb",self.db]) |
1255 | + run(["createdb",self.db_all]) |
1256 | + |
1257 | + def run_log(self, cmd, logfile, env=None): |
1258 | + env = dict(os.environ, **env) if env else None |
1259 | + log("run", *cmd, logfile=logfile) |
1260 | + out=open(logfile,"w") |
1261 | + p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True, env=env) |
1262 | + self.running_server_pid=p.pid |
1263 | + p.communicate() |
1264 | + return p |
1265 | + |
1266 | + def resolve_server_bin_path(self): |
1267 | + # This can be done only if the files are present otherwise the if |
1268 | + # will always fail. Alternatively, server_bin_path could be a property. |
1269 | + if not os.path.exists(self.server_bin_path): # for 6.0 branches |
1270 | + self.server_bin_path=os.path.join(self.server_path,"bin","openerp-server.py") |
1271 | + |
1272 | + def start_test_base(self): |
1273 | + log("job-start-server-base") |
1274 | + cmd = [self.server_bin_path,"-d",self.db,"-i","base","--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"] |
1275 | + _has_test_enable_flag = False |
1276 | + if has_test_enable_flag(self.server_bin_path): |
1277 | + cmd.append("--test-enable") |
1278 | + _has_test_enable_flag = True |
1279 | + cmd = ["coverage","run","--branch"] + cmd |
1280 | + self.run_log(cmd, logfile=self.test_base_path,env={'COVERAGE_FILE': self.coverage_file_path}) |
1281 | + run(["coverage","html","-d",self.coverage_base_path,"--ignore-errors","--include=*.py"],env={'COVERAGE_FILE': self.coverage_file_path}) |
1282 | + if _has_test_enable_flag: |
1283 | + success_message = "openerp.modules.loading: Modules loaded." |
1284 | + rc = not bool(run(["grep",success_message,self.test_base_path])) |
1285 | + else: |
1286 | + rc = bool(run(["grep","Traceback",self.test_base_path])) |
1287 | + return rc |
1288 | + |
1289 | + def start_test_all(self, without_demo=False): |
1290 | + log("job-start-server-all") |
1291 | + mods = [] |
1292 | + if not os.path.exists(os.path.join(self.server_path,"openerp/addons")): |
1293 | + mods = os.listdir(os.path.join(self.server_path,"bin/addons")) |
1294 | + elif os.path.exists(os.path.join(self.server_path,"openerp/addons")): |
1295 | + mods = os.listdir(os.path.join(self.server_path,"openerp/addons")) |
1296 | + bl = [ |
1297 | + 'document_ftp', 'l10n_lu', |
1298 | + '.bzrignore', '__init__.pyc', '__init__.py', 'base_quality_interrogation.py', |
1299 | + ] |
1300 | + mods = [m for m in mods if m not in bl] |
1301 | + if self.modules: |
1302 | + # TODO feedback when a requested modules does'nt exist. |
1303 | + mods = [m for m in mods if m in self.modules] |
1304 | + mods = ",".join(mods) |
1305 | + cmd = [self.server_bin_path,"-d",self.db_all,"-i",mods,"--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"] |
1306 | + _has_test_enable_flag = False |
1307 | + if has_test_enable_flag(self.server_bin_path): |
1308 | + cmd.append("--test-enable") |
1309 | + _has_test_enable_flag = True |
1310 | + if without_demo: |
1311 | + cmd.append("--without-demo=True") |
1312 | + cmd = ["coverage","run","--branch"] + cmd |
1313 | + self.run_log(cmd, logfile=self.test_all_path) |
1314 | + run(["coverage","html","-d",self.coverage_all_path,"--ignore-errors","--include=*.py"]) |
1315 | + if _has_test_enable_flag: |
1316 | + success_message = "openerp.modules.loading: Modules loaded." |
1317 | + rc = not bool(run(["grep",success_message,self.test_all_path])) |
1318 | + else: |
1319 | + rc = bool(run(["grep","Traceback",self.test_all_path])) |
1320 | + return rc |
1321 | + |
1322 | + def start_server(self): |
1323 | + port = self.port |
1324 | + log("job-start-server",branch=self.name,port=port) |
1325 | + cmd=[self.server_bin_path,"--no-xmlrpcs","--netrpc-port=%d"%(self.server_net_port+port),"--xmlrpc-port=%d"%(self.server_xml_port+port)] |
1326 | + if os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi.py')) \ |
1327 | + or os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi', 'core.py')): |
1328 | + cmd+=["--db-filter=%d(_.*)?$","--load=web"] |
1329 | + log("run",*cmd,log=self.server_log_path) |
1330 | + out=open(self.server_log_path,"w") |
1331 | + p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True) |
1332 | + self.running_server_pid=p.pid |
1333 | + |
1334 | + def start_client_web(self): |
1335 | + if not self.client_web_src: |
1336 | + return |
1337 | + port = self.port |
1338 | + log("job-start-client-web",branch=self.name,port=port) |
1339 | + config=""" |
1340 | + [global] |
1341 | + server.environment = "development" |
1342 | + server.socket_host = "0.0.0.0" |
1343 | + server.socket_port = %d |
1344 | + server.thread_pool = 10 |
1345 | + tools.sessions.on = True |
1346 | + log.access_level = "INFO" |
1347 | + log.error_level = "INFO" |
1348 | + tools.csrf.on = False |
1349 | + tools.log_tracebacks.on = False |
1350 | + tools.cgitb.on = True |
1351 | + openerp.server.host = 'localhost' |
1352 | + openerp.server.port = '%d' |
1353 | + openerp.server.protocol = 'socket' |
1354 | + openerp.server.timeout = 450 |
1355 | + [openerp-web] |
1356 | + dblist.filter = 'BOTH' |
1357 | + dbbutton.visible = True |
1358 | + company.url = '' |
1359 | + openerp.server.host = 'localhost' |
1360 | + openerp.server.port = '%d' |
1361 | + openerp.server.protocol = 'socket' |
1362 | + openerp.server.timeout = 450 |
1363 | + """%(self.client_web_port+port,self.server_net_port+port,self.server_net_port+port) |
1364 | + config=config.replace("\n ","\n") |
1365 | + cfgs = [os.path.join(self.client_web_path,"doc","openerp-web.cfg"), os.path.join(self.client_web_path,"openerp-web.cfg")] |
1366 | + for i in cfgs: |
1367 | + f=open(i,"w") |
1368 | + f.write(config) |
1369 | + f.close() |
1370 | + |
1371 | + cmd=[self.client_web_bin_path] |
1372 | + log("run",*cmd,log=self.client_web_log_path) |
1373 | + out=open(self.client_web_log_path,"w") |
1374 | + p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True) |
1375 | + self.client_web_pid=p.pid |
1376 | + |
1377 | + def start(self): |
1378 | + log("job-start",branch=self.name,port=self.port) |
1379 | + self.start_rsync() |
1380 | + self.resolve_server_bin_path() |
1381 | + self.start_createdb() |
1382 | + try: |
1383 | + if self.test: |
1384 | + r = self.start_test_base() |
1385 | + if not self.debug: |
1386 | + self.test_result = r and self.start_test_all() |
1387 | + else: |
1388 | + self.test_result = r |
1389 | + self.start_server() |
1390 | + self.start_client_web() |
1391 | + except OSError,e: |
1392 | + log("branch-start-error",exception=e) |
1393 | + except IOError,e: |
1394 | + log("branch-start-error",exception=e) |
1395 | + self.running_since=time.time() |
1396 | + log("branch-started",branch=self.name,port=self.port) |
1397 | + |
1398 | + def stop(self): |
1399 | + log("Stopping job", id=self.job_id, branch=self.name) |
1400 | + if self.running_server_pid: |
1401 | + kill(self.running_server_pid) |
1402 | + if self.client_web_pid: |
1403 | + kill(self.client_web_pid) |
1404 | + |
1405 | + def save_json(self): |
1406 | + """Append the point data to a JSON file for posterity.""" |
1407 | + # A job must be saved only once. |
1408 | + if hasattr(self, 'json_saved'): |
1409 | + log("=== save_json() called more than once. ===", job_id=self.job_id) |
1410 | + self.json_saved = True |
1411 | + |
1412 | + path = self.json_path |
1413 | + state = {} |
1414 | + |
1415 | + if os.path.exists(path): |
1416 | + with open(path, 'r') as h: |
1417 | + state = simplejson.loads(h.read()) |
1418 | + |
1419 | + value = {} |
1420 | + committers = [] |
1421 | + for p in ('addons', 'server', 'web'): |
1422 | + committers += [ |
1423 | + p + '_committer_name', |
1424 | + p + '_committer_xgram', |
1425 | + # p + '_committer_email', |
1426 | + ] |
1427 | + for a in [ |
1428 | + 'job_id', 'db', 'running_path', 'subdomain', 'server_rev', |
1429 | + 'addons_rev', 'web_rev', 'need_run_reason', |
1430 | + 'test_result', |
1431 | + ] + committers: |
1432 | + value[a] = getattr(self, a) |
1433 | + state.setdefault('jobs', []) |
1434 | + state['jobs'].append(value) |
1435 | + |
1436 | + with open(path, 'w') as h: |
1437 | + data = simplejson.dumps(state, sort_keys=True, indent=4) |
1438 | + h.write(data) |
1439 | + |
1440 | |
1441 | === added file 'openerp-runbot/openerprunbot/jobs/migrate_script.py' |
1442 | --- openerp-runbot/openerprunbot/jobs/migrate_script.py 1970-01-01 00:00:00 +0000 |
1443 | +++ openerp-runbot/openerprunbot/jobs/migrate_script.py 2012-07-04 11:46:17 +0000 |
1444 | @@ -0,0 +1,61 @@ |
1445 | +import os |
1446 | +import subprocess |
1447 | +import threading |
1448 | +import time |
1449 | +import traceback |
1450 | + |
1451 | +from install_all import InstallAllJob |
1452 | +from ..misc import * |
1453 | + |
1454 | +# Job states |
1455 | +STATE_ALLOCATED = 'allocated' |
1456 | +STATE_BROKEN = 'broken' |
1457 | +STATE_PULLING = 'pulling' |
1458 | +STATE_RUNNING = 'running' |
1459 | +STATE_TESTING = 'testing' |
1460 | + |
1461 | +class MigrateScriptJob(InstallAllJob): |
1462 | + |
1463 | + def start(self, migrate=False): |
1464 | + log("job-start",branch=self.name,port=self.port) |
1465 | + self.migration_log_path=os.path.join(self.log_path,'migration.txt') |
1466 | + |
1467 | + self.start_rsync() |
1468 | + self.resolve_server_bin_path() |
1469 | + self.start_createdb() |
1470 | + try: |
1471 | + rc0 = self.start_test_all(without_demo=True) |
1472 | + |
1473 | + # Run the migration towards trunk |
1474 | + if False: # if migrate |
1475 | + run(['rm', '-rf', 'migration-' + str(self.job_id)]) |
1476 | + run(['cp', '-r', self.migration_script_src, 'migration-' + str(self.job_id)]) |
1477 | + run(['cp', self.migration_platform_src + '/Makefile', 'migration-' + str(self.job_id)]) |
1478 | + cmd = 'make pre update post updatelog reset-demoflag'.split() + [ |
1479 | + 'OPENERP_BIN=' + self.server_bin_path, |
1480 | + 'DATABASE=' + self.db_all, |
1481 | + 'VERSION=trunk'] |
1482 | + run_output(cmd, cwd='migration-' + str(self.job_id)) |
1483 | + |
1484 | + # Check the migrated database with an 'update all' and tests enabled. |
1485 | + cmd = ['psql', self.db_all, '-c', "update ir_module_module set demo='t'"] |
1486 | + run(cmd) |
1487 | + cmd = [self.server_bin_path, "-d", self.db_all, "-u", "base", "--stop-after-init", "--no-xmlrpc", "--no-xmlrpcs", "--no-netrpc", "--log-level=test"] |
1488 | + _has_test_enable_flag = False |
1489 | + if has_test_enable_flag(self.server_bin_path): |
1490 | + cmd.append("--test-enable") |
1491 | + _has_test_enable_flag = True |
1492 | + self.run_log(cmd, logfile=self.test_all_path) |
1493 | + if _has_test_enable_flag: |
1494 | + success_message = "openerp.modules.loading: Modules loaded." |
1495 | + rc = not bool(run(["grep",success_message,self.test_all_path])) |
1496 | + else: |
1497 | + rc = bool(run(["grep","Traceback",self.test_all_path])) |
1498 | + self.test_result = rc0 and rc |
1499 | + |
1500 | + except OSError,e: |
1501 | + log("branch-start-error",exception=e) |
1502 | + except IOError,e: |
1503 | + log("branch-start-error",exception=e) |
1504 | + self.running_since=time.time() |
1505 | + |
1506 | |
1507 | === modified file 'openerp-runbot/openerprunbot/misc.py' |
1508 | --- openerp-runbot/openerprunbot/misc.py 2011-11-24 10:09:50 +0000 |
1509 | +++ openerp-runbot/openerprunbot/misc.py 2012-07-04 11:46:17 +0000 |
1510 | @@ -3,6 +3,7 @@ |
1511 | """ |
1512 | |
1513 | import fcntl |
1514 | +import re |
1515 | import signal |
1516 | import subprocess |
1517 | import threading |
1518 | @@ -10,7 +11,7 @@ |
1519 | import os |
1520 | |
1521 | __all__ = [ |
1522 | - 'kill', 'log', 'run', 'run_output' |
1523 | + 'kill', 'log', 'run', 'run_output', 'get_committer_info', 'underscore', 'has_test_enable_flag', 'project_name_from_unique_name' |
1524 | ] |
1525 | |
1526 | log_lock = threading.Lock() |
1527 | @@ -62,4 +63,56 @@ |
1528 | pass |
1529 | |
1530 | def run_output(l, cwd=None): |
1531 | + log("run_output",l) |
1532 | return subprocess.Popen(l, stdout=subprocess.PIPE, cwd=cwd).communicate()[0] |
1533 | + |
1534 | +#---------------------------------------------------------- |
1535 | +# OpenERP RunBot misc |
1536 | +#---------------------------------------------------------- |
1537 | + |
1538 | +def underscore(s): |
1539 | + return s.replace("~","").replace(":","_").replace("/","_").replace(".","_").replace(" ","_") |
1540 | + |
1541 | +def get_committer_info(repo_path): |
1542 | + committer_name = None |
1543 | + committer_xgram = None |
1544 | + committer_email = None |
1545 | + |
1546 | + output = run_output(["bzr", "log", "--long", "-r-1"], cwd=repo_path) |
1547 | + |
1548 | + committer_re = re.compile('committer: *(.+)<(.+)@(.+)>') |
1549 | + for i in output.split('\n'): |
1550 | + m = committer_re.match(i) |
1551 | + if m: |
1552 | + committer_name = m.group(1).strip() |
1553 | + committer_xgram = m.group(2) |
1554 | + committer_email = m.group(2) + '@' + m.group(3) |
1555 | + break |
1556 | + |
1557 | + return committer_name, committer_xgram, committer_email |
1558 | + |
1559 | +def has_test_enable_flag(server_bin_path): |
1560 | + """ |
1561 | + Test whether an openerp-server executable has the --test-enable flag. |
1562 | + (When the flag is present, testing for success/failure is done in a |
1563 | + different way.) |
1564 | + """ |
1565 | + p1 = subprocess.Popen([server_bin_path, "--help"], |
1566 | + stdout=subprocess.PIPE) |
1567 | + p2 = subprocess.Popen(["grep", "test-enable"], stdin=p1.stdout, |
1568 | + stdout=subprocess.PIPE) |
1569 | + p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. |
1570 | + output = p2.communicate()[0] |
1571 | + return output == " --test-enable Enable YAML and unit tests.\n" |
1572 | + |
1573 | +def project_name_from_unique_name(unique_name): |
1574 | + m = re.search("/(openobject|openerp)-(addons|server|client-web|web)/", unique_name) |
1575 | + if m: |
1576 | + n = m.group(2) |
1577 | + if n == 'client-web': |
1578 | + return 'web' |
1579 | + else: |
1580 | + return n |
1581 | + else: |
1582 | + return None |
1583 | + |
1584 | |
1585 | === modified file 'openerp-runbot/openerprunbot/templates/branches.html.mako' |
1586 | --- openerp-runbot/openerprunbot/templates/branches.html.mako 2012-05-23 13:07:13 +0000 |
1587 | +++ openerp-runbot/openerprunbot/templates/branches.html.mako 2012-07-04 11:46:17 +0000 |
1588 | @@ -106,11 +106,12 @@ |
1589 | <h2>Runtime data</h2> |
1590 | <div class="stats"> |
1591 | (The following data are global to the Runbot.)<br /> |
1592 | + Default job type: ${r.default_job_type}.<br /> |
1593 | Maximum ${r.number} concurrent branches.<br /> |
1594 | Maximum ${r.workers} concurrent jobs.<br /> |
1595 | ${r.current_job_id} processed jobs.<br /> |
1596 | ${r.workers - r.available_workers()} ongoing jobs.<br /> |
1597 | - ${r.manual_build_count} manual build requests to go.<br /> |
1598 | + ${r.next_build_number} manual build requests to go.<br /> |
1599 | </div> |
1600 | </div> |
1601 | |
1602 | |
1603 | === modified file 'openerp-runbot/openerprunbot/templates/defs.html.mako' |
1604 | --- openerp-runbot/openerprunbot/templates/defs.html.mako 2012-06-21 10:24:31 +0000 |
1605 | +++ openerp-runbot/openerprunbot/templates/defs.html.mako 2012-07-04 11:46:17 +0000 |
1606 | @@ -14,9 +14,9 @@ |
1607 | <span class="i action-toggle dropdown-toggle" data-toggle="dropdown">B</span> |
1608 | <ul class="dropdown-menu"> |
1609 | <li><a href="http://${r.domain}/${p.subdomain}/logs/test-all.txt"> |
1610 | - % if p.test_all_result==False: |
1611 | + % if p.test_result==False: |
1612 | Install logs (failure) |
1613 | - % elif p.test_all_result==True: |
1614 | + % elif p.test_result==True: |
1615 | Install logs (success) |
1616 | % else: |
1617 | Install logs (ongoing) |
1618 | @@ -30,25 +30,31 @@ |
1619 | % if 'build' in p.need_run_reason: |
1620 | <li>(Build manually requested)</li> |
1621 | % endif |
1622 | - % if p.manual_build != sys.maxint: |
1623 | - <li>(${p.manual_build})</li> |
1624 | + % if p.build_number != sys.maxint: |
1625 | + <li>(${p.build_number})</li> |
1626 | % endif |
1627 | </ul> |
1628 | % if p.state == 'testing': |
1629 | <p><span class="status"> </span> |
1630 | -% elif p.test_base_result==True and p.test_all_result==True: |
1631 | +% elif p.test_result==True: |
1632 | <p><span class="status green"> </span> |
1633 | -% elif p.test_base_result==False or p.test_all_result==False: |
1634 | +% elif p.test_result==False: |
1635 | <p><span class="status red"> </span> |
1636 | % else: |
1637 | <p><span class="status"> </span> |
1638 | % endif |
1639 | % if p.state == 'testing': |
1640 | <span class="testing">Testing...</span></p> |
1641 | -% elif p.running_t0 and p.test_all_result: |
1642 | - <span class="running-long">Age: ${r.nginx_index_time(t-p.running_t0)}</span></p> |
1643 | -% elif p.running_t0: |
1644 | - <span class="testing">Age: ${r.nginx_index_time(t-p.running_t0)}</span></p> |
1645 | +% elif p.state == 'pulling': |
1646 | + <span class="testing">Pulling...</span></p> |
1647 | +% elif p.state == 'allocated': |
1648 | + <span class="testing">Allocated...</span></p> |
1649 | +% elif p.state == 'broken': |
1650 | + <span class="testing">Internal error</span></p> |
1651 | +% elif p.running_since and p.test_result: |
1652 | + <span class="running-long">Age: ${r.nginx_index_time(t-p.running_since)}</span></p> |
1653 | +% elif p.running_since: |
1654 | + <span class="testing">Age: ${r.nginx_index_time(t-p.running_since)}</span></p> |
1655 | % else: |
1656 | <span class="testing">Internal error</span></p> |
1657 | % endif |
1658 | @@ -58,7 +64,7 @@ |
1659 | ${rev_(r,g,b,p)} |
1660 | % endfor |
1661 | </ul> |
1662 | -% if ((p.job_id > r.current_job_id - r.number) and p.running_t0) or g.is_sticky_job(p): |
1663 | +% if ((p.job_id > r.current_job_id - r.number) or g.is_sticky_job(p)) and p.running_since: |
1664 | <form target="_blank" method="GET" action="http://${p.db}.${r.domain}/"> |
1665 | <button type="submit">Connect</button> |
1666 | </form> |
1667 | @@ -69,7 +75,7 @@ |
1668 | </%def> |
1669 | |
1670 | <%def name="branch_(r,g,b)"> |
1671 | -% if b.overriden_repo_path: |
1672 | +% if not b.trigger_build: |
1673 | <span class="label"> |
1674 | % else: |
1675 | <span class="label notice"> |
1676 | @@ -97,8 +103,8 @@ |
1677 | % if g.wrong_matching: |
1678 | <span class="wrong-matching">These branches are not correctly named (<a href="/#how">see the instructions</a>).</span> |
1679 | % endif |
1680 | -% if g.manual_build != sys.maxint: |
1681 | -<p><span>(build n.${g.manual_build})</span></p> |
1682 | +% if g.build_number != sys.maxint: |
1683 | +<p><span>(build n.${g.build_number})</span></p> |
1684 | % elif not g.sticky: |
1685 | ${build_button_(r, g)} |
1686 | % endif |
1687 | @@ -126,16 +132,16 @@ |
1688 | <tr> |
1689 | <td> |
1690 | <form method="POST" action="http://${r.domain}/a?build=${g.name}&team=${g.team_name}"> |
1691 | -% if g.manual_build != sys.maxint: |
1692 | -<strong>${g.manual_build}.</strong> |
1693 | +% if g.build_number != sys.maxint: |
1694 | +<strong>${g.build_number}.</strong> |
1695 | % endif |
1696 | ${g.name} |
1697 | <span>(${g.team_name})</span> |
1698 | -% if g.manual_build == sys.maxint: |
1699 | +% if g.build_number == sys.maxint: |
1700 | <button type="submit">Force Build</button> |
1701 | % endif |
1702 | % for b in g.repo_updates(): |
1703 | - % if not b.overriden_repo_path: |
1704 | + % if b.trigger_build: |
1705 | ${branch_(r,g,b)} |
1706 | % endif |
1707 | % endfor |
1708 | |
1709 | === modified file 'openerp-runbot/openerprunbot/templates/nginx.conf.mako' |
1710 | --- openerp-runbot/openerprunbot/templates/nginx.conf.mako 2012-04-27 10:25:23 +0000 |
1711 | +++ openerp-runbot/openerprunbot/templates/nginx.conf.mako 2012-07-04 11:46:17 +0000 |
1712 | @@ -10,7 +10,7 @@ |
1713 | client_body_temp_path nginx; proxy_temp_path nginx; fastcgi_temp_path nginx; access_log nginx/access.log; index index.html; |
1714 | % for g in r.groups.values(): |
1715 | % for i in g.points: |
1716 | - % if i and (i.job_id > r.current_job_id - r.number) and i.running_t0: |
1717 | + % if i and (i.job_id > r.current_job_id - r.number or g.is_sticky_job(i)) and i.running_since: |
1718 | % if i.team_name == 'openerp-dev' and g.is_sticky_job(i): |
1719 | server { # openerp-dev sticky job (i.e. first job of a sticky branch) |
1720 | listen ${r.nginx_port}; |
1721 | |
1722 | === added file 'openerp-runbot/starting.html' |
1723 | --- openerp-runbot/starting.html 1970-01-01 00:00:00 +0000 |
1724 | +++ openerp-runbot/starting.html 2012-07-04 11:46:17 +0000 |
1725 | @@ -0,0 +1,1 @@ |
1726 | +The runbot is starting over. |
1727 | |
1728 | === added file 'openerp-runbot/stopped.html' |
1729 | --- openerp-runbot/stopped.html 1970-01-01 00:00:00 +0000 |
1730 | +++ openerp-runbot/stopped.html 2012-07-04 11:46:17 +0000 |
1731 | @@ -0,0 +1,1 @@ |
1732 | +The runbot is down for "planned" maintenance. |
1733 | |
1734 | === modified file 'openerp-runbot/try-template.py' |
1735 | --- openerp-runbot/try-template.py 2012-04-18 13:00:01 +0000 |
1736 | +++ openerp-runbot/try-template.py 2012-07-04 11:46:17 +0000 |
1737 | @@ -12,8 +12,8 @@ |
1738 | unique_name = '~openerp-dev/openobject-server/trunk-dummy' |
1739 | merge_count = 1 |
1740 | local_revision_count = 333 |
1741 | - overriden_repo_path = None |
1742 | - overriden_project_name = None |
1743 | + repo_path = None |
1744 | + project_name = None |
1745 | |
1746 | class Point(object): |
1747 | """ Dummy Point class to test mako templates. """ |
1748 | @@ -24,8 +24,8 @@ |
1749 | subdomain = 'trunk_dummy_2' |
1750 | port = 22 |
1751 | need_run_reason = ['server'] |
1752 | - manual_build = sys.maxint |
1753 | - running_t0 = 100000 |
1754 | + build_number = sys.maxint |
1755 | + running_since = 100000 |
1756 | repo_updates = [RunBotBranch()] |
1757 | version = 'trunk' |
1758 | |
1759 | @@ -33,7 +33,7 @@ |
1760 | """ Dummy Group class to test mako templates. """ |
1761 | name = 'trunk-dummy' |
1762 | team_name = 'openerp-dev' |
1763 | - manual_build = sys.maxint |
1764 | + build_number = sys.maxint |
1765 | wrong_matching = False |
1766 | points = [None, None, Point()] |
1767 | version = 'trunk' |
1768 | @@ -46,7 +46,7 @@ |
1769 | number = 555 |
1770 | workers = 666 |
1771 | current_job_id = 777 |
1772 | - manual_build_count = 888 |
1773 | + next_build_number = 888 |
1774 | server_net_port = 12000 |
1775 | server_xml_port = 12100 |
1776 | def nginx_index_time(self, t): |