Merge lp:~openerp-dev/openerp-tools/trunk-configured-branches-vmt into lp:openerp-tools

[Vo Minh Thu]

[lp:~openerp-dev/openerp-tools/trunk-configured-branches-vmt]

201. By OpenERP Online on 2012-07-04

[FIX] runbot: the completing branches were used in place of the original ones.

=== modified file 'openerp-runbot/openerp-runbot'
--- openerp-runbot/openerp-runbot	2012-04-19 15:34:05 +0000
+++ openerp-runbot/openerp-runbot	2012-07-04 11:46:17 +0000
@@ -57,6 +57,7 @@
     parser.add_option("--test", metavar="INT", default=1, help="run tests flag (%default)")
     parser.add_option("--workers", metavar="INT", default=4, help="number of workers (%default)")
     parser.add_option("--start-job-id", metavar="INT", default=0, help="initial job id (%default)")
+    parser.add_option("--job-type", metavar="JOB_TYPE", default='install_all', help="name of the default job type (%default)")
     parser.add_option("--debug", action="store_true", help="ease debugging by e.g. limiting branches")
     o, a = parser.parse_args(sys.argv)
     if o.init:
@@ -64,6 +65,9 @@
     elif o.clean:
         runbot_clean(o.dir)
     elif o.run:
+        assert o.job_type in openerprunbot.jobs.JOBS
+        path = os.path.dirname(sys.modules['__main__'].__file__)
+        openerprunbot.core.run(["cp", os.path.join(path, "starting.html"), "static/index.html"])
         openerprunbot.server.read_state()
         openerprunbot.server.start_server(int(o.nginx_port)-1)
         server_net_port = int(o.nginx_port) + int(o.number) * 2
@@ -76,10 +80,11 @@
         r = openerprunbot.core.RunBot(o.dir, o.poll, server_net_port,
             server_xml_port, client_web_port, o.number, o.nginx_port,
             o.nginx_domain, o.test, o.workers, int(o.start_job_id),
-            o.debug, lp)
+            o.debug, lp, o.job_type)
         runbot_kill_msg()
         r.loop()
         runbot_kill_msg()
+        openerprunbot.core.run(["cp", os.path.join(path, "stopped.html"), "static/index.html"])
         print "Last used job id:", r.current_job_id
     else:
         parser.print_help()
 
=== modified file 'openerp-runbot/openerprunbot/__init__.py'
--- openerp-runbot/openerprunbot/__init__.py	2011-12-06 23:17:14 +0000
+++ openerp-runbot/openerprunbot/__init__.py	2012-07-04 11:46:17 +0000
@@ -5,6 +5,7 @@
 import re
 
 from . import core
+from . import jobs
 from . import launchpad
 from . import misc
 from . import server
 
=== modified file 'openerp-runbot/openerprunbot/core.py'
--- openerp-runbot/openerprunbot/core.py	2012-05-07 07:21:32 +0000
+++ openerp-runbot/openerprunbot/core.py	2012-07-04 11:46:17 +0000
@@ -16,6 +16,7 @@
 
 import openerprunbot
 from openerprunbot.misc import *
+from openerprunbot.jobs.install_all import InstallAllJob
 
 # Hard-coded branches we always want, used to complement monitored branches
 # (i.e. to create branch groups).
@@ -31,67 +32,36 @@
     'web_trunk': '~openerp/openerp-web/trunk',
 }
 
+migration_scripts_branch = None
+migration_platform_branch = None
+
 # Number of build 'slots' per branch group.
 POINTS = 5
 
+# Job states
+STATE_ALLOCATED = 'allocated'
+STATE_BROKEN = 'broken'
+STATE_PULLING = 'pulling'
+STATE_RUNNING = 'running'
+STATE_TESTING = 'testing'
+
 # This constant matches community_dashboard.py.
 NEW_MERGE_STATUS = ['Needs review', 'Code failed to merge', 'Approved']
 
 #----------------------------------------------------------
-# OpenERP RunBot misc
-#----------------------------------------------------------
-
-def underscore(s):
-    return s.replace("~","").replace(":","_").replace("/","_").replace(".","_").replace(" ","_")
-
-def get_committer_info(repo_path):
-    committer_name = None
-    committer_xgram = None
-    committer_email = None
-
-    output = run_output(["bzr", "log", "--long", "-r-1"], cwd=repo_path)
-
-    committer_re = re.compile('committer: *(.+)<(.+)@(.+)>')
-    for i in output.split('\n'):
-        m = committer_re.match(i)
-        if m:
-            committer_name = m.group(1).strip()
-            committer_xgram = m.group(2)
-            committer_email = m.group(2) + '@' + m.group(3)
-            break
-
-    return committer_name, committer_xgram, committer_email
-
-def has_test_enable_flag(server_bin_path):
-    """
-    Test whether an openerp-server executable has the --test-enable flag.
-    (When the flag is present, testing for success/failure is done in a
-    different way.)
-    """
-    p1 = subprocess.Popen([server_bin_path, "--help"],
-        stdout=subprocess.PIPE)
-    p2 = subprocess.Popen(["grep", "test-enable"], stdin=p1.stdout,
-        stdout=subprocess.PIPE)
-    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
-    output = p2.communicate()[0]
-    return output == "    --test-enable       Enable YAML and unit tests.\n"
-
-#----------------------------------------------------------
 # OpenERP RunBot Branch
 #----------------------------------------------------------
 
 class RunBotBranch(object):
     def __init__(self, runbot, branch, group=None,
-        overriden_repo_path=None, overriden_project_name=None):
+        repo_path=None, project_name=None, trigger_build=False):
         """
         Represent a single Launchpad branch, e.g.
         `~openerp-dev/openobject-server/trunk-foo-bar`.
-
-        A branch has an overriden_repo_path when it is a trunk or 6.{0,1} branch
-        used to complement another 'real' branch.
         """
         self.runbot = runbot
         self.group = group
+        self.trigger_build = trigger_build
 
         # e.g. trunk
         self.name = branch.name
@@ -100,13 +70,10 @@
         self.unique_name = branch.unique_name
         self.unique_name_underscore = underscore(self.unique_name)
         # server, addons, or web (both for openobject-client-web and openerp-web)
-        if overriden_project_name:
-            self.project_name = overriden_project_name
+        if project_name:
+            self.project_name = project_name
         else:
-            self.project_name = re.search("/(openobject|openerp)-(addons|server|client-web|web)/",self.unique_name).group(2)
-        if self.project_name == 'client-web':
-            self.project_name = 'web'
-        assert self.project_name in ('server', 'addons', 'web')
+            self.project_name = project_name_from_unique_name(self.unique_name)
 
         self.lp_date_last_modified = branch.date_last_modified
         self.lp_revision_count = branch.revision_count
@@ -116,10 +83,10 @@
         self.merge_count = 0
 
         # Repository path <Root>/repo/<branchuniquename>
-        self.repo_path=os.path.join(self.runbot.wd,'repo',self.unique_name_underscore)
-        self.overriden_repo_path = overriden_repo_path
-        if overriden_repo_path:
-            self.repo_path=overriden_repo_path
+        if repo_path:
+            self.repo_path=repo_path
+        else:
+            self.repo_path=os.path.join(self.runbot.wd,'repo',self.unique_name_underscore)
 
         self.committer_name = None
         self.committer_xgram = None
@@ -134,7 +101,7 @@
         if self.local_revision_count != self.lp_revision_count:
             self.local_date_last_modified = self.lp_date_last_modified
             self.local_revision_count = self.lp_revision_count
-            if not self.overriden_repo_path or self.group.is_configured():
+            if self.trigger_build:
                 name = self.project_name
                 self.group.need_run_reason.append(name)
 
@@ -142,50 +109,87 @@
 # OpenERP RunBot Grouped Branch
 #----------------------------------------------------------
 
-class RunBotGroupedBranchBase(object):
-    def __init__(self, runbot, sticky):
+class RunBotGroupedBranch(object):
+    """
+    A single 'branch group' represents a single branch name for all our
+    projects. E.g. trunk is available in server, addons, web, and client-web.
+
+    When the corresponding project doesn't have that branch, trunk is used.
+    For instance, a branch group can represent
+        `~openerp-dev/openobject-server/trunk-foo-bar`
+        `~openerp-dev/openobject-addons/trunk-foo-bar`
+    and
+        `~openerp/openobject-web-client/6.0`
+        `~openerp/openerp-web/trunk`
+    will be used to complete the group, building the four branches together.
+
+    Alternatively a 'branch group' can be configured. In that case, all
+    branches are manually specified; no attempt is done to complete the group.
+    """
+    def __init__(self, runbot, team_name, name, version, sticky,
+            server_branch=None, client_web_branch=None,
+            web_branch=None, addons_branches=None, modules=None, job_type=None):
         self.runbot = runbot
         self.sticky = sticky
+        self.job_type = job_type
+
         # Points are build 'slots'
         self.points = [None for x in xrange(POINTS)]
+
+        self.team_name = team_name
+        self.name = name # group name or manually chosen name.
+        self.name_underscore = underscore(self.name)
+        self.version = version # '6.0', '6.1',  or 'trunk'
+
         # Reason to do a build, e.g. because 'addons' was commited to.
         self.need_run_reason = []
-        self.version = None
+
+        self.server_branch = server_branch
+        self.web_branch = client_web_branch or web_branch
+        self.addons_branches = addons_branches
+
+        self.server = None
+        self.web = None
+        self.addons = []
+
+        self.json_path=os.path.join(runbot.wd,'static',"%s-%s.json"%(team_name, self.name.replace('_','-').replace('.','-')))
+
+        # List of modules to install instead of `all`.
+        self.modules = modules.split(',') if modules else []
+
+        self.configured = False
+
+        # A normal build_number is maxint, a build_number is assigned when a
+        # build is manually requested or a triggering branch has a new revision.
+        # The build number is used to assign a position in the queue.
+        # The value will be reset to maxint after the build is done.
+        self.build_number = sys.maxint
+
         # True when it is not possible to provide default branches based on
         # the name.
         self.wrong_matching = False
 
-        # A normal manual_build is maxint, a manual_build is used only when
-        # manually requesting a build with a 'build' command pushed in the
-        # queue. The value will be reset to maxint after the build is done.
-        self.manual_build = sys.maxint
-
-        # List of modules to install instead of `all`.
-        self.modules = []
-
     def add_point(self, j):
-        assert not j.completed
-        p = Point(self, j)
-        self.points = self.points[1:] + [p] # TODO delete db and on-disk data
+        assert j.state == STATE_ALLOCATED
+        self.points = self.points[1:] + [j] # TODO delete db and on-disk data
 
     def complete_point(self, j):
-        assert j.completed
+        assert j.state in (STATE_RUNNING, STATE_BROKEN)
         for p in self.points + [None]:
             if p and p.job_id == j.job_id:
                 break
-        if p and p.state != 'running':
-            p.update(j)
+        if p and p.state != j.state:
             p.save_json()
 
     def all_points_completed(self):
         for p in self.points:
-            if p is not None and p.state != 'running':
+            if p is not None and p.state not in (STATE_RUNNING, STATE_BROKEN):
                 return False
         return True
 
     def is_sticky_job(self, j):
         """ Return True if the job is the latest running point of a sticky branch. """
-        runnings = filter(lambda x: x and x.running_t0, self.points)
+        runnings = filter(lambda x: x and x.state == STATE_RUNNING, self.points)
         return self.sticky and runnings and runnings[-1].job_id == j.job_id
 
     def update_state(self, state):
@@ -198,43 +202,33 @@
         if self.sticky != previous:
             log("stickiness changed", team=self.team_name, branch=self.name, now=self.sticky)
 
-    def is_configured(self):
-        return isinstance(self, ConfiguredGroup)
-
-class ConfiguredGroup(RunBotGroupedBranchBase):
-    def __init__(self, runbot, team_name, name, version, sticky,
-            server_branch=None, client_web_branch=None,
-            web_branch=None, addons_branches=None, modules=None):
-        super(ConfiguredGroup, self).__init__(runbot, sticky)
-        self.team_name = team_name
-        self.name = name # Unique manually-chosen name, not necessarily the group name.
-        self.name_underscore = underscore(self.name)
-        self.version = version # '6.0', '6.1',  or 'trunk'
-
-        self.server_branch = server_branch
-        self.web_branch = client_web_branch or web_branch
-        self.addons_branches = addons_branches
-
-        self.server = None
-        self.web = None
-        self.addons = []
-
-        self.json_path=os.path.join(runbot.wd,'static',"%s-%s.json"%(team_name, self.name.replace('_','-').replace('.','-')))
-
-        self.modules = modules.split(',') if modules else []
+    def repo_updates(self):
+        return [copy.copy(x) for x in self.all_branches()]
+
+    def all_branches(self):
+        r = []
+        r.append(self.server)
+        r.extend([x for x in self.addons])
+        r.append(self.web)
+        if hasattr(self, 'migration_script'):
+            r.extend([self.migration_script, self.migration_platform])
+        return r
+
+    def lp_date_last_modified(self):
+        return max(p.lp_date_last_modified for p in self.all_branches() if p and p.trigger_build)
 
     def is_ok(self):
         """
-        Test whether the group is useable, i.e. if self.populate_branches()
+        Test whether the group is useable, i.e. if self.configure_branches()
         didn't early return.
         """
-        if self.server and self.web and self.addons:
-            return True
-        else:
-            return False
+        return self.server and self.web and self.addons
 
-    def populate_branches(self, launchpad):
+    # TODO all the xxxx_branch argument in __init__ can be moved here.
+    def configure_branches(self, launchpad):
+        """ Fetch the configured branches. """
         log("runbot-populate-configured-branches")
+        self.configured = True
         repo = os.path.join(self.runbot.wd, 'repo')
         filters = {'status': NEW_MERGE_STATUS}
 
@@ -243,7 +237,7 @@
         if not b:
             log("WARNING:no such unique name", name=self.server_branch)
             return
-        self.server = RunBotBranch(self.runbot, b, group=self, overriden_repo_path=path, overriden_project_name='server')
+        self.server = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='server', trigger_build=True)
         self.server.update_launchpad(b)
         self.server.merge_count = len(list(b.getMergeProposals(**filters)))
 
@@ -253,7 +247,7 @@
             if not b:
                 log("WARNING:no such unique name", name=self.web_branch)
                 return
-            self.web = RunBotBranch(self.runbot, b, group=self, overriden_repo_path=path, overriden_project_name='web')
+            self.web = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='web', trigger_build=True)
             self.web.update_launchpad(b)
             self.web.merge_count = len(list(b.getMergeProposals(**filters)))
         else:
@@ -267,59 +261,13 @@
                 log("WARNING:no such unique name", name=self.addons_branches[x])
                 self.addons = []
                 return
-            bb = RunBotBranch(self.runbot, b, group=self, overriden_repo_path=path, overriden_project_name='addons')
+            bb = RunBotBranch(self.runbot, b, group=self, repo_path=path, project_name='addons', trigger_build=True)
             bb.update_launchpad(b)
             bb.merge_count = len(list(b.getMergeProposals(**filters)))
             self.addons.append(bb)
 
-    def repo_updates(self):
-        r = []
-        r.append(copy.copy(self.server))
-        for x in self.addons:
-            r.append(copy.copy(x))
-        r.append(copy.copy(self.web))
-        return r
-
-    def lp_date_last_modified(self):
-        lp_date_last_modified = None
-        for p in [self.server, self.web] + self.addons:
-            if p and (not lp_date_last_modified or p.lp_date_last_modified > lp_date_last_modified):
-                lp_date_last_modified = p.lp_date_last_modified
-        assert lp_date_last_modified # at least one project in the group
-        return lp_date_last_modified
-
-class RunBotGroupedBranch(RunBotGroupedBranchBase):
-    """
-    A single 'branch group' represents a single branch name for all our
-    projects. E.g. trunk is available in server, addons, web, and client-web.
-
-    When the corresponding project doesn't have that branch, trunk is used.
-    For instance, a branch group can represent
-        `~openerp-dev/openobject-server/trunk-foo-bar`
-        `~openerp-dev/openobject-addons/trunk-foo-bar`
-    and
-        `~openerp/openobject-web-client/6.0`
-        `~openerp/openerp-web/trunk`
-    will be used to complete the group, building the four branches together.
-
-    Alternatively a 'branch group' can be configured. In that case, all
-    branches are manually specified; no attempt is done to complete the group.
-    """
-
-    def __init__(self, runbot, b, team_name, sticky):
-        """A grouped branch can be created from any branch."""
-        super(RunBotGroupedBranch, self).__init__(runbot, sticky)
-        self.team_name = team_name
-        self.name = b.name
-        self.name_underscore = underscore(self.name)
-        self.server = None
-        self.addons = [] # Only one branch in a non-configured group.
-        self.web = None
-        self.json_path=os.path.join(runbot.wd,'static',"%s-%s.json"%(team_name, self.name.replace('_','-').replace('.','-')))
-
-        self.add_branch(b)
-
     def add_branch(self, b):
+        """ Add a single branch (when not using configure_branches(). """
         assert b.name == self.name
         b.group = self
         if b.project_name == 'addons':
@@ -363,403 +311,28 @@
         for p in ('server', 'web'):
             if not getattr(self, p):
                 b = self.runbot.main_branches[p+ending]
-                setattr(self, p, RunBotBranch(self.runbot,b,group=self,overriden_repo_path=paths[p]))
+                setattr(self, p, RunBotBranch(self.runbot,b,group=self,repo_path=paths[p]))
                 getattr(self, p).update_launchpad(b)
-            elif getattr(self, p).overriden_repo_path:
+            elif not getattr(self, p).trigger_build:
                 b = self.runbot.main_branches[p+ending]
                 getattr(self, p).update_launchpad(b)
         if not self.addons:
             b = self.runbot.main_branches['addons'+ending]
-            self.addons = [RunBotBranch(self.runbot,b,group=self,overriden_repo_path=paths['addons'])]
+            self.addons = [RunBotBranch(self.runbot,b,group=self,repo_path=paths['addons'])]
             self.addons[0].update_launchpad(b)
-        elif self.addons[0].overriden_repo_path:
+        elif not self.addons[0].trigger_build:
             b = self.runbot.main_branches['addons'+ending]
             self.addons[0].update_launchpad(b)
 
-    def repo_updates(self):
-        r = []
-        r.append(copy.copy(self.server))
-        r.append(copy.copy(self.addons[0]))
-        r.append(copy.copy(self.web))
-        return r
-
-    def lp_date_last_modified(self):
-        lp_date_last_modified = None
-        for p in ('server', 'web'):
-            if getattr(self, p) and not getattr(self, p).overriden_repo_path \
-                and (not lp_date_last_modified or getattr(self, p).lp_date_last_modified > lp_date_last_modified):
-                lp_date_last_modified = getattr(self, p).lp_date_last_modified
-        if self.addons and not self.addons[0].overriden_repo_path \
-            and (not lp_date_last_modified or self.addons[0].lp_date_last_modified > lp_date_last_modified):
-            lp_date_last_modified = self.addons[0].lp_date_last_modified
-        assert lp_date_last_modified # at least one project in the group
-        return lp_date_last_modified
-
-#----------------------------------------------------------
-# OpenERP RunBot Worker
-#----------------------------------------------------------
-
-class Job(object):
-    """
-    A Job encapsulates all the necessary data to build a branch group for a
-    given slot. The build is done in its own worker thread.
-    """
-
-    def __init__(self, g, port, test, job_id, debug):
-        self.job_id = job_id
-        self.completed = False
-        self.team_name = g.team_name
-        self.name = g.name
-        self.name_underscore = g.name_underscore
-        self.version = g.version
-        self.debug = debug
-
-        self.repo_updates = g.repo_updates()
-        self.port = port
-        self.test = test
-        self.running_server_pid = 0
-        self.client_web_pid = 0
-        self.running_t0=0
-
-        repo = os.path.join(g.runbot.wd,'repo')
-        self.server_src = g.server.repo_path
-        self.client_web_src = g.web.repo_path if g.version == '6.0' and g.web else None
-        self.web_src = g.web.repo_path if g.web else None
-
-        # if addons is not the full addons branch use trunk
-        self.addons_src = [a.repo_path for a in g.addons]
-
-        # Running path <Root>/static/<domain>
-        self.subdomain = "%s-%s-%s"%(self.team_name, self.name.replace('_','-').replace('.','-'),self.job_id)
-        self.running_path = os.path.join(g.runbot.wd,'static',self.subdomain)
-        self.json_path = g.json_path
-        self.log_path = os.path.join(self.running_path,'logs')
-        self.flags_path = os.path.join(self.running_path,'flags')
-
-        # Server
-        self.server_path=os.path.join(self.running_path,"server")
-        self.server_bin_path=os.path.join(self.server_path,"openerp-server")
-        self.server_log_path=os.path.join(self.log_path,'server.txt')
-        self.server_log_base_path=os.path.join(self.log_path,'test-base.txt')
-        self.server_log_all_path=os.path.join(self.log_path,'test-all.txt')
-
-        # coverage
-        self.coverage_file_path=os.path.join(self.log_path,'coverage.pickle')
-        self.coverage_base_path=os.path.join(self.log_path,'coverage-base')
-        self.coverage_all_path=os.path.join(self.log_path,'coverage-all')
-
-        # Web60
-        self.client_web_pid=None
-        self.client_web_path=os.path.join(self.running_path,"client-web")
-        self.client_web_bin_path=os.path.join(self.client_web_path,"openerp-web.py")
-        self.client_web_doc_path=os.path.join(self.client_web_path,"doc")
-        self.client_web_log_path=os.path.join(self.log_path,'client-web.txt')
-
-        # test
-        self.test_base_result=None
-        self.test_base_path=os.path.join(self.log_path,'test-base.txt')
-        self.test_all_result=None
-        self.test_all_path=os.path.join(self.log_path,'test-all.txt')
-
-        self.server_net_port=g.runbot.server_net_port
-        self.server_xml_port=g.runbot.server_xml_port
-        self.client_web_port=g.runbot.client_web_port
-
-        self.db = self.name_underscore + '_' + str(self.job_id)
-        self.db_all = "%s_all" % self.db
-
-        for p in ('addons', 'server', 'web'):
-            setattr(self, p + '_committer_name', None)
-            setattr(self, p + '_committer_xgram', None)
-            setattr(self, p + '_committer_email', None)
-        self.start_time = 0
-        self.completed_time = 0
-
-        self.modules = g.modules
-
-    def spawn(self):
-        log("runbot-spawn-worker-" + str(self.job_id), group=self.name)
-        t = threading.Thread(target=self.work, name=('runbot-group-worker-' + str(self.job_id)))
-        t.daemon = True
-        t.start()
-
-    def work(self):
-        try:
-            self.pull_branches()
-            self.start_time = time.time()
-            self.start()
-            self.completed_time = time.time()
-            self.completed = True
-            log("runbot-end-worker", job=self.name)
-        except Exception, e:
-            self.completed = True
-            log("runbot-end-worker [with exception]", job=self.name)
-            print traceback.format_exc()
-
-    def pull_branches(self):
-        for b in self.repo_updates:
-            log("branch-update",branch=b.unique_name)
-            if os.path.exists(b.repo_path):
-                run(["bzr", "pull", "-d", b.repo_path, "--overwrite"])
-            else:
-                run(["bzr", "branch", "lp:%s"%b.unique_name, b.repo_path])
-            run(["bzr", "update", "-r", str(b.local_revision_count), b.repo_path])
-
-            committer_name, committer_xgram, committer_email = \
-                get_committer_info(b.repo_path)
-            b.committer_name = committer_name
-            b.committer_xgram = committer_xgram
-            b.committer_email = committer_email
-
-    def start_rsync(self):
-        log("job-start-rsync",branch=self.name)
-        for i in [self.running_path,self.log_path,self.flags_path,self.client_web_doc_path]:
-            if not os.path.exists(i):
-                os.makedirs(i)
-        # copy server
-        run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.server_src, self.server_path])
-
-        # copy addons (6.0 uses bin/addons, 6.1 and trunk use openerp/addons)
-        addons_dest = "openerp/addons"
-        if not os.path.exists(os.path.join(self.server_path,addons_dest)):
-            addons_dest = "bin/addons"
-        addons_path = os.path.join(self.server_path,addons_dest)
-        for a in self.addons_src:
-            run(["rsync","-a","--exclude",".bzr", "%s/"%a, addons_path])
-        if self.web_src and self.version != '6.0':
-            run(["rsync","-a","--exclude",".bzr", "%s/addons/"%self.web_src, addons_path])
-
-        # copy web-client
-        if self.client_web_src:
-            run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.client_web_src, self.client_web_path])
-
-    def start_createdb(self):
-        run(["psql","template1","-c","select pg_terminate_backend(procpid) from pg_stat_activity where datname in ('%s','%s')"%(self.db,self.db_all)])
-        time.sleep(3)
-        run(["dropdb",self.db])
-        run(["dropdb",self.db_all])
-        run(["createdb",self.db])
-        run(["createdb",self.db_all])
-
-    def run_log(self, cmd, logfile, env=None):
-        env = dict(os.environ, **env) if env else None
-        log("run", *cmd, logfile=logfile)
-        out=open(logfile,"w")
-        p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True, env=env)
-        self.running_server_pid=p.pid
-        p.communicate()
-        return p
-
-    def resolve_server_bin_path(self):
-        # This can be done only if the files are present otherwise the if
-        # will always fail. Alternatively, server_bin_path could be a property.
-        if not os.path.exists(self.server_bin_path): # for 6.0 branches
-            self.server_bin_path=os.path.join(self.server_path,"bin","openerp-server.py")
-
-    def start_test_base(self):
-        log("job-start-server-base")
-        cmd = [self.server_bin_path,"-d",self.db,"-i","base","--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"]
-        _has_test_enable_flag = False
-        if has_test_enable_flag(self.server_bin_path):
-            cmd.append("--test-enable")
-            _has_test_enable_flag = True
-        cmd = ["coverage","run","--branch"] + cmd
-        self.run_log(cmd, logfile=self.test_base_path,env={'COVERAGE_FILE': self.coverage_file_path})
-        run(["coverage","html","-d",self.coverage_base_path,"--ignore-errors","--include=*.py"],env={'COVERAGE_FILE': self.coverage_file_path})
-        if _has_test_enable_flag:
-            success_message = "openerp.modules.loading: Modules loaded."
-            rc = not bool(run(["grep",success_message,self.test_base_path]))
-        else:
-            rc = bool(run(["grep","Traceback",self.test_base_path]))
-        self.test_base_result = rc
-
-    def start_test_all(self):
-        log("job-start-server-all")
-        mods = []
-        if not os.path.exists(os.path.join(self.server_path,"openerp/addons")):
-            mods = os.listdir(os.path.join(self.server_path,"bin/addons"))
-        elif os.path.exists(os.path.join(self.server_path,"openerp/addons")):
-            mods = os.listdir(os.path.join(self.server_path,"openerp/addons"))
-        bl = [
-            'document_ftp', 'l10n_lu',
-            '.bzrignore', '__init__.pyc', '__init__.py', 'base_quality_interrogation.py',
-        ]
-        mods = [m for m in mods if m not in bl]
-        if self.modules:
-            # TODO feedback when a requested modules does'nt exist.
-            mods = [m for m in mods if m in self.modules]
-        mods = ",".join(mods)
-        cmd = [self.server_bin_path,"-d",self.db_all,"-i",mods,"--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"]
-        _has_test_enable_flag = False
-        if has_test_enable_flag(self.server_bin_path):
-            cmd.append("--test-enable")
-            _has_test_enable_flag = True
-        cmd = ["coverage","run","--branch"] + cmd
-        self.run_log(cmd, logfile=self.test_all_path)
-        run(["coverage","html","-d",self.coverage_all_path,"--ignore-errors","--include=*.py"])
-        if _has_test_enable_flag:
-            success_message = "openerp.modules.loading: Modules loaded."
-            rc = not bool(run(["grep",success_message,self.test_all_path]))
-        else:
-            rc = bool(run(["grep","Traceback",self.test_all_path]))
-        self.test_all_result = rc
-
-    def start_server(self):
-        port = self.port
-        log("job-start-server",branch=self.name,port=port)
-        cmd=[self.server_bin_path,"--no-xmlrpcs","--netrpc-port=%d"%(self.server_net_port+port),"--xmlrpc-port=%d"%(self.server_xml_port+port)]
-        if os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi.py')) \
-            or os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi', 'core.py')):
-            cmd+=["--db-filter=%d(_.*)?$","--load=web"]
-        log("run",*cmd,log=self.server_log_path)
-        out=open(self.server_log_path,"w")
-        p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True)
-        self.running_server_pid=p.pid
-
-    def start_client_web(self):
-        if not self.client_web_src:
-            return
-        port = self.port
-        log("job-start-client-web",branch=self.name,port=port)
-        config="""
-        [global]
-        server.environment = "development"
-        server.socket_host = "0.0.0.0"
-        server.socket_port = %d
-        server.thread_pool = 10
-        tools.sessions.on = True
-        log.access_level = "INFO"
-        log.error_level = "INFO"
-        tools.csrf.on = False
-        tools.log_tracebacks.on = False
-        tools.cgitb.on = True
-        openerp.server.host = 'localhost'
-        openerp.server.port = '%d'
-        openerp.server.protocol = 'socket'
-        openerp.server.timeout = 450
-        [openerp-web]
-        dblist.filter = 'BOTH'
-        dbbutton.visible = True
-        company.url = ''
-        openerp.server.host = 'localhost'
-        openerp.server.port = '%d'
-        openerp.server.protocol = 'socket'
-        openerp.server.timeout = 450
-        """%(self.client_web_port+port,self.server_net_port+port,self.server_net_port+port)
-        config=config.replace("\n        ","\n")
-        cfgs = [os.path.join(self.client_web_path,"doc","openerp-web.cfg"), os.path.join(self.client_web_path,"openerp-web.cfg")]
-        for i in cfgs:
-            f=open(i,"w")
-            f.write(config)
-            f.close()
-
-        cmd=[self.client_web_bin_path]
-        log("run",*cmd,log=self.client_web_log_path)
-        out=open(self.client_web_log_path,"w")
-        p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True)
-        self.client_web_pid=p.pid
-
-    def start(self):
-        log("job-start",branch=self.name,port=self.port)
-        self.start_rsync()
-        self.resolve_server_bin_path()
-        self.start_createdb()
-        try:
-            if self.test:
-                self.start_test_base()
-                if not self.debug:
-                    self.start_test_all()
-                else:
-                    self.test_all_result = True
-            self.start_server()
-            self.start_client_web()
-        except OSError,e:
-            log("branch-start-error",exception=e)
-        except IOError,e:
-            log("branch-start-error",exception=e)
-        self.running_t0=time.time()
-        log("branch-started",branch=self.name,port=self.port)
-
-    def stop(self):
-        log("Stopping job", id=self.job_id, branch=self.name)
-        if self.running_server_pid:
-            kill(self.running_server_pid)
-        if self.client_web_pid:
-            kill(self.client_web_pid)
-
-#----------------------------------------------------------
-# OpenERP RunBot Build Slot
-#----------------------------------------------------------
-
-class Point(object):
-    """A point is a build slot and associated to a worker thread."""
-    def __init__(self, g, j):
-        """Create a Point from a given group and job."""
-        self.version = g.version
-        self.port = j.port
-        self.job_id = j.job_id
-        self.team_name = j.team_name
-        self.name_underscore = j.name_underscore
-        self.db = j.db
-        self.running_path = j.running_path
-        self.json_path = j.json_path
-        self.subdomain = j.subdomain
-        self.repo_updates = j.repo_updates # committer date not yet available, available after j.work() is called.
-        self.server_rev = g.server.local_revision_count
-        self.addons_rev = [a.local_revision_count for a in g.addons]
-        self.web_rev = g.web.local_revision_count if g.web else 0
-        self.need_run_reason = g.need_run_reason
-        for p in ('addons', 'server', 'web'):
-            setattr(self, p + '_committer_name', getattr(j, p + '_committer_name'))
-            setattr(self, p + '_committer_xgram', getattr(j, p + '_committer_xgram'))
-            setattr(self, p + '_committer_email', getattr(j, p + '_committer_email'))
-        self.update(j)
-        self.manual_build = g.manual_build
-
-    def update(self, j):
-        """
-        Update the Point from a job. The job should be the same than the one
-        used in `__init__()`.
-        """
-        self.state = 'running' if j.completed else 'testing'
-        self.running_t0 = j.running_t0
-        self.test_base_result = j.test_base_result
-        self.test_all_result = j.test_all_result
-
-    def save_json(self):
-        """Append the point data to a JSON file for posterity."""
-        # A job must be saved only once.
-        if hasattr(self, 'json_saved'):
-            log("=== save_json() called more than once. ===", job_id=self.job_id)
-        self.json_saved = True
-
-        path = self.json_path
-        state = {}
-
-        if os.path.exists(path):
-            with open(path, 'r') as h:
-                state = simplejson.loads(h.read())
-
-        value = {}
-        committers = []
-        for p in ('addons', 'server', 'web'):
-            committers += [
-                p + '_committer_name',
-                p + '_committer_xgram',
-                # p + '_committer_email',
-            ]
-        for a in [
-            'job_id', 'db', 'running_path', 'subdomain', 'server_rev',
-            'addons_rev', 'web_rev', 'need_run_reason',
-            'test_base_result', 'test_all_result',
-            ] + committers:
-            value[a] = getattr(self, a)
-        state.setdefault('jobs', [])
-        state['jobs'].append(value)
-
-        with open(path, 'w') as h:
-            data = simplejson.dumps(state, sort_keys=True, indent=4)
-            h.write(data)
+        if self.job_type == 'migrate_script':
+            if not hasattr(self, 'migration_script'):
+                self.migration_script = RunBotBranch(self.runbot,
+                    migration_scripts_branch, group=self, repo_path='repo/openerp_migration_script', project_name='migration-scripts', trigger_build=True)
+                self.migration_platform = RunBotBranch(self.runbot,
+                    migration_platform_branch, group=self, repo_path='repo/openerp_migration_platform', project_name='migration-platform', trigger_build=True)
+            self.migration_script.update_launchpad(migration_scripts_branch)
+            self.migration_platform.update_launchpad(migration_platform_branch)
+            print "initialized group with job type <migration_script>"
 
 #----------------------------------------------------------
 # OpenERP RunBot Engine
@@ -769,7 +342,7 @@
     """Used as a singleton, to manage almost all the Runbot state and logic."""
     def __init__(self, wd, poll, server_net_port, server_xml_port,
         client_web_port,  number, nginx_port, domain, test, workers,
-        current_job_id, debug, lp):
+        current_job_id, debug, lp, default_job_type='install_all'):
         self.wd=wd
         self.server_net_port=int(server_net_port)
         self.server_xml_port=int(server_xml_port)
@@ -788,9 +361,10 @@
         # realizing a queue of groups to be processed).
         self.current_job_id = current_job_id
         self.debug = debug
-        self.manual_build_count = 0
+        self.next_build_number = 0
         self.launchpad = lp
         self.checked_date_last_modified = None
+        self.default_job_type = default_job_type
 
     def registered_teams(self):
         return openerprunbot.state.get('registered-teams', []) + ['openerp-dev']
@@ -833,7 +407,7 @@
         return gs
 
     def nginx_groups_registered(self,team_name):
-        gs = [(g.name,g) for g in self.groups.values() if not any(g.points) and g.team_name==team_name and g.manual_build == sys.maxint]
+        gs = [(g.name,g) for g in self.groups.values() if not any(g.points) and g.team_name==team_name and g.build_number == sys.maxint]
         gs.sort()
         return [g for (x,g) in gs]
 
@@ -897,26 +471,28 @@
 
     def process_add(self, b, sticky, team_name):
         log("runbot-process-add", b.unique_name)
-        if not re.search("/(openobject|openerp)-(addons|server|client-web|web)/", b.unique_name):
+        if not project_name_from_unique_name(b.unique_name):
             log("WARNING: can't add branch: project name is not standard.", unique_name=b.unique_name)
             return
         if b.unique_name not in self.branches:
-            self.branches[b.unique_name] = RunBotBranch(self, b)
+            self.branches[b.unique_name] = RunBotBranch(self, b, trigger_build=True)
         bb = self.branches[b.unique_name]
         if (team_name, bb.name) not in self.groups:
-            self.groups[(team_name, bb.name)] = RunBotGroupedBranch(self, bb, team_name, sticky)
-        else:
-            self.groups[(team_name, bb.name)].add_branch(bb)
+            self.groups[(team_name, bb.name)] = RunBotGroupedBranch(self, team_name, bb.name, None, sticky, job_type=self.default_job_type)
+        self.groups[(team_name, bb.name)].add_branch(bb)
         bb.update_launchpad(b)
         filters = {'status': NEW_MERGE_STATUS}
-        bb.merge_count = len(list(b.getMergeProposals(**filters)))
+        try:
+            bb.merge_count = len(list(b.getMergeProposals(**filters)))
+        except Exception, e:
+            bb.merge_count = 0
 
     def register_configured_branches(self):
         for team, v in openerprunbot.state.get('configured-branches', {}).items():
             if not team: continue
             for name, c in v.items():
                 if not name: continue
-                g = ConfiguredGroup(self, team, name, c['version'], 0,
+                g = RunBotGroupedBranch(self, team, name, c['version'], 0,
                     server_branch=c['server_branch'],
                     client_web_branch=c.get('client_web_branch'),
                     web_branch=c.get('web_branch'),
@@ -925,7 +501,7 @@
 
                 # TODO or if it is already there but with different branches.
                 if (g.team_name, g.name) not in self.groups:
-                    g.populate_branches(self.launchpad)
+                    g.configure_branches(self.launchpad)
                     if g.is_ok():
                         log("adding configured group", team=g.team_name, name=g.name)
                         self.groups[(g.team_name, g.name)] = g
@@ -936,6 +512,11 @@
     def populate_branches(self):
         """Return all LP branches matching our teams and projects."""
         log("runbot-populate-branches")
+        # Get migration branches
+        global migration_scripts_branch
+        global migration_platform_branch
+        migration_scripts_branch = self.launchpad.get_branch(unique_name='~openerp-dev/openerp-int/migration-scripts')
+        migration_platform_branch = self.launchpad.get_branch(unique_name='~openerp-dev/openerp-int/migration-platform')
         # Register main sticky branches
         for k, v in MAIN_BRANCHES.items():
             b = self.launchpad.get_branch(unique_name=v)
@@ -944,6 +525,7 @@
 
         # Register other branches
         for v in openerprunbot.state.get('registered-branches', []):
+            break
             m = openerprunbot.branch_input_re.match(v)
             if m:
                 b = self.launchpad.get_branch(unique_name=v)
@@ -961,26 +543,19 @@
             if self.debug: break
             team_branches = self.launchpad.get_team_branches(team_name)
             for b in team_branches:
-                if re.search("/(openobject|openerp)-(addons|server|client-web|web)/",b.unique_name):
+                if project_name_from_unique_name(b.unique_name):
                     self.process_add(b, 0, team_name)
 
         # Register configured branches
         self.register_configured_branches()
 
         for g in self.groups.values():
-            if not g.is_configured():
+            if not g.configured:
                 g.add_main_branches()
 
         self.checked_date_last_modified = max(x.lp_date_last_modified() for x in self.groups.values())
         self.assign_positions()
         return self.get_queue()
-        #return self.sorted_groups()
-
-    def sorted_groups(self):
-        gs = [(g.sticky, -g.manual_build, g.lp_date_last_modified(), g) for g in self.groups.values()]
-        gs.sort(reverse=1)
-        gs = [g for (x,y,z,g) in gs][:self.number]
-        return gs
 
     def assign_positions(self):
         # Sort by last modification date, then assign a position in the queue.
@@ -992,13 +567,13 @@
                 continue
             if not g.sticky:
                 self.checked_date_last_modified = g.lp_date_last_modified()
-            if g.need_run_reason and g.manual_build == sys.maxint:
-                self.manual_build_count +=1
-                g.manual_build = self.manual_build_count
+            if g.need_run_reason and g.build_number == sys.maxint:
+                self.next_build_number +=1
+                g.build_number = self.next_build_number
 
     def get_queue(self):
-        gs = sorted(self.groups.values(), key=lambda x: x.manual_build)
-        return filter(lambda g: g.manual_build != sys.maxint, gs)
+        gs = sorted(self.groups.values(), key=lambda x: x.build_number)
+        return filter(lambda g: g.build_number != sys.maxint, gs)
 
     def available_workers(self):
         return self.workers - len([t for t in threading.enumerate() if t.name.startswith('runbot-group-worker-')])
@@ -1008,32 +583,32 @@
             command, params = openerprunbot.queue.get()
             if command == 'build':
                 team_name, group_name = params
-                if (team_name, group_name) in self.groups and self.groups[(team_name, group_name)].manual_build == sys.maxint:
-                    self.manual_build_count += 1
-                    self.groups[(team_name, group_name)].manual_build = self.manual_build_count
+                if (team_name, group_name) in self.groups and self.groups[(team_name, group_name)].build_number == sys.maxint:
+                    self.next_build_number += 1
+                    self.groups[(team_name, group_name)].build_number = self.next_build_number
                     self.groups[(team_name, group_name)].need_run_reason.append('build')
             else:
                 log("WARNING: unknown command", command)
 
     def reset_build_numbers(self):
-        gs = [(g.manual_build, g) for g in self.groups.values()]
+        gs = [(g.build_number, g) for g in self.groups.values()]
         gs.sort()
         gs = [g for (x,g) in gs]
-        self.manual_build_count = 0
+        self.next_build_number = 0
         sticky_branches = 0
         for g in gs:
             if g.sticky:
                 sticky_branches += 1
-            if g.manual_build == sys.maxint:
+            if g.build_number == sys.maxint:
                 break
-            self.manual_build_count += 1
-            g.manual_build = self.manual_build_count
+            self.next_build_number += 1
+            g.build_number = self.next_build_number
         self.number = max(sticky_branches + 1, self.number)
 
     def complete_jobs(self):
         """Update all slots with the completed jobs."""
         for job in self.jobs.values():
-            if job.completed:
+            if job.state in (STATE_RUNNING, STATE_BROKEN):
                 for g in self.groups.values():
                     if g.name == job.name:
                         g.complete_point(job)
@@ -1052,7 +627,7 @@
             for job in self.jobs.itervalues():
                 if self.is_sticky_job(job):
                     continue
-                if not job.running_t0:
+                if not job.running_since:
                     continue
                 if not victim or job.job_id < victim.job_id:
                     victim = job
@@ -1082,10 +657,11 @@
             if g.need_run_reason and g.all_points_completed():
                 port = self.allocate_port()
                 self.current_job_id += 1
-                job = Job(g, port, self.test, self.current_job_id, self.debug)
+                job_class = openerprunbot.jobs.JOBS[g.job_type] if g.job_type else openerprunbot.jobs.JOBS[self.default_job_type]
+                job = job_class(g, port, self.test, self.current_job_id, self.debug)
                 g.add_point(job)
                 g.need_run_reason = []
-                g.manual_build = sys.maxint
+                g.build_number = sys.maxint
                 self.jobs[job.job_id] = job
                 job.spawn()
 
 
=== added directory 'openerp-runbot/openerprunbot/jobs'
=== added file 'openerp-runbot/openerprunbot/jobs/__init__.py'
--- openerp-runbot/openerprunbot/jobs/__init__.py	1970-01-01 00:00:00 +0000
+++ openerp-runbot/openerprunbot/jobs/__init__.py	2012-07-04 11:46:17 +0000
@@ -0,0 +1,7 @@
+import install_all
+import migrate_script
+
+JOBS = {
+  'install_all': install_all.InstallAllJob,
+  'migrate_script': migrate_script.MigrateScriptJob,
+  }
 
=== added file 'openerp-runbot/openerprunbot/jobs/install_all.py'
--- openerp-runbot/openerprunbot/jobs/install_all.py	1970-01-01 00:00:00 +0000
+++ openerp-runbot/openerprunbot/jobs/install_all.py	2012-07-04 11:46:17 +0000
@@ -0,0 +1,366 @@
+import os
+import subprocess
+import threading
+import time
+import traceback
+
+from ..misc import *
+
+# Job states
+STATE_ALLOCATED = 'allocated'
+STATE_BROKEN = 'broken'
+STATE_PULLING = 'pulling'
+STATE_RUNNING = 'running'
+STATE_TESTING = 'testing'
+
+class InstallAllJob(object):
+    """
+    A Job encapsulates all the necessary data to build a branch group for a
+    given slot. The build is done in its own worker thread.
+    """
+
+    def __init__(self, g, port, test, job_id, debug):
+        self.job_id = job_id
+        self.state = STATE_ALLOCATED
+        self.team_name = g.team_name
+        self.name = g.name
+        self.name_underscore = g.name_underscore
+        self.version = g.version
+        self.debug = debug
+        self.need_run_reason = g.need_run_reason
+
+        self.repo_updates = g.repo_updates()
+        self.port = port
+        self.test = test
+        self.running_server_pid = 0
+        self.client_web_pid = 0
+        self.running_since=0
+
+        repo = os.path.join(g.runbot.wd,'repo')
+        job_suffix = '_job-' + str(self.job_id)
+        self.server_src = g.server.repo_path + job_suffix
+        self.client_web_src = (g.web.repo_path + job_suffix) if g.version == '6.0' and g.web else None
+        self.web_src = (g.web.repo_path + job_suffix) if g.web else None
+
+        # if addons is not the full addons branch use trunk
+        self.addons_src = [a.repo_path + job_suffix for a in g.addons]
+
+        if hasattr(g, 'migration_script'):
+            self.migration_script_src = g.migration_script.repo_path + job_suffix
+            self.migration_platform_src = g.migration_platform.repo_path + job_suffix
+
+        # Running path <Root>/static/<domain>
+        self.subdomain = "%s-%s-%s"%(self.team_name, self.name.replace('_','-').replace('.','-'),self.job_id)
+        self.running_path = os.path.join(g.runbot.wd,'static',self.subdomain)
+        self.json_path = g.json_path
+        self.log_path = os.path.join(self.running_path,'logs')
+        self.flags_path = os.path.join(self.running_path,'flags')
+
+        # Server
+        self.server_path=os.path.join(self.running_path,"server")
+        self.server_bin_path=os.path.join(self.server_path,"openerp-server")
+        self.server_log_path=os.path.join(self.log_path,'server.txt')
+        self.server_log_base_path=os.path.join(self.log_path,'test-base.txt')
+        self.server_log_all_path=os.path.join(self.log_path,'test-all.txt')
+
+        # coverage
+        self.coverage_file_path=os.path.join(self.log_path,'coverage.pickle')
+        self.coverage_base_path=os.path.join(self.log_path,'coverage-base')
+        self.coverage_all_path=os.path.join(self.log_path,'coverage-all')
+
+        # Web60
+        self.client_web_pid=None
+        self.client_web_path=os.path.join(self.running_path,"client-web")
+        self.client_web_bin_path=os.path.join(self.client_web_path,"openerp-web.py")
+        self.client_web_doc_path=os.path.join(self.client_web_path,"doc")
+        self.client_web_log_path=os.path.join(self.log_path,'client-web.txt')
+
+        # test
+        self.test_base_path=os.path.join(self.log_path,'test-base.txt')
+        self.test_result=None
+        self.test_all_path=os.path.join(self.log_path,'test-all.txt')
+
+        self.server_net_port=g.runbot.server_net_port
+        self.server_xml_port=g.runbot.server_xml_port
+        self.client_web_port=g.runbot.client_web_port
+
+        self.db = self.name_underscore + '_' + str(self.job_id)
+        self.db_all = "%s_all" % self.db
+
+        for p in ('addons', 'server', 'web'):
+            setattr(self, p + '_committer_name', None)
+            setattr(self, p + '_committer_xgram', None)
+            setattr(self, p + '_committer_email', None)
+        self.start_time = 0
+        self.completed_time = 0
+
+        self.modules = g.modules
+
+
+        self.server_rev = g.server.local_revision_count
+        self.addons_rev = [a.local_revision_count for a in g.addons]
+        self.web_rev = g.web.local_revision_count if g.web else 0
+        self.build_number = g.build_number
+
+    def spawn(self):
+        log("runbot-spawn-worker-" + str(self.job_id), group=self.name)
+        t = threading.Thread(target=self.work, name=('runbot-group-worker-' + str(self.job_id)))
+        t.daemon = True
+        t.start()
+
+    def work(self):
+        try:
+            self.state = STATE_PULLING
+            r = self.pull_branches()
+            if r:
+                self.state = STATE_TESTING
+                self.start_time = time.time()
+                self.start()
+                self.completed_time = time.time()
+                self.state = STATE_RUNNING
+                log("runbot-end-worker", job=self.name)
+            else:
+                self.state = STATE_BROKEN
+                log("runbot-end-worker [failed pull]", job=self.name)
+        except Exception, e:
+            self.state = STATE_BROKEN
+            log("runbot-end-worker [with exception]", job=self.name)
+            print traceback.format_exc()
+
+    def pull_branches(self):
+        job_suffix = '_job-' + str(self.job_id)
+        for b in self.repo_updates:
+            log("branch-update",branch=b.unique_name)
+            repo_path = b.repo_path + job_suffix
+            if os.path.exists(repo_path):
+                rc = run(["bzr", "clean-tree", "-d", repo_path, "--force", "--quiet"])
+                if rc: return False
+                rc = run(["bzr", "pull", "-d", repo_path, "--overwrite", "--quiet"])
+                if rc: return False
+            else:
+                rc = run(["bzr", "branch", "lp:%s" % b.unique_name, repo_path, "--quiet"])
+                if rc: return False
+            rc = run(["bzr", "update", "-r", str(b.local_revision_count), repo_path, "--quiet"])
+            if rc: return False
+
+            committer_name, committer_xgram, committer_email = \
+                get_committer_info(repo_path)
+            b.committer_name = committer_name
+            b.committer_xgram = committer_xgram
+            b.committer_email = committer_email
+            log("get-commiter-info", name=committer_name, xgram=committer_xgram, email=committer_email)
+        return True
+
+    def start_rsync(self):
+        log("job-start-rsync",branch=self.name)
+        for i in [self.running_path,self.log_path,self.flags_path,self.client_web_doc_path]:
+            if not os.path.exists(i):
+                os.makedirs(i)
+        # copy server
+        run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.server_src, self.server_path])
+
+        # copy addons (6.0 uses bin/addons, 6.1 and trunk use openerp/addons)
+        addons_dest = "openerp/addons"
+        if not os.path.exists(os.path.join(self.server_path,addons_dest)):
+            addons_dest = "bin/addons"
+        addons_path = os.path.join(self.server_path,addons_dest)
+        for a in self.addons_src:
+            run(["rsync","-a","--exclude",".bzr", "%s/"%a, addons_path])
+        if self.web_src and self.version != '6.0':
+            run(["rsync","-a","--exclude",".bzr", "%s/addons/"%self.web_src, addons_path])
+
+        # copy web-client
+        if self.client_web_src:
+            run(["rsync","-a","--exclude",".bzr","--delete", "%s/"%self.client_web_src, self.client_web_path])
+
+    def start_createdb(self):
+        run(["psql","template1","-c","select pg_terminate_backend(procpid) from pg_stat_activity where datname in ('%s','%s')"%(self.db,self.db_all)])
+        time.sleep(3)
+        run(["dropdb",self.db])
+        run(["dropdb",self.db_all])
+        run(["createdb",self.db])
+        run(["createdb",self.db_all])
+
+    def run_log(self, cmd, logfile, env=None):
+        env = dict(os.environ, **env) if env else None
+        log("run", *cmd, logfile=logfile)
+        out=open(logfile,"w")
+        p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True, env=env)
+        self.running_server_pid=p.pid
+        p.communicate()
+        return p
+
+    def resolve_server_bin_path(self):
+        # This can be done only if the files are present otherwise the if
+        # will always fail. Alternatively, server_bin_path could be a property.
+        if not os.path.exists(self.server_bin_path): # for 6.0 branches
+            self.server_bin_path=os.path.join(self.server_path,"bin","openerp-server.py")
+
+    def start_test_base(self):
+        log("job-start-server-base")
+        cmd = [self.server_bin_path,"-d",self.db,"-i","base","--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"]
+        _has_test_enable_flag = False
+        if has_test_enable_flag(self.server_bin_path):
+            cmd.append("--test-enable")
+            _has_test_enable_flag = True
+        cmd = ["coverage","run","--branch"] + cmd
+        self.run_log(cmd, logfile=self.test_base_path,env={'COVERAGE_FILE': self.coverage_file_path})
+        run(["coverage","html","-d",self.coverage_base_path,"--ignore-errors","--include=*.py"],env={'COVERAGE_FILE': self.coverage_file_path})
+        if _has_test_enable_flag:
+            success_message = "openerp.modules.loading: Modules loaded."
+            rc = not bool(run(["grep",success_message,self.test_base_path]))
+        else:
+            rc = bool(run(["grep","Traceback",self.test_base_path]))
+        return rc
+
+    def start_test_all(self, without_demo=False):
+        log("job-start-server-all")
+        mods = []
+        if not os.path.exists(os.path.join(self.server_path,"openerp/addons")):
+            mods = os.listdir(os.path.join(self.server_path,"bin/addons"))
+        elif os.path.exists(os.path.join(self.server_path,"openerp/addons")):
+            mods = os.listdir(os.path.join(self.server_path,"openerp/addons"))
+        bl = [
+            'document_ftp', 'l10n_lu',
+            '.bzrignore', '__init__.pyc', '__init__.py', 'base_quality_interrogation.py',
+        ]
+        mods = [m for m in mods if m not in bl]
+        if self.modules:
+            # TODO feedback when a requested modules does'nt exist.
+            mods = [m for m in mods if m in self.modules]
+        mods = ",".join(mods)
+        cmd = [self.server_bin_path,"-d",self.db_all,"-i",mods,"--stop-after-init","--no-xmlrpc","--no-xmlrpcs","--no-netrpc","--log-level=test"]
+        _has_test_enable_flag = False
+        if has_test_enable_flag(self.server_bin_path):
+            cmd.append("--test-enable")
+            _has_test_enable_flag = True
+        if without_demo:
+            cmd.append("--without-demo=True")
+        cmd = ["coverage","run","--branch"] + cmd
+        self.run_log(cmd, logfile=self.test_all_path)
+        run(["coverage","html","-d",self.coverage_all_path,"--ignore-errors","--include=*.py"])
+        if _has_test_enable_flag:
+            success_message = "openerp.modules.loading: Modules loaded."
+            rc = not bool(run(["grep",success_message,self.test_all_path]))
+        else:
+            rc = bool(run(["grep","Traceback",self.test_all_path]))
+        return rc
+
+    def start_server(self):
+        port = self.port
+        log("job-start-server",branch=self.name,port=port)
+        cmd=[self.server_bin_path,"--no-xmlrpcs","--netrpc-port=%d"%(self.server_net_port+port),"--xmlrpc-port=%d"%(self.server_xml_port+port)]
+        if os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi.py')) \
+            or os.path.exists(os.path.join(self.server_path, 'openerp', 'wsgi', 'core.py')):
+            cmd+=["--db-filter=%d(_.*)?$","--load=web"]
+        log("run",*cmd,log=self.server_log_path)
+        out=open(self.server_log_path,"w")
+        p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True)
+        self.running_server_pid=p.pid
+
+    def start_client_web(self):
+        if not self.client_web_src:
+            return
+        port = self.port
+        log("job-start-client-web",branch=self.name,port=port)
+        config="""
+        [global]
+        server.environment = "development"
+        server.socket_host = "0.0.0.0"
+        server.socket_port = %d
+        server.thread_pool = 10
+        tools.sessions.on = True
+        log.access_level = "INFO"
+        log.error_level = "INFO"
+        tools.csrf.on = False
+        tools.log_tracebacks.on = False
+        tools.cgitb.on = True
+        openerp.server.host = 'localhost'
+        openerp.server.port = '%d'
+        openerp.server.protocol = 'socket'
+        openerp.server.timeout = 450
+        [openerp-web]
+        dblist.filter = 'BOTH'
+        dbbutton.visible = True
+        company.url = ''
+        openerp.server.host = 'localhost'
+        openerp.server.port = '%d'
+        openerp.server.protocol = 'socket'
+        openerp.server.timeout = 450
+        """%(self.client_web_port+port,self.server_net_port+port,self.server_net_port+port)
+        config=config.replace("\n        ","\n")
+        cfgs = [os.path.join(self.client_web_path,"doc","openerp-web.cfg"), os.path.join(self.client_web_path,"openerp-web.cfg")]
+        for i in cfgs:
+            f=open(i,"w")
+            f.write(config)
+            f.close()
+
+        cmd=[self.client_web_bin_path]
+        log("run",*cmd,log=self.client_web_log_path)
+        out=open(self.client_web_log_path,"w")
+        p=subprocess.Popen(cmd, stdout=out, stderr=out, close_fds=True)
+        self.client_web_pid=p.pid
+
+    def start(self):
+        log("job-start",branch=self.name,port=self.port)
+        self.start_rsync()
+        self.resolve_server_bin_path()
+        self.start_createdb()
+        try:
+            if self.test:
+                r = self.start_test_base()
+                if not self.debug:
+                    self.test_result = r and self.start_test_all()
+                else:
+                    self.test_result = r
+            self.start_server()
+            self.start_client_web()
+        except OSError,e:
+            log("branch-start-error",exception=e)
+        except IOError,e:
+            log("branch-start-error",exception=e)
+        self.running_since=time.time()
+        log("branch-started",branch=self.name,port=self.port)
+
+    def stop(self):
+        log("Stopping job", id=self.job_id, branch=self.name)
+        if self.running_server_pid:
+            kill(self.running_server_pid)
+        if self.client_web_pid:
+            kill(self.client_web_pid)
+
+    def save_json(self):
+        """Append the point data to a JSON file for posterity."""
+        # A job must be saved only once.
+        if hasattr(self, 'json_saved'):
+            log("=== save_json() called more than once. ===", job_id=self.job_id)
+        self.json_saved = True
+
+        path = self.json_path
+        state = {}
+
+        if os.path.exists(path):
+            with open(path, 'r') as h:
+                state = simplejson.loads(h.read())
+
+        value = {}
+        committers = []
+        for p in ('addons', 'server', 'web'):
+            committers += [
+                p + '_committer_name',
+                p + '_committer_xgram',
+                # p + '_committer_email',
+            ]
+        for a in [
+            'job_id', 'db', 'running_path', 'subdomain', 'server_rev',
+            'addons_rev', 'web_rev', 'need_run_reason',
+            'test_result',
+            ] + committers:
+            value[a] = getattr(self, a)
+        state.setdefault('jobs', [])
+        state['jobs'].append(value)
+
+        with open(path, 'w') as h:
+            data = simplejson.dumps(state, sort_keys=True, indent=4)
+            h.write(data)
+
 
=== added file 'openerp-runbot/openerprunbot/jobs/migrate_script.py'
--- openerp-runbot/openerprunbot/jobs/migrate_script.py	1970-01-01 00:00:00 +0000
+++ openerp-runbot/openerprunbot/jobs/migrate_script.py	2012-07-04 11:46:17 +0000
@@ -0,0 +1,61 @@
+import os
+import subprocess
+import threading
+import time
+import traceback
+
+from install_all import InstallAllJob
+from ..misc import *
+
+# Job states
+STATE_ALLOCATED = 'allocated'
+STATE_BROKEN = 'broken'
+STATE_PULLING = 'pulling'
+STATE_RUNNING = 'running'
+STATE_TESTING = 'testing'
+
+class MigrateScriptJob(InstallAllJob):
+
+    def start(self, migrate=False):
+        log("job-start",branch=self.name,port=self.port)
+        self.migration_log_path=os.path.join(self.log_path,'migration.txt')
+
+        self.start_rsync()
+        self.resolve_server_bin_path()
+        self.start_createdb()
+        try:
+            rc0 = self.start_test_all(without_demo=True)
+
+            # Run the migration towards trunk
+            if False: # if migrate
+                run(['rm', '-rf', 'migration-' + str(self.job_id)])
+                run(['cp', '-r', self.migration_script_src, 'migration-' + str(self.job_id)])
+                run(['cp', self.migration_platform_src + '/Makefile', 'migration-' + str(self.job_id)])
+                cmd = 'make pre update post updatelog reset-demoflag'.split() + [
+                    'OPENERP_BIN=' + self.server_bin_path,
+                    'DATABASE=' + self.db_all,
+                    'VERSION=trunk']
+                run_output(cmd, cwd='migration-' + str(self.job_id))
+
+            # Check the migrated database with an 'update all' and tests enabled.
+            cmd = ['psql', self.db_all, '-c', "update ir_module_module set demo='t'"]
+            run(cmd)
+            cmd = [self.server_bin_path, "-d", self.db_all, "-u", "base", "--stop-after-init", "--no-xmlrpc", "--no-xmlrpcs", "--no-netrpc", "--log-level=test"]
+            _has_test_enable_flag = False
+            if has_test_enable_flag(self.server_bin_path):
+                cmd.append("--test-enable")
+                _has_test_enable_flag = True
+            self.run_log(cmd, logfile=self.test_all_path)
+            if _has_test_enable_flag:
+                success_message = "openerp.modules.loading: Modules loaded."
+                rc = not bool(run(["grep",success_message,self.test_all_path]))
+            else:
+                rc = bool(run(["grep","Traceback",self.test_all_path]))
+            self.test_result = rc0 and rc
+
+        except OSError,e:
+            log("branch-start-error",exception=e)
+        except IOError,e:
+            log("branch-start-error",exception=e)
+        self.running_since=time.time()
+
 
=== modified file 'openerp-runbot/openerprunbot/misc.py'
--- openerp-runbot/openerprunbot/misc.py	2011-11-24 10:09:50 +0000
+++ openerp-runbot/openerprunbot/misc.py	2012-07-04 11:46:17 +0000
@@ -3,6 +3,7 @@
 """
 
 import fcntl
+import re
 import signal
 import subprocess
 import threading
@@ -10,7 +11,7 @@
 import os
 
 __all__ = [
-    'kill', 'log', 'run', 'run_output'
+    'kill', 'log', 'run', 'run_output', 'get_committer_info', 'underscore', 'has_test_enable_flag', 'project_name_from_unique_name'
     ]
 
 log_lock = threading.Lock()
@@ -62,4 +63,56 @@
         pass
 
 def run_output(l, cwd=None):
+    log("run_output",l)
     return subprocess.Popen(l, stdout=subprocess.PIPE, cwd=cwd).communicate()[0]
+
+#----------------------------------------------------------
+# OpenERP RunBot misc
+#----------------------------------------------------------
+
+def underscore(s):
+    return s.replace("~","").replace(":","_").replace("/","_").replace(".","_").replace(" ","_")
+
+def get_committer_info(repo_path):
+    committer_name = None
+    committer_xgram = None
+    committer_email = None
+
+    output = run_output(["bzr", "log", "--long", "-r-1"], cwd=repo_path)
+
+    committer_re = re.compile('committer: *(.+)<(.+)@(.+)>')
+    for i in output.split('\n'):
+        m = committer_re.match(i)
+        if m:
+            committer_name = m.group(1).strip()
+            committer_xgram = m.group(2)
+            committer_email = m.group(2) + '@' + m.group(3)
+            break
+
+    return committer_name, committer_xgram, committer_email
+
+def has_test_enable_flag(server_bin_path):
+    """
+    Test whether an openerp-server executable has the --test-enable flag.
+    (When the flag is present, testing for success/failure is done in a
+    different way.)
+    """
+    p1 = subprocess.Popen([server_bin_path, "--help"],
+        stdout=subprocess.PIPE)
+    p2 = subprocess.Popen(["grep", "test-enable"], stdin=p1.stdout,
+        stdout=subprocess.PIPE)
+    p1.stdout.close()  # Allow p1 to receive a SIGPIPE if p2 exits.
+    output = p2.communicate()[0]
+    return output == "    --test-enable       Enable YAML and unit tests.\n"
+
+def project_name_from_unique_name(unique_name):
+    m = re.search("/(openobject|openerp)-(addons|server|client-web|web)/", unique_name)
+    if m:
+        n = m.group(2)
+        if n == 'client-web':
+            return 'web'
+        else:
+            return n
+    else:
+        return None
+
 
=== modified file 'openerp-runbot/openerprunbot/templates/branches.html.mako'
--- openerp-runbot/openerprunbot/templates/branches.html.mako	2012-05-23 13:07:13 +0000
+++ openerp-runbot/openerprunbot/templates/branches.html.mako	2012-07-04 11:46:17 +0000
@@ -106,11 +106,12 @@
       <h2>Runtime data</h2>
         <div class="stats">
         (The following data are global to the Runbot.)<br />
+        Default job type: ${r.default_job_type}.<br />
         Maximum ${r.number} concurrent branches.<br />
         Maximum ${r.workers} concurrent jobs.<br />
         ${r.current_job_id} processed jobs.<br />
         ${r.workers - r.available_workers()} ongoing jobs.<br />
-        ${r.manual_build_count} manual build requests to go.<br />
+        ${r.next_build_number} manual build requests to go.<br />
         </div>
     </div>
 
 
=== modified file 'openerp-runbot/openerprunbot/templates/defs.html.mako'
--- openerp-runbot/openerprunbot/templates/defs.html.mako	2012-06-21 10:24:31 +0000
+++ openerp-runbot/openerprunbot/templates/defs.html.mako	2012-07-04 11:46:17 +0000
@@ -14,9 +14,9 @@
             <span class="i action-toggle dropdown-toggle" data-toggle="dropdown">B</span>
     <ul class="dropdown-menu">
         <li><a href="http://${r.domain}/${p.subdomain}/logs/test-all.txt">
-        % if p.test_all_result==False:
+        % if p.test_result==False:
             Install logs (failure)
-        % elif p.test_all_result==True:
+        % elif p.test_result==True:
             Install logs (success)
         % else:
             Install logs (ongoing)
@@ -30,25 +30,31 @@
         % if 'build' in p.need_run_reason:
           <li>(Build manually requested)</li>
         % endif
-        % if p.manual_build != sys.maxint:
-              <li>(${p.manual_build})</li>
+        % if p.build_number != sys.maxint:
+              <li>(${p.build_number})</li>
         % endif
     </ul>
 % if p.state == 'testing':
             <p><span class="status"> </span>
-% elif p.test_base_result==True and p.test_all_result==True:
+% elif p.test_result==True:
             <p><span class="status green"> </span>
-% elif p.test_base_result==False or p.test_all_result==False:
+% elif p.test_result==False:
             <p><span class="status red"> </span>
 % else:
             <p><span class="status"> </span>
 % endif
 % if p.state == 'testing':
 	<span class="testing">Testing...</span></p>
-% elif p.running_t0 and p.test_all_result:
-	<span class="running-long">Age: ${r.nginx_index_time(t-p.running_t0)}</span></p>
-% elif p.running_t0:
-	<span class="testing">Age: ${r.nginx_index_time(t-p.running_t0)}</span></p>
+% elif p.state == 'pulling':
+	<span class="testing">Pulling...</span></p>
+% elif p.state == 'allocated':
+	<span class="testing">Allocated...</span></p>
+% elif p.state == 'broken':
+	<span class="testing">Internal error</span></p>
+% elif p.running_since and p.test_result:
+	<span class="running-long">Age: ${r.nginx_index_time(t-p.running_since)}</span></p>
+% elif p.running_since:
+	<span class="testing">Age: ${r.nginx_index_time(t-p.running_since)}</span></p>
 % else:
 	<span class="testing">Internal error</span></p>
 % endif
@@ -58,7 +64,7 @@
     ${rev_(r,g,b,p)}
 % endfor
             </ul>
-% if ((p.job_id > r.current_job_id - r.number) and p.running_t0) or g.is_sticky_job(p):
+% if ((p.job_id > r.current_job_id - r.number) or g.is_sticky_job(p)) and p.running_since:
     <form target="_blank" method="GET" action="http://${p.db}.${r.domain}/">
         <button type="submit">Connect</button>
     </form>
@@ -69,7 +75,7 @@
 </%def>
 
 <%def name="branch_(r,g,b)">
-% if b.overriden_repo_path:
+% if not b.trigger_build:
 <span class="label">
 % else:
 <span class="label notice">
@@ -97,8 +103,8 @@
 % if g.wrong_matching:
   <span class="wrong-matching">These branches are not correctly named (<a href="/#how">see the instructions</a>).</span>
 % endif
-% if g.manual_build != sys.maxint:
-<p><span>(build n.${g.manual_build})</span></p>
+% if g.build_number != sys.maxint:
+<p><span>(build n.${g.build_number})</span></p>
 % elif not g.sticky:
     ${build_button_(r, g)}
 % endif
@@ -126,16 +132,16 @@
 <tr>
     <td>
 <form method="POST" action="http://${r.domain}/a?build=${g.name}&amp;team=${g.team_name}">
-% if g.manual_build != sys.maxint:
-<strong>${g.manual_build}.</strong>
+% if g.build_number != sys.maxint:
+<strong>${g.build_number}.</strong>
 % endif
 ${g.name}
 <span>(${g.team_name})</span>
-% if g.manual_build == sys.maxint:
+% if g.build_number == sys.maxint:
     <button type="submit">Force Build</button>
 % endif
 % for b in g.repo_updates():
-    % if not b.overriden_repo_path:
+    % if b.trigger_build:
         ${branch_(r,g,b)}
     % endif
 % endfor
 
=== modified file 'openerp-runbot/openerprunbot/templates/nginx.conf.mako'
--- openerp-runbot/openerprunbot/templates/nginx.conf.mako	2012-04-27 10:25:23 +0000
+++ openerp-runbot/openerprunbot/templates/nginx.conf.mako	2012-07-04 11:46:17 +0000
@@ -10,7 +10,7 @@
   client_body_temp_path nginx; proxy_temp_path nginx; fastcgi_temp_path nginx; access_log nginx/access.log; index index.html;
   % for g in r.groups.values():
   % for i in g.points:
-  % if i and (i.job_id > r.current_job_id - r.number) and i.running_t0:
+  % if i and (i.job_id > r.current_job_id - r.number or g.is_sticky_job(i)) and i.running_since:
     % if i.team_name == 'openerp-dev' and g.is_sticky_job(i):
   server { # openerp-dev sticky job (i.e. first job of a sticky branch)
     listen ${r.nginx_port};
 
=== added file 'openerp-runbot/starting.html'
--- openerp-runbot/starting.html	1970-01-01 00:00:00 +0000
+++ openerp-runbot/starting.html	2012-07-04 11:46:17 +0000
@@ -0,0 +1,1 @@
+The runbot is starting over.
 
=== added file 'openerp-runbot/stopped.html'
--- openerp-runbot/stopped.html	1970-01-01 00:00:00 +0000
+++ openerp-runbot/stopped.html	2012-07-04 11:46:17 +0000
@@ -0,0 +1,1 @@
+The runbot is down for "planned" maintenance.
 
=== modified file 'openerp-runbot/try-template.py'
--- openerp-runbot/try-template.py	2012-04-18 13:00:01 +0000
+++ openerp-runbot/try-template.py	2012-07-04 11:46:17 +0000
@@ -12,8 +12,8 @@
         unique_name = '~openerp-dev/openobject-server/trunk-dummy'
         merge_count = 1
         local_revision_count = 333
-        overriden_repo_path = None
-        overriden_project_name = None
+        repo_path = None
+        project_name = None
 
     class Point(object):
         """ Dummy Point class to test mako templates. """
@@ -24,8 +24,8 @@
         subdomain = 'trunk_dummy_2'
         port = 22
         need_run_reason = ['server']
-        manual_build = sys.maxint
-        running_t0 = 100000
+        build_number = sys.maxint
+        running_since = 100000
         repo_updates = [RunBotBranch()]
         version = 'trunk'
 
@@ -33,7 +33,7 @@
         """ Dummy Group class to test mako templates. """
         name = 'trunk-dummy'
         team_name = 'openerp-dev'
-        manual_build = sys.maxint
+        build_number = sys.maxint
         wrong_matching = False
         points = [None, None, Point()]
         version = 'trunk'
@@ -46,7 +46,7 @@
         number = 555
         workers = 666
         current_job_id = 777
-        manual_build_count = 888
+        next_build_number = 888
         server_net_port = 12000
         server_xml_port = 12100
         def nginx_index_time(self, t):