Merge lp:~benji/charms/oneiric/buildbot-master/remove-base64 into lp:~yellow/charms/oneiric/buildbot-master/trunk

[Benji York]

This branch removes the need to base64 encode the Buildbot config file.

(and it increases the timeout on wait_for_unit so slow unit-creation on EC2 won't bite us so hard)

[lp:~benji/charms/oneiric/buildbot-master/remove-base64]

26. By Benji York on 2012-02-10

merge from trunk and fix conflict

[Brad Crittenden (bac)]

a-ok

review: Approve (code)

=== modified file 'README.txt'
--- README.txt	2012-02-09 14:33:51 +0000
+++ README.txt	2012-02-10 21:32:19 +0000
@@ -6,16 +6,6 @@
 repository.
 
 juju bootstrap
-juju deploy --repository=./charms local:buildbot-master
-juju deploy --repository=./charms local:buildbot-slave
-juju set buildbot-master extra-packages=git
-juju set buildbot-master config-file=`base64 -w0 charms/oneiric/buildbot-master/examples/master.cfg`
-juju set buildbot-slave builders=runtests
-juju add-relation buildbot-slave buildbot-master
-
-Demo of a full buildbot master directory:
-
-juju bootstrap
 juju deploy --config=./charms/oneiric/buildbot-master/examples/lpbuildbot.yaml --repository=./charms local:buildbot-master
 juju deploy --config=./charms/oneiric/buildbot-slave/config.setuplxc.yaml --repository=./charms local:buildbot-slave
 juju add-relation buildbot-slave buildbot-master
@@ -28,4 +18,3 @@
 
 juju deploy --repository=./charms local:buildbot-slave
 juju set buildbot-slave builders=lucid_lp,lucid_db_lp
-

=== modified file 'config.yaml'
--- config.yaml	2012-02-09 18:54:03 +0000
+++ config.yaml	2012-02-10 21:32:19 +0000
@@ -13,11 +13,8 @@
     default: /var/lib/buildbot/masters/master
   config-file:
     description: |
-        An encoded master.cfg file.  Use of this configuration is
-        mutually exclusive with the use of config-transport and
-        config-url.  Use it with the `base64` command like this:
-            juju set buildbot-master \
-                config-file=`base64 -w0 ./examples/master.cfg`
+        A master.cfg file.  Use of this configuration is mutually exclusive
+        with the use of config-transport and config-url.
     type: string
   config-transport:
     description: |

=== modified file 'examples/pyflakes.yaml'
--- examples/pyflakes.yaml	2012-02-09 16:31:10 +0000
+++ examples/pyflakes.yaml	2012-02-10 21:32:19 +0000
@@ -2,77 +2,120 @@
   extra-packages: git
   installdir: /tmp/buildbot
   config-file: |
-        IyAtKi0gcHl0aG9uIC0qLQojIGV4OiBzZXQgc3ludGF4PXB5dGhvbjoKCiMgVGhpcyBpcyBhIHNh
-        bXBsZSBidWlsZG1hc3RlciBjb25maWcgZmlsZS4gSXQgbXVzdCBiZSBpbnN0YWxsZWQgYXMKIyAn
-        bWFzdGVyLmNmZycgaW4geW91ciBidWlsZG1hc3RlcidzIGJhc2UgZGlyZWN0b3J5LgoKIyBUaGlz
-        IGlzIHRoZSBkaWN0aW9uYXJ5IHRoYXQgdGhlIGJ1aWxkbWFzdGVyIHBheXMgYXR0ZW50aW9uIHRv
-        LiBXZSBhbHNvIHVzZQojIGEgc2hvcnRlciBhbGlhcyB0byBzYXZlIHR5cGluZy4KYyA9IEJ1aWxk
-        bWFzdGVyQ29uZmlnID0ge30KCiMjIyMjIyMgQlVJTERTTEFWRVMKCiMgVGhlICdzbGF2ZXMnIGxp
-        c3QgZGVmaW5lcyB0aGUgc2V0IG9mIHJlY29nbml6ZWQgYnVpbGRzbGF2ZXMuIEVhY2ggZWxlbWVu
-        dCBpcwojIGEgQnVpbGRTbGF2ZSBvYmplY3QsIHNwZWNpZnlpbmcgYSB1c2VybmFtZSBhbmQgcGFz
-        c3dvcmQuICBUaGUgc2FtZSB1c2VybmFtZSBhbmQKIyBwYXNzd29yZCBtdXN0IGJlIGNvbmZpZ3Vy
-        ZWQgb24gdGhlIHNsYXZlLgpmcm9tIGJ1aWxkYm90LmJ1aWxkc2xhdmUgaW1wb3J0IEJ1aWxkU2xh
-        dmUKY1snc2xhdmVzJ10gPSBbXQoKIyAnc2xhdmVQb3J0bnVtJyBkZWZpbmVzIHRoZSBUQ1AgcG9y
-        dCB0byBsaXN0ZW4gb24gZm9yIGNvbm5lY3Rpb25zIGZyb20gc2xhdmVzLgojIFRoaXMgbXVzdCBt
-        YXRjaCB0aGUgdmFsdWUgY29uZmlndXJlZCBpbnRvIHRoZSBidWlsZHNsYXZlcyAod2l0aCB0aGVp
-        cgojIC0tbWFzdGVyIG9wdGlvbikKY1snc2xhdmVQb3J0bnVtJ10gPSA5OTg5CgojIyMjIyMjIENI
-        QU5HRVNPVVJDRVMKCiMgdGhlICdjaGFuZ2Vfc291cmNlJyBzZXR0aW5nIHRlbGxzIHRoZSBidWls
-        ZG1hc3RlciBob3cgaXQgc2hvdWxkIGZpbmQgb3V0CiMgYWJvdXQgc291cmNlIGNvZGUgY2hhbmdl
-        cy4gIEhlcmUgd2UgcG9pbnQgdG8gdGhlIGJ1aWxkYm90IGNsb25lIG9mIHB5Zmxha2VzLgoKZnJv
-        bSBidWlsZGJvdC5jaGFuZ2VzLmdpdHBvbGxlciBpbXBvcnQgR2l0UG9sbGVyCmNbJ2NoYW5nZV9z
-        b3VyY2UnXSA9IEdpdFBvbGxlcigKICAgICAgICAnZ2l0Oi8vZ2l0aHViLmNvbS9idWlsZGJvdC9w
-        eWZsYWtlcy5naXQnLAogICAgICAgIGJyYW5jaD0nbWFzdGVyJywgcG9sbGludGVydmFsPTEyMDAp
-        CgojIyMjIyMjIFNDSEVEVUxFUlMKCiMgQ29uZmlndXJlIHRoZSBTY2hlZHVsZXJzLCB3aGljaCBk
-        ZWNpZGUgaG93IHRvIHJlYWN0IHRvIGluY29taW5nIGNoYW5nZXMuICBJbiB0aGlzCiMgY2FzZSwg
-        anVzdCBraWNrIG9mZiBhICdydW50ZXN0cycgYnVpbGQKCmZyb20gYnVpbGRib3Quc2NoZWR1bGVy
-        IGltcG9ydCBTY2hlZHVsZXIKY1snc2NoZWR1bGVycyddID0gW10KY1snc2NoZWR1bGVycyddLmFw
-        cGVuZChTY2hlZHVsZXIobmFtZT0iYWxsIiwgYnJhbmNoPU5vbmUsCiAgICAgICAgICAgICAgICAg
-        ICAgICAgICAgICAgICAgIHRyZWVTdGFibGVUaW1lcj1Ob25lLAogICAgICAgICAgICAgICAgICAg
-        ICAgICAgICAgICAgICBidWlsZGVyTmFtZXM9WyJydW50ZXN0cyJdKSkKCiMjIyMjIyMgQlVJTERF
-        UlMKCiMgVGhlICdidWlsZGVycycgbGlzdCBkZWZpbmVzIHRoZSBCdWlsZGVycywgd2hpY2ggdGVs
-        bCBCdWlsZGJvdCBob3cgdG8gcGVyZm9ybSBhIGJ1aWxkOgojIHdoYXQgc3RlcHMsIGFuZCB3aGlj
-        aCBzbGF2ZXMgY2FuIGV4ZWN1dGUgdGhlbS4gIE5vdGUgdGhhdCBhbnkgcGFydGljdWxhciBidWls
-        ZCB3aWxsCiMgb25seSB0YWtlIHBsYWNlIG9uIG9uZSBzbGF2ZS4KCmZyb20gYnVpbGRib3QucHJv
-        Y2Vzcy5mYWN0b3J5IGltcG9ydCBCdWlsZEZhY3RvcnkKZnJvbSBidWlsZGJvdC5zdGVwcy5zb3Vy
-        Y2UgaW1wb3J0IEdpdApmcm9tIGJ1aWxkYm90LnN0ZXBzLnNoZWxsIGltcG9ydCBTaGVsbENvbW1h
-        bmQKCmZhY3RvcnkgPSBCdWlsZEZhY3RvcnkoKQojIGNoZWNrIG91dCB0aGUgc291cmNlCmZhY3Rv
-        cnkuYWRkU3RlcChHaXQocmVwb3VybD0nZ2l0Oi8vZ2l0aHViLmNvbS9idWlsZGJvdC9weWZsYWtl
-        cy5naXQnLCBtb2RlPSdjb3B5JykpCiMgcnVuIHRoZSB0ZXN0cyAobm90ZSB0aGF0IHRoaXMgd2ls
-        bCByZXF1aXJlIHRoYXQgJ3RyaWFsJyBpcyBpbnN0YWxsZWQpCmZhY3RvcnkuYWRkU3RlcChTaGVs
-        bENvbW1hbmQoY29tbWFuZD1bInRyaWFsIiwgInB5Zmxha2VzIl0pKQoKZnJvbSBidWlsZGJvdC5j
-        b25maWcgaW1wb3J0IEJ1aWxkZXJDb25maWcKCmNbJ2J1aWxkZXJzJ10gPSBbCiAgICBCdWlsZGVy
-        Q29uZmlnKG5hbWU9InJ1bnRlc3RzIiwKICAgICAgIyBCdWlsZGJvdCBlbmZvcmNlcyB0aGF0IHRo
-        ZSBzbGF2ZW5hbWVzIGxpc3QgbXVzdCBub3QgYmUgZW1wdHkuIE91cgogICAgICAjIHdyYXBwZXIg
-        d2lsbCByZW1vdmUgdGhlIGVtcHR5IHN0cmluZyBhbmQgcmVwbGFjZSBpdCB3aXRoIHByb3BlciB2
-        YWx1ZXMuCiAgICAgIHNsYXZlbmFtZXM9WycnXSwKICAgICAgZmFjdG9yeT1mYWN0b3J5KSwKICAg
-        IF0KCiMjIyMjIyMgU1RBVFVTIFRBUkdFVFMKCiMgJ3N0YXR1cycgaXMgYSBsaXN0IG9mIFN0YXR1
-        cyBUYXJnZXRzLiBUaGUgcmVzdWx0cyBvZiBlYWNoIGJ1aWxkIHdpbGwgYmUKIyBwdXNoZWQgdG8g
-        dGhlc2UgdGFyZ2V0cy4gYnVpbGRib3Qvc3RhdHVzLyoucHkgaGFzIGEgdmFyaWV0eSB0byBjaG9v
-        c2UgZnJvbSwKIyBpbmNsdWRpbmcgd2ViIHBhZ2VzLCBlbWFpbCBzZW5kZXJzLCBhbmQgSVJDIGJv
-        dHMuCgpjWydzdGF0dXMnXSA9IFtdCgpmcm9tIGJ1aWxkYm90LnN0YXR1cyBpbXBvcnQgaHRtbApm
-        cm9tIGJ1aWxkYm90LnN0YXR1cy53ZWIgaW1wb3J0IGF1dGgsIGF1dGh6CmF1dGh6X2NmZz1hdXRo
-        ei5BdXRoeigKICAgICMgY2hhbmdlIGFueSBvZiB0aGVzZSB0byBUcnVlIHRvIGVuYWJsZTsgc2Vl
-        IHRoZSBtYW51YWwgZm9yIG1vcmUKICAgICMgb3B0aW9ucwogICAgZ3JhY2VmdWxTaHV0ZG93biA9
-        IEZhbHNlLAogICAgZm9yY2VCdWlsZCA9IFRydWUsICMgdXNlIHRoaXMgdG8gdGVzdCB5b3VyIHNs
-        YXZlIG9uY2UgaXQgaXMgc2V0IHVwCiAgICBmb3JjZUFsbEJ1aWxkcyA9IEZhbHNlLAogICAgcGlu
-        Z0J1aWxkZXIgPSBGYWxzZSwKICAgIHN0b3BCdWlsZCA9IEZhbHNlLAogICAgc3RvcEFsbEJ1aWxk
-        cyA9IEZhbHNlLAogICAgY2FuY2VsUGVuZGluZ0J1aWxkID0gRmFsc2UsCikKY1snc3RhdHVzJ10u
-        YXBwZW5kKGh0bWwuV2ViU3RhdHVzKGh0dHBfcG9ydD04MDEwLCBhdXRoej1hdXRoel9jZmcpKQoK
-        IyMjIyMjIyBQUk9KRUNUIElERU5USVRZCgojIHRoZSAncHJvamVjdE5hbWUnIHN0cmluZyB3aWxs
-        IGJlIHVzZWQgdG8gZGVzY3JpYmUgdGhlIHByb2plY3QgdGhhdCB0aGlzCiMgYnVpbGRib3QgaXMg
-        d29ya2luZyBvbi4gRm9yIGV4YW1wbGUsIGl0IGlzIHVzZWQgYXMgdGhlIHRpdGxlIG9mIHRoZQoj
-        IHdhdGVyZmFsbCBIVE1MIHBhZ2UuIFRoZSAncHJvamVjdFVSTCcgc3RyaW5nIHdpbGwgYmUgdXNl
-        ZCB0byBwcm92aWRlIGEgbGluawojIGZyb20gYnVpbGRib3QgSFRNTCBwYWdlcyB0byB5b3VyIHBy
-        b2plY3QncyBob21lIHBhZ2UuCgpjWydwcm9qZWN0TmFtZSddID0gIlB5Zmxha2VzIgpjWydwcm9q
-        ZWN0VVJMJ10gPSAiaHR0cDovL2Rpdm1vZC5vcmcvdHJhYy93aWtpL0Rpdm1vZFB5Zmxha2VzIgoK
-        IyB0aGUgJ2J1aWxkYm90VVJMJyBzdHJpbmcgc2hvdWxkIHBvaW50IHRvIHRoZSBsb2NhdGlvbiB3
-        aGVyZSB0aGUgYnVpbGRib3QncwojIGludGVybmFsIHdlYiBzZXJ2ZXIgKHVzdWFsbHkgdGhlIGh0
-        bWwuV2ViU3RhdHVzIHBhZ2UpIGlzIHZpc2libGUuIFRoaXMKIyB0eXBpY2FsbHkgdXNlcyB0aGUg
-        cG9ydCBudW1iZXIgc2V0IGluIHRoZSBXYXRlcmZhbGwgJ3N0YXR1cycgZW50cnksIGJ1dAojIHdp
-        dGggYW4gZXh0ZXJuYWxseS12aXNpYmxlIGhvc3QgbmFtZSB3aGljaCB0aGUgYnVpbGRib3QgY2Fu
-        bm90IGZpZ3VyZSBvdXQKIyB3aXRob3V0IHNvbWUgaGVscC4KCmNbJ2J1aWxkYm90VVJMJ10gPSAi
-        aHR0cDovL2xvY2FsaG9zdDo4MDEwLyIKCiMjIyMjIyMgREIgVVJMCgojIFRoaXMgc3BlY2lmaWVz
-        IHdoYXQgZGF0YWJhc2UgYnVpbGRib3QgdXNlcyB0byBzdG9yZSBjaGFuZ2UgYW5kIHNjaGVkdWxl
-        cgojIHN0YXRlLiAgWW91IGNhbiBsZWF2ZSB0aGlzIGF0IGl0cyBkZWZhdWx0IGZvciBhbGwgYnV0
-        IHRoZSBsYXJnZXN0CiMgaW5zdGFsbGF0aW9ucy4KY1snZGJfdXJsJ10gPSAic3FsaXRlOi8vL3N0
-        YXRlLnNxbGl0ZSIKCg==
+    # -*- python -*-
+    # ex: set syntax=python:
+    
+    # This is a sample buildmaster config file. It must be installed as
+    # 'master.cfg' in your buildmaster's base directory.
+    
+    # This is the dictionary that the buildmaster pays attention to. We also use
+    # a shorter alias to save typing.
+    c = BuildmasterConfig = {}
+    
+    ####### BUILDSLAVES
+    
+    # The 'slaves' list defines the set of recognized buildslaves. Each element is
+    # a BuildSlave object, specifying a username and password.  The same username and
+    # password must be configured on the slave.
+    from buildbot.buildslave import BuildSlave
+    c['slaves'] = []
+    
+    # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
+    # This must match the value configured into the buildslaves (with their
+    # --master option)
+    c['slavePortnum'] = 9989
+    
+    ####### CHANGESOURCES
+    
+    # the 'change_source' setting tells the buildmaster how it should find out
+    # about source code changes.  Here we point to the buildbot clone of pyflakes.
+    
+    from buildbot.changes.gitpoller import GitPoller
+    c['change_source'] = GitPoller(
+            'git://github.com/buildbot/pyflakes.git',
+            branch='master', pollinterval=1200)
+    
+    ####### SCHEDULERS
+    
+    # Configure the Schedulers, which decide how to react to incoming changes.  In this
+    # case, just kick off a 'runtests' build
+    
+    from buildbot.scheduler import Scheduler
+    c['schedulers'] = []
+    c['schedulers'].append(Scheduler(name="all", branch=None,
+                                     treeStableTimer=None,
+                                     builderNames=["runtests"]))
+    
+    ####### BUILDERS
+    
+    # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
+    # what steps, and which slaves can execute them.  Note that any particular build will
+    # only take place on one slave.
+    
+    from buildbot.process.factory import BuildFactory
+    from buildbot.steps.source import Git
+    from buildbot.steps.shell import ShellCommand
+    
+    factory = BuildFactory()
+    # check out the source
+    factory.addStep(Git(repourl='git://github.com/buildbot/pyflakes.git', mode='copy'))
+    # run the tests (note that this will require that 'trial' is installed)
+    factory.addStep(ShellCommand(command=["trial", "pyflakes"]))
+    
+    from buildbot.config import BuilderConfig
+    
+    c['builders'] = [
+        BuilderConfig(name="runtests",
+          # Buildbot enforces that the slavenames list must not be empty. Our
+          # wrapper will remove the empty string and replace it with proper values.
+          slavenames=[''],
+          factory=factory),
+        ]
+    
+    ####### STATUS TARGETS
+    
+    # 'status' is a list of Status Targets. The results of each build will be
+    # pushed to these targets. buildbot/status/*.py has a variety to choose from,
+    # including web pages, email senders, and IRC bots.
+    
+    c['status'] = []
+    
+    from buildbot.status import html
+    from buildbot.status.web import auth, authz
+    authz_cfg=authz.Authz(
+        # change any of these to True to enable; see the manual for more
+        # options
+        gracefulShutdown = False,
+        forceBuild = True, # use this to test your slave once it is set up
+        forceAllBuilds = False,
+        pingBuilder = False,
+        stopBuild = False,
+        stopAllBuilds = False,
+        cancelPendingBuild = False,
+    )
+    c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg))
+    
+    ####### PROJECT IDENTITY
+    
+    # the 'projectName' string will be used to describe the project that this
+    # buildbot is working on. For example, it is used as the title of the
+    # waterfall HTML page. The 'projectURL' string will be used to provide a link
+    # from buildbot HTML pages to your project's home page.
+    
+    c['projectName'] = "Pyflakes"
+    c['projectURL'] = "http://divmod.org/trac/wiki/DivmodPyflakes"
+    
+    # the 'buildbotURL' string should point to the location where the buildbot's
+    # internal web server (usually the html.WebStatus page) is visible. This
+    # typically uses the port number set in the Waterfall 'status' entry, but
+    # with an externally-visible host name which the buildbot cannot figure out
+    # without some help.
+    
+    c['buildbotURL'] = "http://localhost:8010/"
+    
+    ####### DB URL
+    
+    # This specifies what database buildbot uses to store change and scheduler
+    # state.  You can leave this at its default for all but the largest
+    # installations.
+    c['db_url'] = "sqlite:///state.sqlite"

=== modified file 'hooks/config-changed'
--- hooks/config-changed	2012-02-10 00:22:21 +0000
+++ hooks/config-changed	2012-02-10 21:32:19 +0000
@@ -3,7 +3,6 @@
 # Copyright 2012 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-import base64
 import json
 import os
 import os.path
@@ -100,8 +99,8 @@
         # This file will be moved to master.cfg.original in
         # initialize_buildbot().
         with su('buildbot'), open(master_cfg_path, 'w') as f:
-            base64.decode(StringIO.StringIO(config_file), f)
-        log('config_file decoded and written.')
+            f.write(config_file)
+        log('config_file written.')
         restart_required = True
     elif (config_transport == 'bzr' and config_url and
           'config-transport' in added_or_changed and

=== modified file 'hooks/helpers.py'
--- hooks/helpers.py	2012-02-10 17:17:47 +0000
+++ hooks/helpers.py	2012-02-10 21:32:19 +0000
@@ -25,7 +25,6 @@
     'unit_info',
     ]
 
-import base64
 from collections import namedtuple
 from contextlib import contextmanager
 import json
@@ -114,13 +113,6 @@
     return cmd(*args)
 
 
-def encode_file(filename):
-    """base64 encode the contents of a file and return it."""
-    with open(filename) as f:
-        contents = f.read()
-    return base64.b64encode(contents)
-
-
 def grep(content, filename):
     with open(filename) as f:
         for line in f:
@@ -237,7 +229,7 @@
     return item
 
 
-def wait_for_unit(service_name, timeout=120):
+def wait_for_unit(service_name, timeout=480):
     start_time = time.time()
     while True:
         state = unit_info(service_name, 'state')

=== modified file 'tests/buildbot-master.test'
--- tests/buildbot-master.test	2012-02-10 17:17:47 +0000
+++ tests/buildbot-master.test	2012-02-10 21:32:19 +0000
@@ -4,7 +4,6 @@
 
 from helpers import (
     command,
-    encode_file,
     make_charm_config_file,
     unit_info,
     wait_for_page_contents,
@@ -45,7 +44,7 @@
             'buildbot-master': {
                 'extra-packages': 'git',
                 'installdir': '/tmp/buildbot',
-                'config-file': encode_file(bb_config_path),
+                'config-file': open(bb_config_path).read(),
             }}
         deploy(charm_config)