Merge lp:~stub/pqm/config-reload into lp:pqm

[Stuart Bishop]

[Stuart Bishop (stub)]

Reload config and global state after processing each job in --run mode.

[Robert Collins (lifeless)]

The intent looks good, and given the massive globals of the time, reload() looks roughly right. However Tim has been busily refactoring things to make this better, so this is conflicting with trunk. Minimally it needs conflicts resolved, but it may be deeper than that.

review: Approve

[Robert Collins (lifeless)]

Ping; still conflicts.

Unmerged revisions

184. By Stuart Bishop on 2008-11-18

Reload config and all global state every job

=== modified file 'bin/pqm'
--- bin/pqm	2008-11-21 00:43:51 +0000
+++ bin/pqm	2009-03-26 03:50:11 +0000
@@ -72,7 +72,138 @@
         raise PQMTlaFailure(sender, ["VCS command %s %s failed (%s): %s" % (cmd, args, status, msg)] + output)
     return output
 
+<<<<<<< TREE
 def do_read_mode(logger, options, mail_reply, mail_server, from_address, fromaddr):
+=======
+def do_run_mode():
+    (goodscripts, badscripts) = ([], [])
+    scripts = find_patches(
+        queuedir, logger, rev_optionhandler, configp, options)
+    while scripts:
+        if os.path.isfile("%s/stop.patch" % queuedir):
+            break
+        run_one_script(logger, script, logdir, goodscripts, badscripts,
+            mail_reply, mail_server, from_address, fromaddr, options)
+        reload()
+        scripts = find_patches(
+            queuedir, logger, rev_optionhandler, configp, options)
+
+    if options.print_report:
+        for (patchname, logname) in goodscripts:
+            print "Patch: " + patchname
+            print "Status: success"
+            print "Log: " + logname
+            print
+        for (patchname, logname) in badscripts:
+            print "Patch: " + patchname
+            print "Status: failure"
+            print "Log: " + logname
+            print
+
+def run_one_script(logger, script, logdir, goodscripts, badscripts,
+                   mail_reply, mail_server, from_address, fromaddr, options):
+    # FIXME: This is currently extremely hard to test. move it to the library,
+    # and test it!
+    try:
+        success = False
+        try:
+            logger.info('trying script ' + script.filename)
+            logname = os.path.join(logdir, os.path.basename(script.filename) + '.log')
+            (sender, subject, msg, sig) = read_email(logger, open(script.filename))
+            if options.verify_sigs:
+                sigid,siguid = verify_sig(
+                    script.getSender(), msg, sig, 0, logger, options.keyring)
+            output = []
+            failedcmd=None
+    
+            # ugly transitional code
+            pqm.logger = logger
+            pqm.workdir = workdir
+            pqm.runtla = runtla
+            pqm.precommit_hook = precommit_hook
+            (successes, unrecognized, output) = script.run()
+
+            logger.info('successes: %s' % (successes,))
+            logger.info('unrecognized: %s' % (unrecognized,))
+            success = True
+            goodscripts.append((script.filename, logname))
+        except PQMCmdFailure, e:
+            badscripts.append((script.filename, logname))
+            successes = e.goodcmds
+            failedcmd = e.badcmd
+            output = e.output
+            unrecognized=[]
+        except PQMException, e:
+            badscripts.append((script.filename, logname))
+            successes = []
+            failedcmd = []
+            output = [str(e)]
+            unrecognized=[]
+        except Exception, e:
+            # catch all to ensure we get some output in uncaught failures
+            output = [str(e)]
+            raise
+        if mail_reply:
+            send_mail_reply(success, successes, unrecognized,
+                            mail_server, from_address, script.getSender(),
+                            fromaddr, failedcmd, output, script)
+        else:
+            logger.info('not sending mail reply')
+    finally:
+        # ensure we always unlink the script file.
+        log_list(logname, output)
+        os.unlink(script.filename)
+
+def send_mail_reply(success, successes, unrecognized, mail_server, from_address, sender, fromaddr, failedcmd, output, script):
+    if success:
+        retmesg = mail_format_successes(successes, "Command was successful.", unrecognized)
+        if len(successes) > 0:
+            statusmsg='success'
+        else:
+            statusmsg='no valid commands given'
+    else:
+        retmesg = mail_format_successes(successes, "Command passed checks, but was not committed.", unrecognized)
+        retmesg+= "\n%s" % failedcmd
+        retmesg+= '\nCommand failed!'
+        if not script.debug:
+            retmesg += '\nLast 20 lines of log output:'
+            retmesg += ''.join(output[-20:])
+        else:
+            retmesg += '\nAll lines of log output:'
+            retmesg += ''.join(output)
+        statusmsg='failure'
+    server = smtplib.SMTP(mail_server)
+    server.sendmail(from_address, [sender], 'From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s\n' % (fromaddr, sender, statusmsg, retmesg))
+    server.quit()
+
+def mail_format_successes(successes, command_msg, unrecognized):
+    retmesg = []
+    for success in successes:
+        retmesg.append('> ' + success)
+        retmesg.append(command_msg)
+        for line in unrecognized:
+            retmesg.append('> ' + line)
+            retmesg.append('Unrecognized command.')
+    return string.join(retmesg, '\n')
+
+def log_list(logname, list):
+    f = open(logname, 'w')
+    for l in list:
+        f.write(l)
+    f.close()
+
+def run():
+    lockfile=LockFile(os.path.join(pqm_subdir, 'pqm.lock'), logger,
+        options.no_act, options.cron_mode)
+    lockfile.acquire()
+    try:
+        if options.run_mode:
+            do_run_mode()
+    finally:
+        lockfile.release()
+
+def do_read_mode():
+>>>>>>> MERGE-SOURCE
     sender = None
     try:
         (sender, subject, msg, sig) = read_email(logger)
@@ -98,6 +229,7 @@
         sys.exit(1)
     sys.exit(0)
 
+<<<<<<< TREE
 arch_path = 'baz'
 arch_impl = None
 logfile_name = 'pqm.log'
@@ -220,9 +352,151 @@
         filehandler.setLevel(logging.DEBUG)
     logger.addHandler(filehandler)
     filehandler.setFormatter(logging.Formatter(
+=======
+def reload():
+    """Load or reload all our configurable state."""
+    global pqm_subdir, queuedir, logger, logdir, mail_reply, mail_server
+    global from_address, fromaddr, options, configp
+
+    arch_path = 'baz'
+    arch_impl = None
+    logfile_name = 'pqm.log'
+    default_mail_log_level = logging.ERROR
+    mail_server = 'localhost'
+    queuedir = None
+    workdir = None
+    logdir = None
+    mail_reply = 1
+    from_address = None
+    precommit_hook = []
+
+    (options, args) = parse_command_line(sys.argv[1:])
+
+    if options.show_version:
+        print "pqm %s" % pqm.__version__
+        sys.exit(0)
+
+    logger = logging.getLogger("pqm")
+    # TODO: move the logging confuration somewhere else.
+    logger.setLevel(logging.DEBUG)
+    stderr_handler = logging.StreamHandler(strm=sys.stderr)
+    stderr_handler.setLevel(options.loglevel)
+    stderr_handler.setFormatter(logging.Formatter(
+        fmt="%(name)s [%(thread)d] %(levelname)s: %(message)s"))
+    logger.addHandler(stderr_handler)
+
+    if not (options.read_mode or options.run_mode):
+        logger.error("Either --read or --run must be specified")
+        sys.exit(1)
+
+    configp = ConfigParser()
+    logger.debug("Reading config files: %s" % (options.configfile_names,))
+    configp.read(options.configfile_names)
+
+    if configp.has_option('DEFAULT', 'arch_path'):
+        arch_path = configp.get('DEFAULT', 'arch_path')
+    elif configp.has_option('DEFAULT', 'tlapath'):
+        logger.warn("Option 'tlapath' is deprecated")
+        arch_path = configp.get('DEFAULT', 'tlapath')
+
+    if os.access(arch_path, os.X_OK):
+        logger.error(
+            "Can't execute \"%s\", please fix arch_path" % (arch_path,))
+        sys.exit(1)
+
+    # ugly transitional code
+    pqm.logger = logger
+    if configp.has_option('DEFAULT', 'arch_impl'):
+        impl = configp.get('DEFAULT', 'arch_impl')
+        if impl == 'tla':
+            arch_impl = TlaHandler()
+        elif impl == 'arx':
+            arch_impl = ArXHandler()
+        elif impl == 'baz':
+            arch_impl = Baz1_1Handler()
+        elif impl == 'baz1.0':
+            arch_impl = Baz1_0Handler()
+        else:
+            logger.error("Unknown arch_impl \"%s\"" % (impl,))
+            sys.exit(1)
+    # FIXME: move this into a nicer place.
+    Command.arch_impl = arch_impl
+    Command.arch_path = arch_path
+
+    pqm.gpgv_path = configp.get_option('DEFAULT', 'gpgv_path', 'gpgv')
+    myname = configp.get_option(
+        'DEFAULT', 'myname', 'Arch Patch Queue Manager')
+
+    if configp.has_option('DEFAULT', 'from_address'):
+        from_address = configp.get('DEFAULT', 'from_address')
+    else:
+        logger.error("No from_address specified")
+        sys.exit(1)
+    fromaddr = '%s <%s>' % (myname, from_address)
+
+    mail_reply=configp.get_boolean_option('DEFAULT', 'mail_reply',1)
+    # The command line parameter overrides the setting in the config file.
+    if options.verify_sigs:
+        options.verify_sigs = configp.get_boolean_option(
+            'DEFAULT', 'verify_sigs', True)
+
+    if options.queuedir:
+        queuedir = options.queuedir
+    else:
+        queuedir = get_queuedir(configp, logger, args)
+    queuedir=os.path.abspath(queuedir)
+    pqm_subdir = os.path.join(queuedir, 'pqm')
+    pqm.pqm_subdir = pqm_subdir
+
+    if not configp.has_option('DEFAULT', 'dont_set_home'):
+        os.environ['HOME'] = queuedir
+
+    workdir=dir_from_option(configp, 'workdir', 'workdir')
+    logdir=dir_from_option(configp, 'logdir', 'logs')
+
+    if not options.keyring:
+        if configp.has_option('DEFAULT', 'keyring'):
+            options.keyring = configp.get('DEFAULT', 'keyring')
+        else:
+            logger.error(
+                "No keyring specified on command line or in config files.")
+            sys.exit(1)
+    if not os.access(options.keyring, os.R_OK):
+        logger.error("Couldn't access keyring %s" % (options.keyring,))
+        sys.exit(1)
+
+    do_mkdir(queuedir, options.no_act)
+    os.chdir(queuedir)
+    do_mkdir(workdir, options.no_act)
+    do_mkdir(logdir, options.no_act)
+    do_mkdir(pqm_subdir, options.no_act)
+
+    rev_optionhandler = pqm.BranchSpecOptionHandler(configp, queuedir=queuedir)
+    if len(rev_optionhandler._specs) == 0:
+        logger.error("No branches to manage!")
+        sys.exit(1)
+    for spec in rev_optionhandler._specs:
+        logger.info("managing branch(s): " + spec)
+
+    if configp.has_option('DEFAULT', 'logfile'):
+        logfile_name = configp.get('DEFAULT', 'logfile')
+
+    if not options.no_log:
+        if not os.path.isabs(logfile_name):
+            logfile_name = os.path.join(pqm_subdir, logfile_name)
+        logger.debug("Adding log file: %s" % (logfile_name,))
+        filehandler = logging.FileHandler(logfile_name)
+        if options.loglevel >= logging.WARN:
+            filehandler.setLevel(logging.INFO)
+        else:
+            filehandler.setLevel(logging.DEBUG)
+        logger.addHandler(filehandler)
+        filehandler.setFormatter(logging.Formatter(
+>>>>>>> MERGE-SOURCE
             fmt="%(asctime)s %(name)s [%(thread)d] %(levelname)s: %(message)s",
             datefmt="%b %d %H:%M:%S"))
 
+<<<<<<< TREE
 if not options.debug_mode:
     # Don't log to stderr past this point
     logger.removeHandler(stderr_handler)
@@ -246,4 +520,27 @@
 
 logger.info("main thread exiting...")
 sys.exit(0)
+=======
+    if not options.debug_mode:
+        # Don't log to stderr past this point
+        logger.removeHandler(stderr_handler)
+
+    transaction_file = os.path.join(queuedir, 'transactions-completed')
+    if os.access(transaction_file, os.R_OK):
+        lines = open(transaction_file).readlines()
+        for line in lines:
+            pqm.used_transactions[line[0:-1]] = 1
+
+if __name__ == '__main__':
+    reload()
+
+    if options.read_mode:
+        do_read_mode(logger, options, transaction_file)
+
+    assert(options.run_mode)
+
+    run()
+    logger.info("main thread exiting...")
+    sys.exit(0)
+>>>>>>> MERGE-SOURCE