Merge lp:~benji/launchpad/faster-wadl into lp:launchpad

[Benji York]

[r=adeuring][ui=none][no-qa] Use the stdlib multiprocess module to run WADL generation in parallel subprocesses.

This branch uses the stdlib multiprocess module to run WADL generation
in parallel subprocesses. This speeds up WADL generation a fair bit:
91.9s to 70.0s on my laptop.

The change was a straight-forward refactoring to lift out the make_files
function and then use it as the target of a Process.

The script can be run by itself like so:

    LPCONFIG=development bin/py ./utilities/create-lp-wadl-and-apidoc.py --force "lib/canonical/launchpad/apidoc/wadl-development-%(version)s.xml"

[Abel Deuring (adeuring)]

Yay, less boredom for staring work on a new branch!

review: Approve (code)

=== modified file 'utilities/create-lp-wadl-and-apidoc.py'
--- utilities/create-lp-wadl-and-apidoc.py	2010-10-25 13:16:10 +0000
+++ utilities/create-lp-wadl-and-apidoc.py	2010-11-19 14:39:37 +0000
@@ -12,6 +12,7 @@
 """
 import _pythonpath # Not lint, actually needed.
 
+from multiprocessing import Process
 import optparse
 import os
 import sys
@@ -32,6 +33,31 @@
     f.close()
 
 
+def make_files(path_template, directory, version, force):
+    wadl_filename = path_template % {'version': version}
+    # If the WADL file doesn't exist or we're being forced to regenerate
+    # it...
+    if (not os.path.exists(wadl_filename) or force):
+        print "Writing WADL for version %s to %s." % (
+            version, wadl_filename)
+        write(wadl_filename, generate_wadl(version))
+    else:
+        print "Skipping already present WADL file:", wadl_filename
+
+    # Now, convert the WADL into an human-readable description and
+    # put the HTML in the same directory as the WADL.
+    html_filename = os.path.join(directory, version + ".html")
+    # If the HTML file doesn't exist or we're being forced to regenerate
+    # it...
+    if (not os.path.exists(html_filename) or force):
+        print "Writing apidoc for version %s to %s" % (
+            version, html_filename)
+        write(html_filename, generate_html(wadl_filename,
+            suppress_stderr=False))
+    else:
+        print "Skipping already present HTML file:", html_filename
+
+
 def main(path_template, force=False):
     WebServiceApplication.cached_wadl = None # do not use cached file version
     execute_zcml_for_scripts()
@@ -47,29 +73,17 @@
     f = open(index_filename, 'w')
     f.write(template(config=config))
 
+    # Start a process to build each set of WADL and HTML files.
+    processes = []
     for version in config.active_versions:
-        wadl_filename = path_template % {'version': version}
-        # If the WADL file doesn't exist or we're being forced to regenerate
-        # it...
-        if (not os.path.exists(wadl_filename) or force):
-            print "Writing WADL for version %s to %s." % (
-                version, wadl_filename)
-            write(wadl_filename, generate_wadl(version))
-        else:
-            print "Skipping already present WADL file:", wadl_filename
+        p = Process(target=make_files,
+            args=(path_template, directory, version, force))
+        p.start()
+        processes.append(p)
 
-        # Now, convert the WADL into an human-readable description and
-        # put the HTML in the same directory as the WADL.
-        html_filename = os.path.join(directory, version + ".html")
-        # If the HTML file doesn't exist or we're being forced to regenerate
-        # it...
-        if (not os.path.exists(html_filename) or force):
-            print "Writing apidoc for version %s to %s" % (
-                version, html_filename)
-            write(html_filename, generate_html(wadl_filename,
-                suppress_stderr=False))
-        else:
-            print "Skipping already present HTML file:", html_filename
+    # Wait for all the subprocesses to finish.
+    for p in processes:
+        p.join()
 
     return 0