Merge lp:~james-w/ubuntu-archive-tools/improved-sync-helper into lp:ubuntu-archive-tools

[James Westby]

[James Westby (james-w)]

Hi Steven,

Please review my changes to sync-helper.py and see if you like
them. I didn't want to unilaterally make them to your script,
but I think these improvements make it a lot more usable.

Thanks for writing the initial version. Processing syncs is much
quicker with this script now.

Thanks,

James

[lp:~james-w/ubuntu-archive-tools/improved-sync-helper]

91. By James Westby on 2010-01-04

Allow changing suite back to unstable.

92. By James Westby on 2010-01-04

Mark commenters that can upload the package with (*).

This makes it easier to see if sponsorship is still needed.

93. By James Westby on 2010-01-04

Get all the info up-front to speed up processing.

94. By James Westby on 2010-01-04

Add progress output.

Also avoid another call to rmadison by re-using the info that we have.

95. By James Westby on 2010-01-04

Fix a thinko that caused crashes when looking up the component

96. By James Westby on 2010-01-04

Also consider package sets when looking for uploaders.

[Steve Kowalik (stevenk)]

Hi James,

This all looks totally awesome! Thanks for the great changes, I'll merge them into trunk soonish.

review: Approve

=== modified file 'sync-helper.py'
--- sync-helper.py	2009-11-20 20:29:12 +0000
+++ sync-helper.py	2010-01-04 19:09:12 +0000
@@ -3,22 +3,108 @@
 # Copyright 2009 Canonical Ltd.
 
 import os, sys
-from launchpadlib.launchpad import Launchpad, STAGING_SERVICE_ROOT
+import subprocess
+from launchpadlib.launchpad import Launchpad, EDGE_SERVICE_ROOT, STAGING_SERVICE_ROOT
+import webbrowser
 
 APP_NAME = 'sync-helper'
 CACHE_DIR = os.path.expanduser('~/.launchpadlib/cache')
-SERVICE_ROOT = STAGING_SERVICE_ROOT
+SERVICE_ROOT = EDGE_SERVICE_ROOT
+
+
+class Bug(object):
+
+    def __init__(self, number):
+        self.number = number
+        self.title = None
+        self.package = None
+        self.reporter = None
+        self.content = None
+
 
 def length(collection):
     # XXX: Workaround bug 274074. Thanks wgrant.
     return int(collection._wadl_resource.representation['total_size'])
 
-def prompt(text):
-    print "%s [Show/sKip/sYnc] " % text
-    return sys.stdin.readline().rstrip()
-
-def sync(bug, syncfile):
-    syncfile.write("sync %s -f\n" % bug)
+def rmadison(distro, package):
+    proc = subprocess.Popen(["/usr/bin/rmadison", "-u", distro, package],
+            stdout=subprocess.PIPE)
+    (output, _) = proc.communicate()
+    return output.splitlines()
+
+def get_components(package, debian_details):
+    components = {}
+    for line in debian_details.splitlines():
+        if line.strip() == "":
+            continue
+        series_component = line.split("|")[2].strip()
+        archs = line.split("|")[3].strip().split(", ")
+        if "source" in archs:
+            if "/" in series_component:
+                series, component = series_component.split("/")
+                components[series] = component
+            else:
+                components[series_component] = "main"
+    return components
+
+def ubuntu_details(package, current_series):
+    output = rmadison("ubuntu", package)
+    new_output = []
+    for line in output:
+        parts = line.split("|")
+        if current_series in parts[2]:
+            new_output.append(line)
+    return new_output
+
+def debian_details(package):
+    return rmadison("debian", package)
+
+def canUpload(distribution, person, package_name):
+    if distribution.main_archive.isSourceUploadAllowed(
+            distroseries=distribution.current_series, person=person,
+            sourcepackagename=package_name):
+        return True
+    pubs = distribution.main_archive.getPublishedSources(
+            distro_series=distribution.current_series, exact_match=True,
+            pocket="Release", status="Published", source_name=package_name)
+    pubs = [pub for pub in pubs]
+    if len(pubs) < 1:
+        return False
+    component = pubs[0].component_name
+    for perm in distribution.main_archive.getPermissionsForPerson(person=person):
+        if perm.permission != 'Archive Upload Rights':
+            continue
+        if perm.component_name == component:
+            return True
+        if perm.source_package_name == package_name:
+            return True
+    return False
+
+def prompt(bug, suite, full=True):
+    sys.stdout.write("%s: %s\n" % (bug.number, bug.title))
+    if full:
+        sys.stdout.write(bug.ubuntu_details + "\n")
+        sys.stdout.write(bug.debian_details + "\n")
+    msg = "[Show/sKip/sYnc from %s/" % (suite,)
+    if suite == "unstable" or suite == "experimental":
+        msg += "switch to Testing/"
+    elif suite == "testing":
+        msg += "switch to Unstable/"
+    if suite == "testing" or suite == "unstable":
+        msg += "switch to Experimental/"
+    elif suite == "experimental":
+        msg += "switch to Unstable/"
+    msg += "Open bug]: "
+    sys.stdout.write(msg)
+    sys.stdout.flush()
+    return sys.stdin.readline().rstrip().upper()
+
+def sync(bug, syncfile, suite="unstable", component="main"):
+    params = {"bug": bug.number,
+              "suite": suite,
+              "component": component,
+             }
+    syncfile.write("sync %(bug)s -f -S %(suite)s -C %(component)s\n" % params)
 
 def main(args):
     f = None
@@ -27,35 +113,78 @@
     except:
         print "%s: Require a filename to write to!" % args[0]
         sys.exit(1)
-    print "Connecting to LP ... "
+    sys.stdout.write("Connecting to LP ... ")
+    sys.stdout.flush()
     launchpad = Launchpad.login_with(APP_NAME, SERVICE_ROOT, CACHE_DIR)
     pillar = launchpad.projects['ubuntu']
     subscriber = launchpad.people['ubuntu-archive']
-    print "Done"
-    print "Loading bugs ... "
-    bugs = pillar.searchTasks(bug_subscriber = subscriber, search_text = 'sync')
-    bugs_cache = {}
-    bug_numbers = {}
+    ubuntu_series = pillar.current_series.name
+    sys.stdout.write("Done\n")
+    sys.stdout.write("Loading bugs (this may take some time, go get a "
+            "cup of tea) ... \n")
+    sync_bugs = []
+    bugs = pillar.searchTasks(bug_subscriber=subscriber, search_text='qcad')
+    num_bugs = length(bugs)
+    bugs_processed = 0
+    prev_line = ""
     for bugtask in bugs:
-        bugs_cache[bugtask.bug.title] = bugtask.bug.description
-        for i in reversed(bugtask.bug.messages_collection):
-            bugs_cache[bugtask.bug.title] += i.content
-            bug_numbers[bugtask.bug.title] = bugtask.bug.id
-    print "Done"
+        bugs_processed += 1
+        sys.stdout.write("\r" + " "*len(prev_line) + "\r")
+        prev_line = "%d/%d" % (bugs_processed, num_bugs)
+        sys.stdout.write(prev_line)
+        sys.stdout.flush()
+        if " (Ubuntu)" not in bugtask.bug_target_name:
+            continue
+        bug = Bug(bugtask.bug.id)
+        bug.title = bugtask.bug.title
+        bug.package = bugtask.bug_target_name.rsplit(" ", 1)[0]
+        next_line_part = ": %s" % (bug.package,)
+        sys.stdout.write(next_line_part)
+        prev_line += next_line_part
+        sys.stdout.flush()
+        bug.content = bugtask.bug.description
+        bug.reporter = bugtask.bug.owner
+        for i in reversed(bugtask.bug.messages):
+            bug.content += "\n=============================================\n"
+            star = ""
+            if canUpload(pillar, i.owner, bug.package):
+                star = "(*)"
+            bug.content += "%s%s said %s on %s:\n\n" % (i.owner.name, star,
+                    i.subject, str(i.date_created))
+            bug.content += i.content
+        bug.ubuntu_details = "\n".join(ubuntu_details(bug.package, ubuntu_series))
+        bug.debian_details = "\n".join(debian_details(bug.package))
+        bug.components = get_components(bug.package, bug.debian_details)
+        sync_bugs.append(bug)
+    sys.stdout.write("\r" + " "*len(prev_line) + "\r")
+    sys.stdout.write("Done, %d packages to consider.\n" % (len(sync_bugs),))
 
-    for title in bugs_cache.keys():
-        ret = prompt(title)
-        if ret == 'S':
-            print bugs_cache[title]
-            ret = prompt(title)
-            if ret == 'Y':
-                sync(bug_numbers[title], f)
+    for bug in sync_bugs:
+        sys.stdout.write("\n==========================\n")
+        finished = False
+        suite = "unstable"
+        component = "main"
+        if suite in bug.components:
+            component = bug.components[suite]
+        full = True
+        while not finished:
+            ret = prompt(bug, suite, full=full)
+            if ret == 'S':
+                print bug.content
             elif ret == 'K':
-                continue
-        elif ret == 'K':
-            continue
-        elif ret == 'Y':
-            sync(bug_numbers[title], f)
+                finished = True
+            elif ret == 'Y':
+                sync(bug, f, suite=suite, component=component)
+                finished = True
+            elif ret == 'T':
+                suite = "testing"
+            elif ret == 'U':
+                suite = "unstable"
+            elif ret == 'E':
+                suite = "experimental"
+            elif ret == 'O':
+                webbrowser.open("https://bugs.launchpad.net/bugs/%s" % bug.number)
+            full = False
     f.close()
     return 0