Merge lp:~racb/ubuntu-archive-tools/handle-packageset-deletions into lp:ubuntu-archive-tools

[Robie Basak]

If a packageset is deleted, the corresponding file doesn't disappear from https://people.canonical.com/~ubuntu-archive/packagesets/. I can think of two approaches to fix this:

1) Adjust packageset-report to delete files not generated by its output. This is what this branch does. However, hypothetical users might find a tool that deletes things surprising, in which case you don't want this branch.

2) Adjust the cron job that calls packageset-report to delete or rename away the output directory first. But I don't have permission to make this adjustment myself, so if you don't like the former option, please could you fix this issue in the cronjob?

Unmerged revisions

1458. By Robie Basak on 2021-04-07

Delete output from packagesets that are deleted

If a packageset is deleted, a subsequent run by this tool will retain
the output prior to that deletion in the target directory. This is
confusing, since the report output then implies that the packageset
still exists, when it doesn't.

Instead, explicitly delete output from inside that is not part of the
current report output.

1457. By Robie Basak on 2021-04-07

Factor out create_report()

Move this code into its own function to make future manipulation easier.

=== modified file 'packageset-report'
--- packageset-report	2020-11-02 18:25:35 +0000
+++ packageset-report	2021-04-07 03:56:33 +0000
@@ -18,13 +18,74 @@
 
 import argparse
 import os
+import stat
 import time
 
 from launchpadlib.launchpad import Launchpad
 from codecs import open
 
+
+def create_report(ubuntu, pkgset, _teams={}):
+    """Create a report for the given packageset
+
+    :param ubuntu: launchpadlib distribution object for Ubuntu
+    :param pkgset: launchpadlib packageset object
+    :param dict _teams: team cache (used internally)
+    :rtype: str
+    :returns: the packageset report as a string
+    """
+    report = ""
+    report += "Name: %s\n" % pkgset.name
+    report += "Description: %s\n" % pkgset.description
+    report += "Owner: %s\n" % pkgset.owner.display_name
+    report += "Creation date: %s\n" % pkgset.date_created
+
+    # List all the source packages
+    report += "\nPackages:\n"
+    for pkg in sorted(list(pkgset.getSourcesIncluded())):
+        report += " - %s\n" % str(pkg)
+
+    # List all the sub-package sets
+    report += "\nSub-package sets:\n"
+    for child in sorted(list(pkgset.setsIncluded(direct_inclusion=True))):
+        report += " - %s\n" % child.name
+
+    # List all the uploaders, when it's a team, show the members count
+    report += "\nUploaders:\n"
+    for archive in ubuntu.archives:
+        for uploader in sorted(list(archive.getUploadersForPackageset(
+                packageset=pkgset)),
+                key=lambda uploader: uploader.person.display_name):
+
+            if uploader.person.is_team:
+                if not uploader.person.name in _teams:
+                    team = uploader.person
+                    _teams[uploader.person.name] = team
+                else:
+                    team = _teams[uploader.person.name]
+
+                report += " - %s (%s) (%s) (%s) (%s members)\n" % \
+                          (team.display_name,
+                           team.name,
+                           uploader.permission,
+                           archive.displayname,
+                           len(team.members))
+                for member in sorted(list(team.members),
+                                     key=lambda person: person.name):
+                    report += "   - %s (%s)\n" % (member.display_name,
+                                                  member.name)
+            else:
+                report += " - %s (%s) (%s)\n" % \
+                          (uploader.person.name,
+                           uploader.person.display_name,
+                           uploader.permission)
+
+    report += "\nGenerated at: %s\n" % time.asctime()
+    return report
+
+
 parser = argparse.ArgumentParser(
-    description="Generate list of packages and uploaders for all packagesets.")
+    description="Update list of packages and uploaders for all packagesets.")
 parser.add_argument("target", metavar="TARGET",
                     help="Target directory")
 parser.add_argument("-a", "--all", action="store_true",
@@ -44,63 +105,29 @@
 else:
     ubuntu_series = [series for series in ubuntu.series if series.active]
 
-# cache
-teams = {}
-
 for series in ubuntu_series:
     series_name = str(series.name)
 
     if not os.path.exists(os.path.join(args.target, series_name)):
         os.makedirs(os.path.join(args.target, series_name))
 
-    for pkgset in lp.packagesets.getBySeries(distroseries=series):
-        report = ""
-        report += "Name: %s\n" % pkgset.name
-        report += "Description: %s\n" % pkgset.description
-        report += "Owner: %s\n" % pkgset.owner.display_name
-        report += "Creation date: %s\n" % pkgset.date_created
-
-        # List all the source packages
-        report += "\nPackages:\n"
-        for pkg in sorted(list(pkgset.getSourcesIncluded())):
-            report += " - %s\n" % str(pkg)
-
-        # List all the sub-package sets
-        report += "\nSub-package sets:\n"
-        for child in sorted(list(pkgset.setsIncluded(direct_inclusion=True))):
-            report += " - %s\n" % child.name
-
-        # List all the uploaders, when it's a team, show the members count
-        report += "\nUploaders:\n"
-        for archive in ubuntu.archives:
-            for uploader in sorted(list(archive.getUploadersForPackageset(
-                    packageset=pkgset)),
-                    key=lambda uploader: uploader.person.display_name):
-
-                if uploader.person.is_team:
-                    if not uploader.person.name in teams:
-                        team = uploader.person
-                        teams[uploader.person.name] = team
-                    else:
-                        team = teams[uploader.person.name]
-
-                    report += " - %s (%s) (%s) (%s) (%s members)\n" % \
-                              (team.display_name,
-                               team.name,
-                               uploader.permission,
-                               archive.displayname,
-                               len(team.members))
-                    for member in sorted(list(team.members),
-                                         key=lambda person: person.name):
-                        report += "   - %s (%s)\n" % (member.display_name,
-                                                      member.name)
-                else:
-                    report += " - %s (%s) (%s)\n" % \
-                              (uploader.person.name,
-                               uploader.person.display_name,
-                               uploader.permission)
-
-        report += "\nGenerated at: %s\n" % time.asctime()
-        with open(os.path.join(args.target, series_name, pkgset.name),
+    reports = {
+        pkgset.name: create_report(ubuntu, pkgset)
+        for pkgset in lp.packagesets.getBySeries(distroseries=series)
+    }
+
+    # Delete files in output directory that didn't result in a report
+    existing_files = set(os.listdir(os.path.join(args.target, series_name)))
+    extraneous_files = existing_files - set(reports.keys())
+    for extraneous_file in extraneous_files:
+        path = os.path.join(args.target, series_name, extraneous_file)
+        # Sanity check: only remove if it's a regular file
+        stat_result = os.lstat(path)
+        if stat.S_ISREG(stat_result.st_mode):
+            os.unlink(path)
+
+    # Write reports to disk
+    for pkgset_name, report in reports.items():
+        with open(os.path.join(args.target, series_name, pkgset_name),
                   "w+", encoding="utf-8") as fd:
             fd.write(report)