Merge lp:~wgrant/launchpad/observer-db into lp:launchpad/db-devel

[William Grant]

Initial DB schema for access policies.

This is the initial DB patch for Disclosure's access policies. The migration process will be long and tortuous, but this is a start.

A private artifact (currently a bug or a branch) will be linked to one of its target's access policies, and attempts to access the artifact will be checked against the policy's permissions. This will shortly supplant the existing subscription-and-other-stuff-based checks.

Permissions can be either policy-global or artifact-specific. In the first case APP.artifact is left unset, letting the permission holder see all artifacts under the policy. For the second case both policy and artifact are set, restricting the access to a specific artifact under a specific policy.

The identification of access policies isn't well-defined yet, but this schema will do for now. The intent is that, at least for the initial pass, projects will have preconfigured and immutable "Private" and "Security" policies, with the existing bug privacy/security checkboxes altered to map onto these policies. So we either have to use an enum, or just treat these as well-known names until a later pass.

=== modified file 'cronscripts/check-teamparticipation.py'
--- cronscripts/check-teamparticipation.py	2011-09-18 03:43:46 +0000
+++ cronscripts/check-teamparticipation.py	2011-11-03 02:37:26 +0000
@@ -20,77 +20,8 @@
 
 import _pythonpath
 
-import transaction
-
-from canonical.database.sqlbase import cursor
-from lp.services.scripts.base import LaunchpadScript, LaunchpadScriptFailure
-
-
-def check_teamparticipation(log):
-    # Check self-participation.
-    query = """
-        SELECT id, name
-        FROM Person WHERE id NOT IN (
-            SELECT person FROM Teamparticipation WHERE person = team
-            ) AND merged IS NULL
-        """
-    cur = cursor()
-    cur.execute(query)
-    non_self_participants = cur.fetchall()
-    if len(non_self_participants) > 0:
-        log.warn("Some people/teams are not members of themselves: %s"
-                 % non_self_participants)
-
-    # Check if there are any circular references between teams.
-    cur.execute("""
-        SELECT tp.team, tp2.team
-        FROM teamparticipation AS tp, teamparticipation AS tp2
-        WHERE tp.team = tp2.person
-            AND tp.person = tp2.team
-            AND tp.id != tp2.id;
-        """)
-    circular_references = cur.fetchall()
-    if len(circular_references) > 0:
-        raise LaunchpadScriptFailure(
-            "Circular references found: %s" % circular_references)
-
-    # Check if there are any missing/spurious TeamParticipation entries.
-    cur.execute("SELECT id FROM Person WHERE teamowner IS NOT NULL")
-    team_ids = cur.fetchall()
-    transaction.abort()
-
-    def get_participants(team):
-        """Recurse through the team's members to get all its participants."""
-        participants = set()
-        for member in team.activemembers:
-            participants.add(member)
-            if member.is_team:
-                participants.update(get_participants(member))
-        return participants
-
-    from lp.registry.model.person import Person
-    batch = team_ids[:50]
-    team_ids = team_ids[50:]
-    while batch:
-        for [id] in batch:
-            team = Person.get(id)
-            expected = get_participants(team)
-            found = set(team.allmembers)
-            difference = expected.difference(found)
-            if len(difference) > 0:
-                people = ", ".join("%s (%s)" % (person.name, person.id)
-                                   for person in difference)
-                log.warn("%s (%s): missing TeamParticipation entries for %s."
-                         % (team.name, team.id, people))
-            reverse_difference = found.difference(expected)
-            if len(reverse_difference) > 0:
-                people = ", ".join("%s (%s)" % (person.name, person.id)
-                                   for person in reverse_difference)
-                log.warn("%s (%s): spurious TeamParticipation entries for %s."
-                         % (team.name, team.id, people))
-            transaction.abort()
-        batch = team_ids[:50]
-        team_ids = team_ids[50:]
+from lp.registry.scripts.teamparticipation import check_teamparticipation
+from lp.services.scripts.base import LaunchpadScript
 
 
 class CheckTeamParticipationScript(LaunchpadScript):
@@ -99,5 +30,6 @@
     def main(self):
         check_teamparticipation(self.logger)
 
+
 if __name__ == '__main__':
     CheckTeamParticipationScript("check-teamparticipation").run()

=== modified file 'database/schema/comments.sql'
--- database/schema/comments.sql	2011-10-07 11:32:23 +0000
+++ database/schema/comments.sql	2011-11-03 02:37:26 +0000
@@ -6,6 +6,26 @@
      GNU Affero General Public License version 3 (see the file LICENSE).
 */
 
+-- AccessPolicy
+
+COMMENT ON TABLE AccessPolicy IS 'A policy to manage access to a pillar\'s artifacts.';
+COMMENT ON COLUMN AccessPolicy.product IS 'The product that this policy applies to.';
+COMMENT ON COLUMN AccessPolicy.distribution IS 'The distribution that this policy applies to.';
+COMMENT ON COLUMN AccessPolicy.display_name IS 'A human-readable name for this policy.';
+
+-- AccessPolicyArtifact
+
+COMMENT ON TABLE AccessPolicyArtifact IS 'An artifact that an access permission can apply to.';
+COMMENT ON COLUMN AccessPolicyArtifact.bug IS 'The bug that this abstract artifact represents.';
+COMMENT ON COLUMN AccessPolicyArtifact.branch IS 'The branch that this abstract artifact represents.';
+
+-- AccessPolicyPermission
+
+COMMENT ON TABLE AccessPolicyPermission IS 'A permission for a person to access a policy\'s artifacts.';
+COMMENT ON COLUMN AccessPolicyPermission.policy IS 'The access policy on which access is granted.';
+COMMENT ON COLUMN AccessPolicyPermission.person IS 'The person that holds the permission.';
+COMMENT ON COLUMN AccessPolicyPermission.artifact IS 'The optional artifact to which the access is restricted.';
+
 -- Announcement
 
 COMMENT ON TABLE Announcement IS 'A project announcement. This is a single item of news or information that the project is communicating. Announcements can be attached to a Project, a Product or a Distribution.';

=== added file 'database/schema/patch-2208-93-1.sql'
--- database/schema/patch-2208-93-1.sql	1970-01-01 00:00:00 +0000
+++ database/schema/patch-2208-93-1.sql	2011-11-03 02:37:26 +0000
@@ -0,0 +1,46 @@
+-- Copyright 2011 Canonical Ltd.  This software is licensed under the
+-- GNU Affero General Public License version 3 (see the file LICENSE).
+SET client_min_messages=ERROR;
+
+CREATE TABLE AccessPolicy (
+    id serial PRIMARY KEY,
+    product integer REFERENCES Product,
+    distribution integer REFERENCES Distribution,
+    display_name text NOT NULL,
+    CONSTRAINT accesspolicy__product__display_name__key
+        UNIQUE (product, display_name),
+    CONSTRAINT accesspolicy__distribution__display_name__key
+        UNIQUE (distribution, display_name),
+    CONSTRAINT has_target CHECK (product IS NULL != distribution IS NULL)
+);
+
+CREATE TABLE AccessPolicyArtifact (
+    id serial PRIMARY KEY,
+    bug integer REFERENCES Bug,
+    branch integer REFERENCES Branch,
+    CONSTRAINT accesspolicyartifact__bug__key UNIQUE (bug),
+    CONSTRAINT accesspolicyartifact__branch__key UNIQUE (branch),
+    CONSTRAINT has_artifact CHECK (bug IS NULL != branch IS NULL)
+);
+
+CREATE TABLE AccessPolicyPermission (
+    id serial PRIMARY KEY,
+    policy integer NOT NULL REFERENCES AccessPolicy,
+    person integer NOT NULL REFERENCES Person,
+    artifact integer REFERENCES AccessPolicyArtifact,
+    CONSTRAINT accesspolicypermission__policy__person__artifact__key
+        UNIQUE (policy, person, artifact)
+);
+
+CREATE UNIQUE INDEX accesspolicypermission__policy__person__key
+    ON AccessPolicyPermission(policy, person) WHERE artifact IS NULL;
+
+ALTER TABLE bug
+    ADD COLUMN access_policy integer REFERENCES AccessPolicy;
+CREATE INDEX bug__access_policy__idx ON bug(access_policy);
+
+ALTER TABLE branch
+    ADD COLUMN access_policy integer REFERENCES AccessPolicy;
+CREATE INDEX branch__access_policy__idx ON branch(access_policy);
+
+INSERT INTO LaunchpadDatabaseRevision VALUES (2208, 93, 1);

=== modified file 'lib/lp/archivepublisher/domination.py'
--- lib/lp/archivepublisher/domination.py	2011-10-18 11:56:09 +0000
+++ lib/lp/archivepublisher/domination.py	2011-11-03 02:37:26 +0000
@@ -53,6 +53,7 @@
 __all__ = ['Dominator']
 
 from datetime import timedelta
+from operator import itemgetter
 
 import apt_pkg
 from storm.expr import (
@@ -67,6 +68,9 @@
     flush_database_updates,
     sqlvalues,
     )
+from canonical.launchpad.components.decoratedresultset import (
+    DecoratedResultSet,
+    )
 from canonical.launchpad.interfaces.lpstorm import IStore
 from lp.registry.model.sourcepackagename import SourcePackageName
 from lp.services.database.bulk import load_related
@@ -258,7 +262,8 @@
         # Go through publications from latest version to oldest.  This
         # makes it easy to figure out which release superseded which:
         # the dominant is always the oldest live release that is newer
-        # than the one being superseded.
+        # than the one being superseded.  In this loop, that means the
+        # dominant is always the last live publication we saw.
         publications = sorted(
             publications, cmp=generalization.compare, reverse=True)
 
@@ -272,25 +277,29 @@
         for pub in publications:
             version = generalization.getPackageVersion(pub)
             # There should never be two published releases with the same
-            # version.  So this comparison is really a string
-            # comparison, not a version comparison: if the versions are
-            # equal by either measure, they're from the same release.
-            if dominant_version is not None and version == dominant_version:
+            # version.  So it doesn't matter whether this comparison is
+            # really a string comparison or a version comparison: if the
+            # versions are equal by either measure, they're from the same
+            # release.
+            if version == dominant_version:
                 # This publication is for a live version, but has been
                 # superseded by a newer publication of the same version.
                 # Supersede it.
                 pub.supersede(current_dominant, logger=self.logger)
                 self.logger.debug2(
                     "Superseding older publication for version %s.", version)
-            elif (version in live_versions or
-                  (not generalization.is_source and
-                   not self._checkArchIndep(pub))):
+            elif version in live_versions:
                 # This publication stays active; if any publications
                 # that follow right after this are to be superseded,
                 # this is the release that they are superseded by.
                 current_dominant = pub
                 dominant_version = version
                 self.logger.debug2("Keeping version %s.", version)
+            elif not (generalization.is_source or self._checkArchIndep(pub)):
+                # As a special case, we keep this version live as well.
+                current_dominant = pub
+                dominant_version = version
+                self.logger.debug2("Keeping version %s.", version)
             elif current_dominant is None:
                 # This publication is no longer live, but there is no
                 # newer version to supersede it either.  Therefore it
@@ -442,72 +451,81 @@
             # always equals to "scheduleddeletiondate - quarantine".
             pub_record.datemadepending = UTC_NOW
 
+    def findBinariesForDomination(self, distroarchseries, pocket):
+        """Find binary publications that need dominating.
+
+        This is only for traditional domination, where the latest published
+        publication is always kept published.  It will ignore publications
+        that have no other publications competing for the same binary package.
+        """
+        # Avoid circular imports.
+        from lp.soyuz.model.publishing import BinaryPackagePublishingHistory
+
+        bpph_location_clauses = [
+            BinaryPackagePublishingHistory.status ==
+                PackagePublishingStatus.PUBLISHED,
+            BinaryPackagePublishingHistory.distroarchseries ==
+                distroarchseries,
+            BinaryPackagePublishingHistory.archive == self.archive,
+            BinaryPackagePublishingHistory.pocket == pocket,
+            ]
+        candidate_binary_names = Select(
+            BinaryPackageName.id,
+            And(
+                BinaryPackageRelease.binarypackagenameID ==
+                    BinaryPackageName.id,
+                BinaryPackagePublishingHistory.binarypackagereleaseID ==
+                    BinaryPackageRelease.id,
+                bpph_location_clauses,
+            ),
+            group_by=BinaryPackageName.id,
+            having=Count(BinaryPackagePublishingHistory.id) > 1)
+        main_clauses = [
+            BinaryPackageRelease.id ==
+                BinaryPackagePublishingHistory.binarypackagereleaseID,
+            BinaryPackageRelease.binarypackagenameID.is_in(
+                candidate_binary_names),
+            BinaryPackageRelease.binpackageformat !=
+                BinaryPackageFormat.DDEB,
+            ]
+        main_clauses.extend(bpph_location_clauses)
+
+        store = IStore(BinaryPackagePublishingHistory)
+        return store.find(BinaryPackagePublishingHistory, *main_clauses)
+
     def dominateBinaries(self, distroseries, pocket):
         """Perform domination on binary package publications.
 
         Dominates binaries, restricted to `distroseries`, `pocket`, and
         `self.archive`.
         """
-        # Avoid circular imports.
-        from lp.soyuz.model.publishing import BinaryPackagePublishingHistory
-
         generalization = GeneralizedPublication(is_source=False)
 
         for distroarchseries in distroseries.architectures:
             self.logger.info(
                 "Performing domination across %s/%s (%s)",
-                distroseries.name, pocket.title,
+                distroarchseries.distroseries.name, pocket.title,
                 distroarchseries.architecturetag)
 
-            bpph_location_clauses = [
-                BinaryPackagePublishingHistory.status ==
-                    PackagePublishingStatus.PUBLISHED,
-                BinaryPackagePublishingHistory.distroarchseries ==
-                    distroarchseries,
-                BinaryPackagePublishingHistory.archive == self.archive,
-                BinaryPackagePublishingHistory.pocket == pocket,
-                ]
-            candidate_binary_names = Select(
-                BinaryPackageName.id,
-                And(
-                    BinaryPackageRelease.binarypackagenameID ==
-                        BinaryPackageName.id,
-                    BinaryPackagePublishingHistory.binarypackagereleaseID ==
-                        BinaryPackageRelease.id,
-                    bpph_location_clauses,
-                ),
-                group_by=BinaryPackageName.id,
-                having=Count(BinaryPackagePublishingHistory.id) > 1)
-            main_clauses = [
-                BinaryPackagePublishingHistory,
-                BinaryPackageRelease.id ==
-                    BinaryPackagePublishingHistory.binarypackagereleaseID,
-                BinaryPackageRelease.binarypackagenameID.is_in(
-                    candidate_binary_names),
-                BinaryPackageRelease.binpackageformat !=
-                    BinaryPackageFormat.DDEB,
-                ]
-            main_clauses.extend(bpph_location_clauses)
-
-            def do_domination(pass2_msg=""):
-                msg = "binaries..." + pass2_msg
-                self.logger.info("Finding %s" % msg)
-                bins = IStore(
-                    BinaryPackagePublishingHistory).find(*main_clauses)
-                self.logger.info("Dominating %s" % msg)
-                sorted_packages = self._sortPackages(bins, generalization)
-                self._dominatePublications(sorted_packages, generalization)
-
-            do_domination()
-
-            # We need to make a second pass to cover the cases where:
-            #  * arch-specific binaries were not all dominated before arch-all
-            #    ones that depend on them
-            #  * An arch-all turned into an arch-specific, or vice-versa
-            #  * A package is completely schizophrenic and changes all of
-            #    its binaries between arch-all and arch-any (apparently
-            #    occurs sometimes!)
-            do_domination("(2nd pass)")
+            self.logger.info("Finding binaries...")
+            bins = self.findBinariesForDomination(distroarchseries, pocket)
+            sorted_packages = self._sortPackages(bins, generalization)
+            self.logger.info("Dominating binaries...")
+            self._dominatePublications(sorted_packages, generalization)
+
+        # We need to make a second pass to cover the cases where:
+        #  * arch-specific binaries were not all dominated before arch-all
+        #    ones that depend on them
+        #  * An arch-all turned into an arch-specific, or vice-versa
+        #  * A package is completely schizophrenic and changes all of
+        #    its binaries between arch-all and arch-any (apparently
+        #    occurs sometimes!)
+        for distroarchseries in distroseries.architectures:
+            self.logger.info("Finding binaries...(2nd pass)")
+            bins = self.findBinariesForDomination(distroarchseries, pocket)
+            sorted_packages = self._sortPackages(bins, generalization)
+            self.logger.info("Dominating binaries...(2nd pass)")
+            self._dominatePublications(sorted_packages, generalization)
 
     def _composeActiveSourcePubsCondition(self, distroseries, pocket):
         """Compose ORM condition for restricting relevant source pubs."""
@@ -522,21 +540,11 @@
             SourcePackagePublishingHistory.pocket == pocket,
             )
 
-    def dominateSources(self, distroseries, pocket):
-        """Perform domination on source package publications.
-
-        Dominates sources, restricted to `distroseries`, `pocket`, and
-        `self.archive`.
-        """
+    def findSourcesForDomination(self, distroseries, pocket):
+        """Find binary publications that need dominating."""
         # Avoid circular imports.
         from lp.soyuz.model.publishing import SourcePackagePublishingHistory
 
-        generalization = GeneralizedPublication(is_source=True)
-
-        self.logger.debug(
-            "Performing domination across %s/%s (Source)",
-            distroseries.name, pocket.title)
-
         spph_location_clauses = self._composeActiveSourcePubsCondition(
             distroseries, pocket)
         having_multiple_active_publications = (
@@ -546,12 +554,33 @@
             And(join_spph_spr(), join_spr_spn(), spph_location_clauses),
             group_by=SourcePackageName.id,
             having=having_multiple_active_publications)
-        sources = IStore(SourcePackagePublishingHistory).find(
-            SourcePackagePublishingHistory,
+
+        # We'll also access the SourcePackageReleases associated with
+        # the publications we find.  Since they're in the join anyway,
+        # load them alongside the publications.
+        # Actually we'll also want the SourcePackageNames, but adding
+        # those to the (outer) query would complicate it, and
+        # potentially slow it down.
+        query = IStore(SourcePackagePublishingHistory).find(
+            (SourcePackagePublishingHistory, SourcePackageRelease),
             join_spph_spr(),
             SourcePackageRelease.sourcepackagenameID.is_in(
                 candidate_source_names),
             spph_location_clauses)
+        return DecoratedResultSet(query, itemgetter(0))
+
+    def dominateSources(self, distroseries, pocket):
+        """Perform domination on source package publications.
+
+        Dominates sources, restricted to `distroseries`, `pocket`, and
+        `self.archive`.
+        """
+        self.logger.debug(
+            "Performing domination across %s/%s (Source)",
+            distroseries.name, pocket.title)
+
+        generalization = GeneralizedPublication(is_source=True)
+        sources = self.findSourcesForDomination(distroseries, pocket)
 
         self.logger.debug("Dominating sources...")
         self._dominatePublications(

=== modified file 'lib/lp/archivepublisher/tests/test_dominator.py'
--- lib/lp/archivepublisher/tests/test_dominator.py	2011-10-27 11:36:13 +0000
+++ lib/lp/archivepublisher/tests/test_dominator.py	2011-11-03 02:37:26 +0000
@@ -361,7 +361,10 @@
 def make_spphs_for_versions(factory, versions):
     """Create publication records for each of `versions`.
 
-    They records are created in the same order in which they are specified.
+    All these publications will be in the same source package, archive,
+    distroseries, and pocket.  They will all be in Published status.
+
+    The records are created in the same order in which they are specified.
     Make the order irregular to prove that version ordering is not a
     coincidence of object creation order etc.
 
@@ -371,6 +374,7 @@
     spn = factory.makeSourcePackageName()
     distroseries = factory.makeDistroSeries()
     pocket = factory.getAnyPocket()
+    archive = distroseries.main_archive
     sprs = [
         factory.makeSourcePackageRelease(
             sourcepackagename=spn, version=version)
@@ -378,11 +382,34 @@
     return [
         factory.makeSourcePackagePublishingHistory(
             distroseries=distroseries, pocket=pocket,
-            sourcepackagerelease=spr,
+            sourcepackagerelease=spr, archive=archive,
             status=PackagePublishingStatus.PUBLISHED)
         for spr in sprs]
 
 
+def make_bpphs_for_versions(factory, versions):
+    """Create publication records for each of `versions`.
+
+    All these publications will be in the same binary package, source
+    package, archive, distroarchseries, and pocket.  They will all be in
+    Published status.
+    """
+    bpn = factory.makeBinaryPackageName()
+    spn = factory.makeSourcePackageName()
+    das = factory.makeDistroArchSeries()
+    archive = das.distroseries.main_archive
+    pocket = factory.getAnyPocket()
+    bprs = [
+        factory.makeBinaryPackageRelease(binarypackagename=bpn)
+        for version in versions]
+    return [
+        factory.makeBinaryPackagePublishingHistory(
+            binarypackagerelease=bpr, binarypackagename=bpn,
+            distroarchseries=das, pocket=pocket, archive=archive,
+            sourcepackagename=spn, status=PackagePublishingStatus.PUBLISHED)
+        for bpr in bprs]
+
+
 def list_source_versions(spphs):
     """Extract the versions from `spphs` as a list, in the same order."""
     return [spph.sourcepackagerelease.version for spph in spphs]
@@ -921,3 +948,144 @@
             [],
             dominator.findPublishedSPPHs(
                 spph.distroseries, spph.pocket, other_package.name))
+
+    def test_findBinariesForDomination_finds_published_publications(self):
+        bpphs = make_bpphs_for_versions(self.factory, ['1.0', '1.1'])
+        dominator = self.makeDominator(bpphs)
+        self.assertContentEqual(
+            bpphs, dominator.findBinariesForDomination(
+                bpphs[0].distroarchseries, bpphs[0].pocket))
+
+    def test_findBinariesForDomination_skips_single_pub_packages(self):
+        # The domination algorithm that uses findBinariesForDomination
+        # always keeps the latest version live.  Thus, a single
+        # publication isn't worth dominating.  findBinariesForDomination
+        # won't return it.
+        bpphs = make_bpphs_for_versions(self.factory, ['1.0'])
+        dominator = self.makeDominator(bpphs)
+        self.assertContentEqual(
+            [], dominator.findBinariesForDomination(
+                bpphs[0].distroarchseries, bpphs[0].pocket))
+
+    def test_findBinariesForDomination_ignores_other_distroseries(self):
+        bpphs = make_bpphs_for_versions(self.factory, ['1.0', '1.1'])
+        dominator = self.makeDominator(bpphs)
+        das = bpphs[0].distroarchseries
+        other_series = self.factory.makeDistroSeries(
+            distribution=das.distroseries.distribution)
+        other_das = self.factory.makeDistroArchSeries(
+            distroseries=other_series, architecturetag=das.architecturetag,
+            processorfamily=das.processorfamily)
+        self.assertContentEqual(
+            [], dominator.findBinariesForDomination(
+                other_das, bpphs[0].pocket))
+
+    def test_findBinariesForDomination_ignores_other_architectures(self):
+        bpphs = make_bpphs_for_versions(self.factory, ['1.0', '1.1'])
+        dominator = self.makeDominator(bpphs)
+        other_das = self.factory.makeDistroArchSeries(
+            distroseries=bpphs[0].distroseries)
+        self.assertContentEqual(
+            [], dominator.findBinariesForDomination(
+                other_das, bpphs[0].pocket))
+
+    def test_findBinariesForDomination_ignores_other_archive(self):
+        bpphs = make_bpphs_for_versions(self.factory, ['1.0', '1.1'])
+        dominator = self.makeDominator(bpphs)
+        dominator.archive = self.factory.makeArchive()
+        self.assertContentEqual(
+            [], dominator.findBinariesForDomination(
+                bpphs[0].distroarchseries, bpphs[0].pocket))
+
+    def test_findBinariesForDomination_ignores_other_pocket(self):
+        bpphs = make_bpphs_for_versions(self.factory, ['1.0', '1.1'])
+        dominator = self.makeDominator(bpphs)
+        for bpph in bpphs:
+            removeSecurityProxy(bpph).pocket = PackagePublishingPocket.UPDATES
+        self.assertContentEqual(
+            [], dominator.findBinariesForDomination(
+                bpphs[0].distroarchseries, PackagePublishingPocket.SECURITY))
+
+    def test_findBinariesForDomination_ignores_other_status(self):
+        # If we have one BPPH for each possible status, plus one
+        # Published one to stop findBinariesForDomination from skipping
+        # the package, findBinariesForDomination returns only the
+        # Published ones.
+        versions = [
+            '1.%d' % self.factory.getUniqueInteger()
+            for status in PackagePublishingStatus.items] + ['0.9']
+        bpphs = make_bpphs_for_versions(self.factory, versions)
+        dominator = self.makeDominator(bpphs)
+
+        for bpph, status in zip(bpphs, PackagePublishingStatus.items):
+            bpph.status = status
+
+        # These are the Published publications.  The other ones will all
+        # be ignored.
+        published_bpphs = [
+            bpph
+            for bpph in bpphs
+                if bpph.status == PackagePublishingStatus.PUBLISHED]
+
+        self.assertContentEqual(
+            published_bpphs,
+            dominator.findBinariesForDomination(
+                bpphs[0].distroarchseries, bpphs[0].pocket))
+
+    def test_findSourcesForDomination_finds_published_publications(self):
+        spphs = make_spphs_for_versions(self.factory, ['2.0', '2.1'])
+        dominator = self.makeDominator(spphs)
+        self.assertContentEqual(
+            spphs, dominator.findSourcesForDomination(
+                spphs[0].distroseries, spphs[0].pocket))
+
+    def test_findSourcesForDomination_skips_single_pub_packages(self):
+        # The domination algorithm that uses findSourcesForDomination
+        # always keeps the latest version live.  Thus, a single
+        # publication isn't worth dominating.  findSourcesForDomination
+        # won't return it.
+        spphs = make_spphs_for_versions(self.factory, ['2.0'])
+        dominator = self.makeDominator(spphs)
+        self.assertContentEqual(
+            [], dominator.findSourcesForDomination(
+                spphs[0].distroseries, spphs[0].pocket))
+
+    def test_findSourcesForDomination_ignores_other_distroseries(self):
+        spphs = make_spphs_for_versions(self.factory, ['2.0', '2.1'])
+        dominator = self.makeDominator(spphs)
+        other_series = self.factory.makeDistroSeries(
+            distribution=spphs[0].distroseries.distribution)
+        self.assertContentEqual(
+            [], dominator.findSourcesForDomination(
+                other_series, spphs[0].pocket))
+
+    def test_findSourcesForDomination_ignores_other_pocket(self):
+        spphs = make_spphs_for_versions(self.factory, ['2.0', '2.1'])
+        dominator = self.makeDominator(spphs)
+        for spph in spphs:
+            removeSecurityProxy(spph).pocket = PackagePublishingPocket.UPDATES
+        self.assertContentEqual(
+            [], dominator.findSourcesForDomination(
+                spphs[0].distroseries, PackagePublishingPocket.SECURITY))
+
+    def test_findSourcesForDomination_ignores_other_status(self):
+        versions = [
+            '1.%d' % self.factory.getUniqueInteger()
+            for status in PackagePublishingStatus.items] + ['0.9']
+        spphs = make_spphs_for_versions(self.factory, versions)
+        dominator = self.makeDominator(spphs)
+
+        for spph, status in zip(spphs, PackagePublishingStatus.items):
+            spph.status = status
+
+        # These are the Published publications.  The other ones will all
+        # be ignored.
+        published_spphs = [
+            spph
+            for spph in spphs
+                if spph.status == PackagePublishingStatus.PUBLISHED]
+
+        self.assertContentEqual(
+            published_spphs,
+            dominator.findSourcesForDomination(
+                spphs[0].distroseries, spphs[0].pocket))

=== modified file 'lib/lp/bugs/javascript/buglisting.js'
--- lib/lp/bugs/javascript/buglisting.js	2011-11-01 17:55:21 +0000
+++ lib/lp/bugs/javascript/buglisting.js	2011-11-03 02:37:26 +0000
@@ -87,9 +87,6 @@
     var navigator = new namespace.ListingNavigator(
         window.location, LP.cache, LP.mustache_listings, target,
         navigation_indices);
-    if (target === null){
-        return;
-    }
     namespace.linkify_navigation();
     navigator.backwards_navigation = Y.all('.first,.previous');
     navigator.forwards_navigation = Y.all('.last,.next');
@@ -111,9 +108,6 @@
  * The template is always LP.mustache_listings.
  */
 namespace.ListingNavigator.prototype.render = function(){
-    if (! Y.Lang.isValue(this.target)){
-        return;
-    }
     var model = this.current_batch.mustache_model;
     var batch_info = Mustache.to_html(this.batch_info_template, {
         start: this.current_batch.start + 1,

=== modified file 'lib/lp/bugs/javascript/tests/test_buglisting.js'
--- lib/lp/bugs/javascript/tests/test_buglisting.js	2011-11-01 17:55:21 +0000
+++ lib/lp/bugs/javascript/tests/test_buglisting.js	2011-11-03 02:37:26 +0000
@@ -27,11 +27,6 @@
         Y.one('#fixture').setContent('');
     },
 
-    test_render_no_client_listing: function() {
-        // Rendering should not error with no #client-listing.
-        var navigator = new module.ListingNavigator();
-        navigator.render();
-    },
     get_render_navigator: function() {
         var target = Y.Node.create('<div id="client-listing"></div>');
         var lp_cache = {
@@ -230,8 +225,9 @@
         };
         var template = "<ol>" +
             "{{#item}}<li>{{name}}</li>{{/item}}</ol>";
-        var navigator = new module.ListingNavigator(window.location, lp_cache,
-                                                    template);
+        var target = Y.Node.create('<div id="client-listing"></div>');
+        var navigator = new module.ListingNavigator(
+            window.location, lp_cache, template, target);
         var key = module.ListingNavigator.get_batch_key({
             order_by: "intensity",
             memo: 'memo1',
@@ -243,10 +239,13 @@
             memo: 'memo1',
             forwards: true,
             start: 5,
-            mustache_model: {item: [
-                {name: 'first'},
-                {name: 'second'}
-            ]}};
+            mustache_model: {
+                item: [
+                    {name: 'first'},
+                    {name: 'second'}
+                ],
+                bugtasks: ['a', 'b', 'c']
+            }};
         navigator.update_from_model(batch);
         Y.lp.testing.assert.assert_equal_structure(
             batch, navigator.batches[key]);
@@ -422,13 +421,6 @@
     getPreviousLink: function(){
         return Y.one('.previous').get('href');
     },
-    test_from_page_no_client: function(){
-        Y.one('#fixture').setContent(
-            '<a class="previous" href="http://example.org/">PreVious</span>');
-        Y.Assert.areSame('http://example.org/', this.getPreviousLink());
-        module.ListingNavigator.from_page();
-        Y.Assert.areSame('http://example.org/', this.getPreviousLink());
-    },
     test_from_page_with_client: function(){
         Y.one('#fixture').setContent(
             '<a class="previous" href="http://example.org/">PreVious</span>' +

=== modified file 'lib/lp/bugs/templates/buglisting-default.pt'
--- lib/lp/bugs/templates/buglisting-default.pt	2011-10-28 19:17:29 +0000
+++ lib/lp/bugs/templates/buglisting-default.pt	2011-11-03 02:37:26 +0000
@@ -18,11 +18,17 @@
     </tal:comment>
     <script type="text/javascript"
       tal:condition="not: view/shouldShowAdvancedForm">
-    LPS.use('lp.registry.structural_subscription', 'lp.bugs.buglisting',
-            'lp.ordering', function(Y) {
+    LPS.use('lp.registry.structural_subscription', function(Y) {
         Y.on('domready', function() {
             Y.lp.registry.structural_subscription.setup(
                 {content_box: "#structural-subscription-content-box"});
+        });
+    });
+  </script>
+  <script type="text/javascript"
+    tal:condition="request/features/bugs.dynamic_bug_listings.enabled">
+    LPS.use('lp.bugs.buglisting', 'lp.ordering', function(Y) {
+        Y.on('domready', function() {
             var navigator = Y.lp.bugs.buglisting.ListingNavigator.from_page();
             var orderby = new Y.lp.ordering.OrderByBar({
                 srcNode: Y.one('#bugs-orderby'),

=== added file 'lib/lp/registry/scripts/teamparticipation.py'
--- lib/lp/registry/scripts/teamparticipation.py	1970-01-01 00:00:00 +0000
+++ lib/lp/registry/scripts/teamparticipation.py	2011-11-03 02:37:26 +0000
@@ -0,0 +1,160 @@
+# Copyright 2011 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Script code relating to team participations."""
+
+__metaclass__ = type
+__all__ = [
+    "check_teamparticipation",
+    ]
+
+from collections import (
+    defaultdict,
+    namedtuple,
+    )
+from itertools import (
+    chain,
+    imap,
+    )
+
+import transaction
+from zope.component import getUtility
+
+from canonical.database.sqlbase import quote
+from canonical.launchpad.webapp.interfaces import (
+    IStoreSelector,
+    MAIN_STORE,
+    SLAVE_FLAVOR,
+    )
+from lp.registry.interfaces.teammembership import ACTIVE_STATES
+from lp.services.scripts.base import LaunchpadScriptFailure
+
+
+def get_store():
+    """Return a slave store.
+
+    Errors in `TeamPartipation` can be detected using a replicated copy.
+    """
+    return getUtility(IStoreSelector).get(MAIN_STORE, SLAVE_FLAVOR)
+
+
+def check_teamparticipation_self(log):
+    """Check self-participation.
+
+    All people and teams should participate in themselves.
+    """
+    query = """
+        SELECT id, name
+          FROM Person
+         WHERE id NOT IN (
+            SELECT person FROM TeamParticipation
+             WHERE person = team)
+           AND merged IS NULL
+        """
+    non_self_participants = list(get_store().execute(query))
+    if len(non_self_participants) > 0:
+        log.warn(
+            "Some people/teams are not members of themselves: %s",
+            non_self_participants)
+
+
+def check_teamparticipation_circular(log):
+    """Check circular references.
+
+    There can be no mutual participation between teams.
+    """
+    query = """
+        SELECT tp.team, tp2.team
+          FROM TeamParticipation AS tp,
+               TeamParticipation AS tp2
+         WHERE tp.team = tp2.person
+           AND tp.person = tp2.team
+           AND tp.id != tp2.id;
+        """
+    circular_references = list(get_store().execute(query))
+    if len(circular_references) > 0:
+        raise LaunchpadScriptFailure(
+            "Circular references found: %s" % circular_references)
+
+
+ConsistencyError = namedtuple(
+    "ConsistencyError", ("type", "team", "people"))
+
+
+def check_teamparticipation_consistency(log):
+    """Check for missing or spurious participations.
+
+    For example, participations for people who are not members, or missing
+    participations for people who are members.
+    """
+    store = get_store()
+
+    # Slurp everything in.
+    people = dict(
+        store.execute(
+            "SELECT id, name FROM Person"
+            " WHERE teamowner IS NULL"
+            "   AND merged IS NULL"))
+    teams = dict(
+        store.execute(
+            "SELECT id, name FROM Person"
+            " WHERE teamowner IS NOT NULL"
+            "   AND merged IS NULL"))
+    team_memberships = defaultdict(set)
+    results = store.execute(
+        "SELECT team, person FROM TeamMembership"
+        " WHERE status in %s" % quote(ACTIVE_STATES))
+    for (team, person) in results:
+        team_memberships[team].add(person)
+    team_participations = defaultdict(set)
+    results = store.execute(
+        "SELECT team, person FROM TeamParticipation")
+    for (team, person) in results:
+        team_participations[team].add(person)
+
+    # Don't hold any locks.
+    transaction.commit()
+
+    # Check team memberships.
+    def get_participants(team):
+        """Recurse through membership records to get participants."""
+        member_people = team_memberships[team].intersection(people)
+        member_people.add(team)  # Teams always participate in themselves.
+        member_teams = team_memberships[team].intersection(teams)
+        return member_people.union(
+            chain.from_iterable(imap(get_participants, member_teams)))
+
+    errors = []
+    for team in teams:
+        participants_observed = team_participations[team]
+        participants_expected = get_participants(team)
+        participants_spurious = participants_expected - participants_observed
+        participants_missing = participants_observed - participants_expected
+        if len(participants_spurious) > 0:
+            error = ConsistencyError("spurious", team, participants_spurious)
+            errors.append(error)
+        if len(participants_missing) > 0:
+            error = ConsistencyError("missing", team, participants_missing)
+            errors.append(error)
+
+    # TODO:
+    # - Check that the only participant of a *person* is the person.
+    # - Check that merged people and teams do not appear in TeamParticipation.
+
+    def get_repr(id):
+        return "%s (%d)" % (people[id] if id in people else teams[id], id)
+
+    for error in errors:
+        people_repr = ", ".join(imap(get_repr, error.people))
+        log.warn(
+            "%s: %s TeamParticipation entries for %s.",
+            get_repr(error.team), error.type, people_repr)
+
+    return errors
+
+
+def check_teamparticipation(log):
+    """Perform various checks on the `TeamParticipation` table."""
+    check_teamparticipation_self(log)
+    check_teamparticipation_circular(log)
+    check_teamparticipation_consistency(log)

=== modified file 'lib/lp/registry/tests/test_teammembership.py'
--- lib/lp/registry/tests/test_teammembership.py	2011-09-01 06:18:57 +0000
+++ lib/lp/registry/tests/test_teammembership.py	2011-11-03 02:37:26 +0000
@@ -9,13 +9,11 @@
     )
 import re
 import subprocess
+from unittest import TestLoader
+
+import pytz
+from testtools.content import text_content
 from testtools.matchers import Equals
-from unittest import (
-    TestCase,
-    TestLoader,
-    )
-
-import pytz
 import transaction
 from zope.component import getUtility
 from zope.security.proxy import removeSecurityProxy
@@ -27,10 +25,6 @@
     flush_database_updates,
     sqlvalues,
     )
-from canonical.launchpad.ftests import (
-    login,
-    login_person,
-    )
 from canonical.launchpad.interfaces.lpstorm import IStore
 from canonical.launchpad.testing.systemdocs import (
     default_optionflags,
@@ -53,16 +47,22 @@
     ITeamMembershipSet,
     TeamMembershipStatus,
     )
-from lp.registry.model.teammembership import (\
+from lp.registry.model.teammembership import (
     find_team_participations,
     TeamMembership,
     TeamParticipation,
     )
+from lp.registry.scripts.teamparticipation import check_teamparticipation
+from lp.services.log.logger import BufferLogger
 from lp.testing import (
+    login,
     login_celebrity,
+    login_person,
     person_logged_in,
+    StormStatementRecorder,
+    TestCase,
     TestCaseWithFactory,
-    StormStatementRecorder)
+    )
 from lp.testing.mail_helpers import pop_notifications
 from lp.testing.matchers import HasQueryCount
 from lp.testing.storm import reload_object
@@ -1047,6 +1047,7 @@
 
 
 class TestCheckTeamParticipationScript(TestCase):
+
     layer = DatabaseFunctionalLayer
 
     def _runScript(self, expected_returncode=0):
@@ -1054,14 +1055,19 @@
             'cronscripts/check-teamparticipation.py', shell=True,
             stdin=subprocess.PIPE, stdout=subprocess.PIPE,
             stderr=subprocess.PIPE)
-        (out, err) = process.communicate()
-        self.assertEqual(process.returncode, expected_returncode, (out, err))
+        out, err = process.communicate()
+        if out != "":
+            self.addDetail("stdout", text_content(out))
+        if err != "":
+            self.addDetail("stderr", text_content(err))
+        self.assertEqual(process.returncode, expected_returncode)
         return out, err
 
     def test_no_output_if_no_invalid_entries(self):
         """No output if there's no invalid teamparticipation entries."""
         out, err = self._runScript()
-        self.assertEqual((out, err), ('', ''))
+        self.assertEqual(0, len(out))
+        self.assertEqual(0, len(err))
 
     def test_report_invalid_teamparticipation_entries(self):
         """The script reports missing/spurious TeamParticipation entries.
@@ -1103,16 +1109,13 @@
         transaction.commit()
 
         out, err = self._runScript()
-        self.assertEqual(out, '', (out, err))
-        self.failUnless(
-            re.search('missing TeamParticipation entries for zzzzz', err),
-            (out, err))
-        self.failUnless(
-            re.search('spurious TeamParticipation entries for zzzzz', err),
-            (out, err))
-        self.failUnless(
-            re.search('not members of themselves:.*zzzzz.*', err),
-            (out, err))
+        self.assertEqual(0, len(out))
+        self.failUnless(
+            re.search('missing TeamParticipation entries for zzzzz', err))
+        self.failUnless(
+            re.search('spurious TeamParticipation entries for zzzzz', err))
+        self.failUnless(
+            re.search('not members of themselves:.*zzzzz.*', err))
 
     def test_report_circular_team_references(self):
         """The script reports circular references between teams.
@@ -1145,9 +1148,34 @@
         import transaction
         transaction.commit()
         out, err = self._runScript(expected_returncode=1)
-        self.assertEqual(out, '', (out, err))
-        self.failUnless(
-            re.search('Circular references found', err), (out, err))
+        self.assertEqual(0, len(out))
+        self.failUnless(re.search('Circular references found', err))
+
+
+class TestCheckTeamParticipationScriptPerformance(TestCaseWithFactory):
+
+    layer = DatabaseFunctionalLayer
+
+    def test_queries(self):
+        """The script does not overly tax the database.
+
+        The whole check_teamparticipation() run executes a constant low number
+        of queries.
+        """
+        # Create a deeply nested team and member structure.
+        team = self.factory.makeTeam()
+        for num in xrange(10):
+            another_team = self.factory.makeTeam()
+            another_person = self.factory.makePerson()
+            with person_logged_in(team.teamowner):
+                team.addMember(another_team, team.teamowner)
+                team.addMember(another_person, team.teamowner)
+            team = another_team
+        transaction.commit()
+        with StormStatementRecorder() as recorder:
+            logger = BufferLogger()
+            check_teamparticipation(logger)
+        self.assertThat(recorder, HasQueryCount(Equals(6)))
 
 
 def test_suite():

=== modified file 'lib/lp/soyuz/model/publishing.py'
--- lib/lp/soyuz/model/publishing.py	2011-10-23 02:58:56 +0000
+++ lib/lp/soyuz/model/publishing.py	2011-11-03 02:37:26 +0000
@@ -33,7 +33,6 @@
     Desc,
     LeftJoin,
     Or,
-    Select,
     Sum,
     )
 from storm.store import Store
@@ -1154,19 +1153,10 @@
         # Avoid circular wotsits.
         from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild
         from lp.soyuz.model.distroarchseries import DistroArchSeries
-        from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease
-        source_select = Select(
-            SourcePackageRelease.id,
-            And(
-                BinaryPackageBuild.source_package_release_id ==
-                    SourcePackageRelease.id,
-                BinaryPackageRelease.build == BinaryPackageBuild.id,
-                self.binarypackagereleaseID == BinaryPackageRelease.id,
-            ))
+
         pubs = [
             BinaryPackageBuild.source_package_release_id ==
-                SourcePackageRelease.id,
-            SourcePackageRelease.id.is_in(source_select),
+                self.binarypackagerelease.build.source_package_release_id,
             BinaryPackageRelease.build == BinaryPackageBuild.id,
             BinaryPackagePublishingHistory.binarypackagereleaseID ==
                 BinaryPackageRelease.id,