Merge ~cjwatson/launchpad:simplify-publisher-suite-sets into launchpad:master

Proposed by Colin Watson
Status: Merged
Approved by: Colin Watson
Approved revision: 0ff12af94d54b644d73831ef66c6ef67d81a5fb9
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~cjwatson/launchpad:simplify-publisher-suite-sets
Merge into: launchpad:master
Diff against target: 670 lines (+97/-131)
6 files modified
lib/lp/archivepublisher/model/ftparchive.py (+3/-4)
lib/lp/archivepublisher/publishing.py (+37/-37)
lib/lp/archivepublisher/scripts/publishdistro.py (+2/-2)
lib/lp/archivepublisher/tests/test_ftparchive.py (+1/-1)
lib/lp/archivepublisher/tests/test_publishdistro.py (+8/-12)
lib/lp/archivepublisher/tests/test_publisher.py (+46/-75)
Reviewer Review Type Date Requested Status
Ioana Lasc (community) Approve
Review via email: mp+410439@code.launchpad.net

Commit message

Simplify sets-of-suites handling in the publisher

Description of the change

Rather than storing sets of tuples of (distroseries name, pocket), it's equivalent and simpler to just store sets of suite names, which are composed from the distroseries name and the pocket. This means that we don't have to worry about details such as whether security-proxied enumeration items behave identically to unproxied items for purposes of set membership checks (in some situations they don't). The code is also slightly shorter and easier to follow this way.

I have no proof that this changes behaviour, although it's possible that it will fix some weirdness that I encountered while trying to force the jammy release pocket to republish to work around a separate bug on 2021-10-16; while debugging that, I noticed some oddities with set membership checks that I never quite got to the bottom of.

To post a comment you must log in.
Revision history for this message
Ioana Lasc (ilasc) wrote :

LGTM

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/lib/lp/archivepublisher/model/ftparchive.py b/lib/lp/archivepublisher/model/ftparchive.py
index d56b881..a5b6e32 100644
--- a/lib/lp/archivepublisher/model/ftparchive.py
+++ b/lib/lp/archivepublisher/model/ftparchive.py
@@ -232,7 +232,7 @@ class FTPArchiveHandler:
232 continue232 continue
233233
234 self.publisher.release_files_needed.add(234 self.publisher.release_files_needed.add(
235 (distroseries.name, pocket))235 distroseries.getSuite(pocket))
236236
237 for comp in components:237 for comp in components:
238 self.createEmptyPocketRequest(distroseries, pocket, comp)238 self.createEmptyPocketRequest(distroseries, pocket, comp)
@@ -768,9 +768,8 @@ class FTPArchiveHandler:
768768
769 Otherwise, we aim to limit our config to certain distroseries769 Otherwise, we aim to limit our config to certain distroseries
770 and pockets. By default, we will exclude release pockets for770 and pockets. By default, we will exclude release pockets for
771 released series, and in addition we exclude any pocket not771 released series, and in addition we exclude any suite not
772 explicitly marked as dirty. dirty_pockets must be a nested772 explicitly marked as dirty.
773 dictionary of booleans, keyed by distroseries.name then pocket.
774 """773 """
775 apt_config = six.StringIO()774 apt_config = six.StringIO()
776 apt_config.write(CONFIG_HEADER % (self._config.archiveroot,775 apt_config.write(CONFIG_HEADER % (self._config.archiveroot,
diff --git a/lib/lp/archivepublisher/publishing.py b/lib/lp/archivepublisher/publishing.py
index 708e9de..6340738 100644
--- a/lib/lp/archivepublisher/publishing.py
+++ b/lib/lp/archivepublisher/publishing.py
@@ -1,4 +1,4 @@
1# Copyright 2009-2019 Canonical Ltd. This software is licensed under the1# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4__all__ = [4__all__ = [
@@ -416,15 +416,16 @@ class Publisher(object):
416416
417 Publishers need the pool root dir and a DiskPool object.417 Publishers need the pool root dir and a DiskPool object.
418418
419 Optionally we can pass a list of tuples, (distroseries.name, pocket),419 Optionally we can pass a list of suite names which will restrict the
420 which will restrict the publisher actions, only suites listed in420 publisher actions; only suites listed in allowed_suites will be
421 allowed_suites will be modified.421 modified.
422 """422 """
423 self.log = log423 self.log = log
424 self._config = config424 self._config = config
425 self.distro = archive.distribution425 self.distro = archive.distribution
426 self.archive = archive426 self.archive = archive
427 self.allowed_suites = allowed_suites427 self.allowed_suites = (
428 None if allowed_suites is None else set(allowed_suites))
428429
429 self._diskpool = diskpool430 self._diskpool = diskpool
430431
@@ -433,14 +434,14 @@ class Publisher(object):
433 else:434 else:
434 self._library = library435 self._library = library
435436
436 # Track which distroseries pockets have been dirtied by a437 # Track which suites have been dirtied by a change, and therefore
437 # change, and therefore need domination/apt-ftparchive work.438 # need domination/apt-ftparchive work. This is a set of suite names
438 # This is a set of tuples in the form (distroseries.name, pocket)439 # as returned by DistroSeries.getSuite.
439 self.dirty_pockets = set()440 self.dirty_suites = set()
440441
441 # Track which pockets need release files. This will contain more442 # Track which suites need release files. This will contain more
442 # than dirty_pockets in the case of a careful index run.443 # than dirty_suites in the case of a careful index run.
443 # This is a set of tuples in the form (distroseries.name, pocket)444 # This is a set of suite names as returned by DistroSeries.getSuite.
444 self.release_files_needed = set()445 self.release_files_needed = set()
445446
446 def setupArchiveDirs(self):447 def setupArchiveDirs(self):
@@ -449,12 +450,12 @@ class Publisher(object):
449450
450 def isDirty(self, distroseries, pocket):451 def isDirty(self, distroseries, pocket):
451 """True if a publication has happened in this release and pocket."""452 """True if a publication has happened in this release and pocket."""
452 return (distroseries.name, pocket) in self.dirty_pockets453 return distroseries.getSuite(pocket) in self.dirty_suites
453454
454 def markPocketDirty(self, distroseries, pocket):455 def markSuiteDirty(self, distroseries, pocket):
455 """Mark a pocket dirty only if it's allowed."""456 """Mark a suite dirty only if it's allowed."""
456 if self.isAllowed(distroseries, pocket):457 if self.isAllowed(distroseries, pocket):
457 self.dirty_pockets.add((distroseries.name, pocket))458 self.dirty_suites.add(distroseries.getSuite(pocket))
458459
459 def isAllowed(self, distroseries, pocket):460 def isAllowed(self, distroseries, pocket):
460 """Whether or not the given suite should be considered.461 """Whether or not the given suite should be considered.
@@ -465,7 +466,7 @@ class Publisher(object):
465 Otherwise, return False.466 Otherwise, return False.
466 """467 """
467 return (not self.allowed_suites or468 return (not self.allowed_suites or
468 (distroseries.name, pocket) in self.allowed_suites)469 distroseries.getSuite(pocket) in self.allowed_suites)
469470
470 @property471 @property
471 def subcomponents(self):472 def subcomponents(self):
@@ -546,7 +547,7 @@ class Publisher(object):
546547
547 Consider records returned by getPendingSourcePublications.548 Consider records returned by getPendingSourcePublications.
548 """549 """
549 dirty_pockets = set()550 dirty_suites = set()
550 all_spphs = self.getPendingSourcePublications(is_careful)551 all_spphs = self.getPendingSourcePublications(is_careful)
551 for (distroseries, pocket), spphs in groupby(552 for (distroseries, pocket), spphs in groupby(
552 all_spphs, attrgetter("distroseries", "pocket")):553 all_spphs, attrgetter("distroseries", "pocket")):
@@ -561,8 +562,8 @@ class Publisher(object):
561 distroseries.status.name))562 distroseries.status.name))
562 else:563 else:
563 self.publishSources(distroseries, pocket, spphs)564 self.publishSources(distroseries, pocket, spphs)
564 dirty_pockets.add((distroseries.name, pocket))565 dirty_suites.add(distroseries.getSuite(pocket))
565 return dirty_pockets566 return dirty_suites
566567
567 def getPendingBinaryPublications(self, is_careful):568 def getPendingBinaryPublications(self, is_careful):
568 """Return the specific group of binary records to be published."""569 """Return the specific group of binary records to be published."""
@@ -602,7 +603,7 @@ class Publisher(object):
602603
603 Consider records returned by getPendingBinaryPublications.604 Consider records returned by getPendingBinaryPublications.
604 """605 """
605 dirty_pockets = set()606 dirty_suites = set()
606 all_bpphs = self.getPendingBinaryPublications(is_careful)607 all_bpphs = self.getPendingBinaryPublications(is_careful)
607 for (distroarchseries, pocket), bpphs in groupby(608 for (distroarchseries, pocket), bpphs in groupby(
608 all_bpphs, attrgetter("distroarchseries", "pocket")):609 all_bpphs, attrgetter("distroarchseries", "pocket")):
@@ -618,8 +619,8 @@ class Publisher(object):
618 distroseries.status.name))619 distroseries.status.name))
619 else:620 else:
620 self.publishBinaries(distroarchseries, pocket, bpphs)621 self.publishBinaries(distroarchseries, pocket, bpphs)
621 dirty_pockets.add((distroseries.name, pocket))622 dirty_suites.add(distroseries.getSuite(pocket))
622 return dirty_pockets623 return dirty_suites
623624
624 def A_publish(self, force_publishing):625 def A_publish(self, force_publishing):
625 """First step in publishing: actual package publishing.626 """First step in publishing: actual package publishing.
@@ -631,9 +632,9 @@ class Publisher(object):
631 """632 """
632 self.log.debug("* Step A: Publishing packages")633 self.log.debug("* Step A: Publishing packages")
633634
634 self.dirty_pockets.update(635 self.dirty_suites.update(
635 self.findAndPublishSources(is_careful=force_publishing))636 self.findAndPublishSources(is_careful=force_publishing))
636 self.dirty_pockets.update(637 self.dirty_suites.update(
637 self.findAndPublishBinaries(is_careful=force_publishing))638 self.findAndPublishBinaries(is_careful=force_publishing))
638639
639 def A2_markPocketsWithDeletionsDirty(self):640 def A2_markPocketsWithDeletionsDirty(self):
@@ -654,9 +655,9 @@ class Publisher(object):
654 table.dateremoved == None,655 table.dateremoved == None,
655 ]656 ]
656657
657 # We need to get a set of (distroseries, pocket) tuples that have658 # We need to get a set of suite names that have publications that
658 # publications that are waiting to be deleted. Each tuple is659 # are waiting to be deleted. Each suite name is added to the
659 # added to the dirty_pockets set.660 # dirty_suites set.
660661
661 # Make the source publications query.662 # Make the source publications query.
662 conditions = base_conditions(SourcePackagePublishingHistory)663 conditions = base_conditions(SourcePackagePublishingHistory)
@@ -688,7 +689,7 @@ class Publisher(object):
688 # stable distroseries, no matter what other bugs689 # stable distroseries, no matter what other bugs
689 # that precede here have dirtied it.690 # that precede here have dirtied it.
690 continue691 continue
691 self.markPocketDirty(distroseries, pocket)692 self.markSuiteDirty(distroseries, pocket)
692693
693 def B_dominate(self, force_domination):694 def B_dominate(self, force_domination):
694 """Second step in publishing: domination."""695 """Second step in publishing: domination."""
@@ -729,7 +730,7 @@ class Publisher(object):
729 continue730 continue
730 self.checkDirtySuiteBeforePublishing(distroseries, pocket)731 self.checkDirtySuiteBeforePublishing(distroseries, pocket)
731732
732 self.release_files_needed.add((distroseries.name, pocket))733 self.release_files_needed.add(distroseries.getSuite(pocket))
733734
734 components = self.archive.getComponentsForSeries(distroseries)735 components = self.archive.getComponentsForSeries(distroseries)
735 for component in components:736 for component in components:
@@ -739,9 +740,9 @@ class Publisher(object):
739 def D_writeReleaseFiles(self, is_careful):740 def D_writeReleaseFiles(self, is_careful):
740 """Write out the Release files for the provided distribution.741 """Write out the Release files for the provided distribution.
741742
742 If is_careful is specified, we include all pockets of all releases.743 If is_careful is specified, we include all suites.
743744
744 Otherwise we include only pockets flagged as true in dirty_pockets.745 Otherwise we include only suites flagged as true in dirty_suites.
745 """746 """
746 self.log.debug("* Step D: Generating Release files.")747 self.log.debug("* Step D: Generating Release files.")
747748
@@ -750,11 +751,10 @@ class Publisher(object):
750 self.archive, container_prefix=u"release:"):751 self.archive, container_prefix=u"release:"):
751 distroseries, pocket = self.distro.getDistroSeriesAndPocket(752 distroseries, pocket = self.distro.getDistroSeriesAndPocket(
752 container[len(u"release:"):])753 container[len(u"release:"):])
753 archive_file_suites.add((distroseries.name, pocket))754 archive_file_suites.add(distroseries.getSuite(pocket))
754755
755 for distroseries in self.distro:756 for distroseries in self.distro:
756 for pocket in self.archive.getPockets():757 for pocket in self.archive.getPockets():
757 ds_pocket = (distroseries.name, pocket)
758 suite = distroseries.getSuite(pocket)758 suite = distroseries.getSuite(pocket)
759 suite_path = os.path.join(self._config.distsroot, suite)759 suite_path = os.path.join(self._config.distsroot, suite)
760 release_path = os.path.join(suite_path, "Release")760 release_path = os.path.join(suite_path, "Release")
@@ -768,9 +768,9 @@ class Publisher(object):
768 # suites. Only force those suites that already have768 # suites. Only force those suites that already have
769 # Release files.769 # Release files.
770 if file_exists(release_path):770 if file_exists(release_path):
771 self.release_files_needed.add(ds_pocket)771 self.release_files_needed.add(suite)
772772
773 write_release = ds_pocket in self.release_files_needed773 write_release = suite in self.release_files_needed
774 if not is_careful:774 if not is_careful:
775 if not self.isDirty(distroseries, pocket):775 if not self.isDirty(distroseries, pocket):
776 self.log.debug("Skipping release files for %s/%s" %776 self.log.debug("Skipping release files for %s/%s" %
@@ -782,7 +782,7 @@ class Publisher(object):
782782
783 if write_release:783 if write_release:
784 self._writeSuite(distroseries, pocket)784 self._writeSuite(distroseries, pocket)
785 elif (ds_pocket in archive_file_suites and785 elif (suite in archive_file_suites and
786 distroseries.publish_by_hash):786 distroseries.publish_by_hash):
787 # We aren't publishing a new Release file for this787 # We aren't publishing a new Release file for this
788 # suite, probably because it's immutable, but we still788 # suite, probably because it's immutable, but we still
diff --git a/lib/lp/archivepublisher/scripts/publishdistro.py b/lib/lp/archivepublisher/scripts/publishdistro.py
index c079484..b82f992 100644
--- a/lib/lp/archivepublisher/scripts/publishdistro.py
+++ b/lib/lp/archivepublisher/scripts/publishdistro.py
@@ -233,7 +233,7 @@ class PublishDistro(PublisherScript):
233 suites = set()233 suites = set()
234 for suite in self.options.suite:234 for suite in self.options.suite:
235 series, pocket = self.findSuite(distribution, suite)235 series, pocket = self.findSuite(distribution, suite)
236 suites.add((series.name, pocket))236 suites.add(series.getSuite(pocket))
237 return suites237 return suites
238238
239 def findExplicitlyDirtySuites(self, archive):239 def findExplicitlyDirtySuites(self, archive):
@@ -311,7 +311,7 @@ class PublishDistro(PublisherScript):
311 """311 """
312 for distroseries, pocket in self.findExplicitlyDirtySuites(archive):312 for distroseries, pocket in self.findExplicitlyDirtySuites(archive):
313 if not cannot_modify_suite(archive, distroseries, pocket):313 if not cannot_modify_suite(archive, distroseries, pocket):
314 publisher.markPocketDirty(distroseries, pocket)314 publisher.markSuiteDirty(distroseries, pocket)
315 if archive.dirty_suites is not None:315 if archive.dirty_suites is not None:
316 # Clear the explicit dirt indicator before we start doing316 # Clear the explicit dirt indicator before we start doing
317 # time-consuming publishing, which might race with an317 # time-consuming publishing, which might race with an
diff --git a/lib/lp/archivepublisher/tests/test_ftparchive.py b/lib/lp/archivepublisher/tests/test_ftparchive.py
index 139392b..1c09bfb 100755
--- a/lib/lp/archivepublisher/tests/test_ftparchive.py
+++ b/lib/lp/archivepublisher/tests/test_ftparchive.py
@@ -582,7 +582,7 @@ class TestFTPArchive(TestCaseWithFactory):
582 # * a-f runs based on this config without any errors582 # * a-f runs based on this config without any errors
583 # * a-f *only* creates the wanted archive indexes.583 # * a-f *only* creates the wanted archive indexes.
584 allowed_suites = set()584 allowed_suites = set()
585 allowed_suites.add(('hoary-test', PackagePublishingPocket.UPDATES))585 allowed_suites.add('hoary-test-updates')
586586
587 publisher = Publisher(587 publisher = Publisher(
588 self._logger, self._config, self._dp,588 self._logger, self._config, self._dp,
diff --git a/lib/lp/archivepublisher/tests/test_publishdistro.py b/lib/lp/archivepublisher/tests/test_publishdistro.py
index da8f548..fb00a9f 100644
--- a/lib/lp/archivepublisher/tests/test_publishdistro.py
+++ b/lib/lp/archivepublisher/tests/test_publishdistro.py
@@ -117,11 +117,11 @@ class TestPublishDistro(TestNativePublishingBase):
117 foo_path = "%s/main/f/foo/foo_666.dsc" % self.pool_dir117 foo_path = "%s/main/f/foo/foo_666.dsc" % self.pool_dir
118 self.assertEqual(open(foo_path).read().strip(), 'foo')118 self.assertEqual(open(foo_path).read().strip(), 'foo')
119119
120 def testDirtyPocketProcessing(self):120 def testDirtySuiteProcessing(self):
121 """Test dirty pocket processing.121 """Test dirty suite processing.
122122
123 Make a DELETED source to see if the dirty pocket processing123 Make a DELETED source to see if the dirty suite processing works for
124 works for deletions.124 deletions.
125 """125 """
126 pub_source_id = self.getPubSource(filecontent=b'foo').id126 pub_source_id = self.getPubSource(filecontent=b'foo').id
127 self.layer.txn.commit()127 self.layer.txn.commit()
@@ -717,14 +717,13 @@ class TestPublishDistroMethods(TestCaseWithFactory):
717 script = self.makeScript(distro)717 script = self.makeScript(distro)
718 self.assertContentEqual([], script.findAllowedSuites(distro))718 self.assertContentEqual([], script.findAllowedSuites(distro))
719719
720 def test_findAllowedSuites_finds_series_and_pocket(self):720 def test_findAllowedSuites_finds_single(self):
721 # findAllowedSuites looks up the requested suites.721 # findAllowedSuites looks up the requested suite.
722 series = self.factory.makeDistroSeries()722 series = self.factory.makeDistroSeries()
723 suite = "%s-updates" % series.name723 suite = "%s-updates" % series.name
724 script = self.makeScript(series.distribution, ['--suite', suite])724 script = self.makeScript(series.distribution, ['--suite', suite])
725 self.assertContentEqual(725 self.assertContentEqual(
726 [(series.name, PackagePublishingPocket.UPDATES)],726 [suite], script.findAllowedSuites(series.distribution))
727 script.findAllowedSuites(series.distribution))
728727
729 def test_findAllowedSuites_finds_multiple(self):728 def test_findAllowedSuites_finds_multiple(self):
730 # Multiple suites may be requested; findAllowedSuites looks them729 # Multiple suites may be requested; findAllowedSuites looks them
@@ -733,10 +732,7 @@ class TestPublishDistroMethods(TestCaseWithFactory):
733 script = self.makeScript(series.distribution, [732 script = self.makeScript(series.distribution, [
734 '--suite', '%s-updates' % series.name,733 '--suite', '%s-updates' % series.name,
735 '--suite', series.name])734 '--suite', series.name])
736 expected_suites = [735 expected_suites = ['%s-updates' % series.name, series.name]
737 (series.name, PackagePublishingPocket.UPDATES),
738 (series.name, PackagePublishingPocket.RELEASE),
739 ]
740 self.assertContentEqual(736 self.assertContentEqual(
741 expected_suites, script.findAllowedSuites(series.distribution))737 expected_suites, script.findAllowedSuites(series.distribution))
742738
diff --git a/lib/lp/archivepublisher/tests/test_publisher.py b/lib/lp/archivepublisher/tests/test_publisher.py
index 247fef3..977b626 100644
--- a/lib/lp/archivepublisher/tests/test_publisher.py
+++ b/lib/lp/archivepublisher/tests/test_publisher.py
@@ -1,4 +1,4 @@
1# Copyright 2009-2020 Canonical Ltd. This software is licensed under the1# Copyright 2009-2021 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4"""Tests for publisher class."""4"""Tests for publisher class."""
@@ -720,11 +720,6 @@ class TestByHashes(TestCaseWithFactory):
720class TestPublisher(TestPublisherBase):720class TestPublisher(TestPublisherBase):
721 """Testing `Publisher` behaviour."""721 """Testing `Publisher` behaviour."""
722722
723 def assertDirtyPocketsContents(self, expected, dirty_pockets):
724 contents = [(str(dr_name), pocket.name) for dr_name, pocket in
725 dirty_pockets]
726 self.assertEqual(expected, contents)
727
728 def assertReleaseContentsMatch(self, release, filename, contents):723 def assertReleaseContentsMatch(self, release, filename, contents):
729 for hash_name, hash_func in (724 for hash_name, hash_func in (
730 ('md5sum', hashlib.md5),725 ('md5sum', hashlib.md5),
@@ -766,8 +761,7 @@ class TestPublisher(TestPublisherBase):
766 self.layer.txn.commit()761 self.layer.txn.commit()
767762
768 pub_source.sync()763 pub_source.sync()
769 self.assertDirtyPocketsContents(764 self.assertEqual({'breezy-autotest'}, publisher.dirty_suites)
770 [('breezy-autotest', 'RELEASE')], publisher.dirty_pockets)
771 self.assertEqual(PackagePublishingStatus.PUBLISHED, pub_source.status)765 self.assertEqual(PackagePublishingStatus.PUBLISHED, pub_source.status)
772766
773 # file got published767 # file got published
@@ -972,8 +966,7 @@ class TestPublisher(TestPublisherBase):
972 publisher.A_publish(force_publishing=False)966 publisher.A_publish(force_publishing=False)
973967
974 # The pocket was dirtied:968 # The pocket was dirtied:
975 self.assertDirtyPocketsContents(969 self.assertEqual({'breezy-autotest'}, publisher.dirty_suites)
976 [('breezy-autotest', 'RELEASE')], publisher.dirty_pockets)
977 # The file was published:970 # The file was published:
978 foo_path = "%s/main/f/foo/foo_666.dsc" % pub_config.poolroot971 foo_path = "%s/main/f/foo/foo_666.dsc" % pub_config.poolroot
979 with open(foo_path) as foo_file:972 with open(foo_path) as foo_file:
@@ -993,8 +986,7 @@ class TestPublisher(TestPublisherBase):
993 """986 """
994 publisher = Publisher(987 publisher = Publisher(
995 self.logger, self.config, self.disk_pool,988 self.logger, self.config, self.disk_pool,
996 self.ubuntutest.main_archive,989 self.ubuntutest.main_archive, allowed_suites=['hoary-test'])
997 allowed_suites=[('hoary-test', PackagePublishingPocket.RELEASE)])
998990
999 pub_source = self.getPubSource(filecontent=b'foo')991 pub_source = self.getPubSource(filecontent=b'foo')
1000 pub_source2 = self.getPubSource(992 pub_source2 = self.getPubSource(
@@ -1006,8 +998,7 @@ class TestPublisher(TestPublisherBase):
1006998
1007 pub_source.sync()999 pub_source.sync()
1008 pub_source2.sync()1000 pub_source2.sync()
1009 self.assertDirtyPocketsContents(1001 self.assertEqual({'hoary-test'}, publisher.dirty_suites)
1010 [('hoary-test', 'RELEASE')], publisher.dirty_pockets)
1011 self.assertEqual(1002 self.assertEqual(
1012 PackagePublishingStatus.PUBLISHED, pub_source2.status)1003 PackagePublishingStatus.PUBLISHED, pub_source2.status)
1013 self.assertEqual(PackagePublishingStatus.PENDING, pub_source.status)1004 self.assertEqual(PackagePublishingStatus.PENDING, pub_source.status)
@@ -1020,8 +1011,7 @@ class TestPublisher(TestPublisherBase):
1020 publisher = Publisher(1011 publisher = Publisher(
1021 self.logger, self.config, self.disk_pool,1012 self.logger, self.config, self.disk_pool,
1022 self.ubuntutest.main_archive,1013 self.ubuntutest.main_archive,
1023 allowed_suites=[('breezy-autotest',1014 allowed_suites=['breezy-autotest-updates'])
1024 PackagePublishingPocket.UPDATES)])
10251015
1026 self.ubuntutest['breezy-autotest'].status = (1016 self.ubuntutest['breezy-autotest'].status = (
1027 SeriesStatus.CURRENT)1017 SeriesStatus.CURRENT)
@@ -1039,8 +1029,7 @@ class TestPublisher(TestPublisherBase):
10391029
1040 pub_source.sync()1030 pub_source.sync()
1041 pub_source2.sync()1031 pub_source2.sync()
1042 self.assertDirtyPocketsContents(1032 self.assertEqual({'breezy-autotest-updates'}, publisher.dirty_suites)
1043 [('breezy-autotest', 'UPDATES')], publisher.dirty_pockets)
1044 self.assertEqual(PackagePublishingStatus.PUBLISHED, pub_source.status)1033 self.assertEqual(PackagePublishingStatus.PUBLISHED, pub_source.status)
1045 self.assertEqual(PackagePublishingStatus.PENDING, pub_source2.status)1034 self.assertEqual(PackagePublishingStatus.PENDING, pub_source2.status)
10461035
@@ -1059,7 +1048,7 @@ class TestPublisher(TestPublisherBase):
1059 # no pockets will be *dirtied*.1048 # no pockets will be *dirtied*.
1060 publisher.A_publish(False)1049 publisher.A_publish(False)
10611050
1062 self.assertDirtyPocketsContents([], publisher.dirty_pockets)1051 self.assertEqual(set(), publisher.dirty_suites)
1063 # nothing got published1052 # nothing got published
1064 foo_path = "%s/main/f/foo/foo_666.dsc" % self.pool_dir1053 foo_path = "%s/main/f/foo/foo_666.dsc" % self.pool_dir
1065 self.assertEqual(False, os.path.exists(foo_path))1054 self.assertEqual(False, os.path.exists(foo_path))
@@ -1087,8 +1076,7 @@ class TestPublisher(TestPublisherBase):
1087 # then we will have a corresponding dirty_pocket entry.1076 # then we will have a corresponding dirty_pocket entry.
1088 publisher.A_publish(True)1077 publisher.A_publish(True)
10891078
1090 self.assertDirtyPocketsContents(1079 self.assertEqual({'breezy-autotest'}, publisher.dirty_suites)
1091 [('breezy-autotest', 'RELEASE')], publisher.dirty_pockets)
1092 # file got published1080 # file got published
1093 foo_path = "%s/main/f/foo/foo_666.dsc" % self.pool_dir1081 foo_path = "%s/main/f/foo/foo_666.dsc" % self.pool_dir
1094 with open(foo_path) as foo_file:1082 with open(foo_path) as foo_file:
@@ -1115,7 +1103,7 @@ class TestPublisher(TestPublisherBase):
1115 publisher.A_publish(False)1103 publisher.A_publish(False)
1116 self.layer.txn.commit()1104 self.layer.txn.commit()
11171105
1118 self.assertDirtyPocketsContents([], publisher.dirty_pockets)1106 self.assertEqual(set(), publisher.dirty_suites)
1119 self.assertEqual(PackagePublishingStatus.PENDING, pub_source.status)1107 self.assertEqual(PackagePublishingStatus.PENDING, pub_source.status)
11201108
1121 # nothing got published1109 # nothing got published
@@ -1146,8 +1134,7 @@ class TestPublisher(TestPublisherBase):
1146 self.layer.txn.commit()1134 self.layer.txn.commit()
11471135
1148 pub_source.sync()1136 pub_source.sync()
1149 self.assertDirtyPocketsContents(1137 self.assertEqual({'breezy-autotest'}, publisher.dirty_suites)
1150 [('breezy-autotest', 'RELEASE')], publisher.dirty_pockets)
1151 self.assertEqual(PackagePublishingStatus.PUBLISHED, pub_source.status)1138 self.assertEqual(PackagePublishingStatus.PUBLISHED, pub_source.status)
11521139
1153 # nothing got published1140 # nothing got published
@@ -1224,8 +1211,7 @@ class TestPublisher(TestPublisherBase):
1224 helper function: 'getPublisher'1211 helper function: 'getPublisher'
1225 """1212 """
1226 # Stub parameters.1213 # Stub parameters.
1227 allowed_suites = [1214 allowed_suites = ['breezy-autotest']
1228 ('breezy-autotest', PackagePublishingPocket.RELEASE)]
12291215
1230 distro_publisher = getPublisher(1216 distro_publisher = getPublisher(
1231 self.ubuntutest.main_archive, allowed_suites, self.logger)1217 self.ubuntutest.main_archive, allowed_suites, self.logger)
@@ -1236,9 +1222,7 @@ class TestPublisher(TestPublisherBase):
1236 self.assertEqual(1222 self.assertEqual(
1237 '/var/tmp/archive/ubuntutest/dists',1223 '/var/tmp/archive/ubuntutest/dists',
1238 distro_publisher._config.distsroot)1224 distro_publisher._config.distsroot)
1239 self.assertEqual(1225 self.assertEqual({'breezy-autotest'}, distro_publisher.allowed_suites)
1240 [('breezy-autotest', PackagePublishingPocket.RELEASE)],
1241 distro_publisher.allowed_suites)
12421226
1243 # Check that the partner archive is built in a different directory1227 # Check that the partner archive is built in a different directory
1244 # to the primary archive.1228 # to the primary archive.
@@ -1263,9 +1247,7 @@ class TestPublisher(TestPublisherBase):
1263 self.assertEqual(1247 self.assertEqual(
1264 '/var/tmp/ppa.test/cprov/ppa/ubuntutest/dists',1248 '/var/tmp/ppa.test/cprov/ppa/ubuntutest/dists',
1265 archive_publisher._config.distsroot)1249 archive_publisher._config.distsroot)
1266 self.assertEqual(1250 self.assertEqual({'breezy-autotest'}, archive_publisher.allowed_suites)
1267 [('breezy-autotest', PackagePublishingPocket.RELEASE)],
1268 archive_publisher.allowed_suites)
12691251
1270 def testPendingArchive(self):1252 def testPendingArchive(self):
1271 """Check Pending Archive Lookup.1253 """Check Pending Archive Lookup.
@@ -1537,9 +1519,8 @@ class TestPublisher(TestPublisherBase):
1537 index_contents)1519 index_contents)
15381520
1539 # We always regenerate all Releases file for a given suite.1521 # We always regenerate all Releases file for a given suite.
1540 self.assertTrue(1522 self.assertIn(
1541 ('breezy-autotest', PackagePublishingPocket.RELEASE) in1523 'breezy-autotest', archive_publisher.release_files_needed)
1542 archive_publisher.release_files_needed)
15431524
1544 # Confirm that i18n files are not created1525 # Confirm that i18n files are not created
1545 i18n_path = os.path.join(archive_publisher._config.distsroot,1526 i18n_path = os.path.join(archive_publisher._config.distsroot,
@@ -1686,9 +1667,8 @@ class TestPublisher(TestPublisherBase):
1686 index_contents)1667 index_contents)
16871668
1688 # We always regenerate all Releases file for a given suite.1669 # We always regenerate all Releases file for a given suite.
1689 self.assertTrue(1670 self.assertIn(
1690 ('breezy-autotest', PackagePublishingPocket.RELEASE) in1671 'breezy-autotest', archive_publisher.release_files_needed)
1691 archive_publisher.release_files_needed)
16921672
1693 # Various compressed Translation-en files are written; ensure that1673 # Various compressed Translation-en files are written; ensure that
1694 # they are the same after decompression.1674 # they are the same after decompression.
@@ -1762,11 +1742,6 @@ class TestPublisher(TestPublisherBase):
1762 self._checkCompressedFiles(1742 self._checkCompressedFiles(
1763 archive_publisher, uncompressed_file_path, ['.xz'])1743 archive_publisher, uncompressed_file_path, ['.xz'])
17641744
1765 def checkDirtyPockets(self, publisher, expected):
1766 """Check dirty_pockets contents of a given publisher."""
1767 sorted_dirty_pockets = sorted(list(publisher.dirty_pockets))
1768 self.assertEqual(expected, sorted_dirty_pockets)
1769
1770 def testDirtyingPocketsWithDeletedPackages(self):1745 def testDirtyingPocketsWithDeletedPackages(self):
1771 """Test that dirtying pockets with deleted packages works.1746 """Test that dirtying pockets with deleted packages works.
17721747
@@ -1779,7 +1754,7 @@ class TestPublisher(TestPublisherBase):
1779 self.ubuntutest.main_archive, allowed_suites, self.logger)1754 self.ubuntutest.main_archive, allowed_suites, self.logger)
17801755
1781 publisher.A2_markPocketsWithDeletionsDirty()1756 publisher.A2_markPocketsWithDeletionsDirty()
1782 self.checkDirtyPockets(publisher, expected=[])1757 self.assertEqual(set(), publisher.dirty_suites)
17831758
1784 # Make a published source, a deleted source in the release1759 # Make a published source, a deleted source in the release
1785 # pocket, a source that's been removed from disk and one that's1760 # pocket, a source that's been removed from disk and one that's
@@ -1811,27 +1786,27 @@ class TestPublisher(TestPublisherBase):
1811 # Run the deletion detection.1786 # Run the deletion detection.
1812 publisher.A2_markPocketsWithDeletionsDirty()1787 publisher.A2_markPocketsWithDeletionsDirty()
18131788
1814 # Only the pockets with pending deletions are marked as dirty.1789 # Only the suites with pending deletions are marked as dirty.
1815 expected_dirty_pockets = [1790 expected_dirty_suites = {
1816 ('breezy-autotest', PackagePublishingPocket.RELEASE),1791 'breezy-autotest',
1817 ('breezy-autotest', PackagePublishingPocket.SECURITY),1792 'breezy-autotest-security',
1818 ('breezy-autotest', PackagePublishingPocket.BACKPORTS),1793 'breezy-autotest-backports',
1819 ]1794 }
1820 self.checkDirtyPockets(publisher, expected=expected_dirty_pockets)1795 self.assertEqual(expected_dirty_suites, publisher.dirty_suites)
18211796
1822 # If the distroseries is CURRENT, then the release pocket is not1797 # If the distroseries is CURRENT, then the release pocket is not
1823 # marked as dirty.1798 # marked as dirty.
1824 self.ubuntutest['breezy-autotest'].status = (1799 self.ubuntutest['breezy-autotest'].status = (
1825 SeriesStatus.CURRENT)1800 SeriesStatus.CURRENT)
18261801
1827 publisher.dirty_pockets = set()1802 publisher.dirty_suites = set()
1828 publisher.A2_markPocketsWithDeletionsDirty()1803 publisher.A2_markPocketsWithDeletionsDirty()
18291804
1830 expected_dirty_pockets = [1805 expected_dirty_suites = {
1831 ('breezy-autotest', PackagePublishingPocket.SECURITY),1806 'breezy-autotest-security',
1832 ('breezy-autotest', PackagePublishingPocket.BACKPORTS),1807 'breezy-autotest-backports',
1833 ]1808 }
1834 self.checkDirtyPockets(publisher, expected=expected_dirty_pockets)1809 self.assertEqual(expected_dirty_suites, publisher.dirty_suites)
18351810
1836 def testDeletionDetectionRespectsAllowedSuites(self):1811 def testDeletionDetectionRespectsAllowedSuites(self):
1837 """Check if the deletion detection mechanism respects allowed_suites.1812 """Check if the deletion detection mechanism respects allowed_suites.
@@ -1845,14 +1820,14 @@ class TestPublisher(TestPublisherBase):
1845 specified suites should be marked as dirty.1820 specified suites should be marked as dirty.
1846 """1821 """
1847 allowed_suites = [1822 allowed_suites = [
1848 ('breezy-autotest', PackagePublishingPocket.SECURITY),1823 'breezy-autotest-security',
1849 ('breezy-autotest', PackagePublishingPocket.UPDATES),1824 'breezy-autotest-updates',
1850 ]1825 ]
1851 publisher = getPublisher(1826 publisher = getPublisher(
1852 self.ubuntutest.main_archive, allowed_suites, self.logger)1827 self.ubuntutest.main_archive, allowed_suites, self.logger)
18531828
1854 publisher.A2_markPocketsWithDeletionsDirty()1829 publisher.A2_markPocketsWithDeletionsDirty()
1855 self.checkDirtyPockets(publisher, expected=[])1830 self.assertEqual(set(), publisher.dirty_suites)
18561831
1857 # Create pending deletions in RELEASE, BACKPORTS, SECURITY and1832 # Create pending deletions in RELEASE, BACKPORTS, SECURITY and
1858 # UPDATES pockets.1833 # UPDATES pockets.
@@ -1873,9 +1848,8 @@ class TestPublisher(TestPublisherBase):
1873 status=PackagePublishingStatus.DELETED)[0]1848 status=PackagePublishingStatus.DELETED)[0]
18741849
1875 publisher.A2_markPocketsWithDeletionsDirty()1850 publisher.A2_markPocketsWithDeletionsDirty()
1876 # Only the pockets with pending deletions in the allowed suites1851 # Only the suites with pending deletions are marked as dirty.
1877 # are marked as dirty.1852 self.assertEqual(set(allowed_suites), publisher.dirty_suites)
1878 self.checkDirtyPockets(publisher, expected=allowed_suites)
18791853
1880 def testReleaseFile(self):1854 def testReleaseFile(self):
1881 """Test release file writing.1855 """Test release file writing.
@@ -1892,9 +1866,7 @@ class TestPublisher(TestPublisherBase):
1892 publisher.A_publish(False)1866 publisher.A_publish(False)
1893 publisher.C_doFTPArchive(False)1867 publisher.C_doFTPArchive(False)
18941868
1895 self.assertIn(1869 self.assertIn('breezy-autotest', publisher.release_files_needed)
1896 ('breezy-autotest', PackagePublishingPocket.RELEASE),
1897 publisher.release_files_needed)
18981870
1899 publisher.D_writeReleaseFiles(False)1871 publisher.D_writeReleaseFiles(False)
19001872
@@ -2140,7 +2112,7 @@ class TestPublisher(TestPublisherBase):
2140 os.makedirs(os.path.dirname(contents_path))2112 os.makedirs(os.path.dirname(contents_path))
2141 with gzip.GzipFile(contents_path, 'wb'):2113 with gzip.GzipFile(contents_path, 'wb'):
2142 pass2114 pass
2143 publisher.markPocketDirty(2115 publisher.markSuiteDirty(
2144 self.ubuntutest.getSeries('breezy-autotest'),2116 self.ubuntutest.getSeries('breezy-autotest'),
2145 PackagePublishingPocket.RELEASE)2117 PackagePublishingPocket.RELEASE)
21462118
@@ -2170,7 +2142,7 @@ class TestPublisher(TestPublisherBase):
2170 for name in dep11_names:2142 for name in dep11_names:
2171 with gzip.GzipFile(os.path.join(dep11_path, name), 'wb') as f:2143 with gzip.GzipFile(os.path.join(dep11_path, name), 'wb') as f:
2172 f.write(six.ensure_binary(name))2144 f.write(six.ensure_binary(name))
2173 publisher.markPocketDirty(2145 publisher.markSuiteDirty(
2174 self.ubuntutest.getSeries('breezy-autotest'),2146 self.ubuntutest.getSeries('breezy-autotest'),
2175 PackagePublishingPocket.RELEASE)2147 PackagePublishingPocket.RELEASE)
21762148
@@ -2200,7 +2172,7 @@ class TestPublisher(TestPublisherBase):
2200 for name in cnf_names:2172 for name in cnf_names:
2201 with lzma.LZMAFile(os.path.join(cnf_path, name), 'wb') as f:2173 with lzma.LZMAFile(os.path.join(cnf_path, name), 'wb') as f:
2202 f.write(six.ensure_binary(name))2174 f.write(six.ensure_binary(name))
2203 publisher.markPocketDirty(2175 publisher.markSuiteDirty(
2204 self.ubuntutest.getSeries('breezy-autotest'),2176 self.ubuntutest.getSeries('breezy-autotest'),
2205 PackagePublishingPocket.RELEASE)2177 PackagePublishingPocket.RELEASE)
22062178
@@ -2460,8 +2432,7 @@ class TestArchiveIndices(TestPublisherBase):
2460 that those in 'absent' are not.2432 that those in 'absent' are not.
2461 """2433 """
24622434
2463 self.assertTrue(2435 self.assertIn(series.getSuite(pocket), publisher.release_files_needed)
2464 (series.name, pocket) in publisher.release_files_needed)
24652436
2466 arch_template = os.path.join(2437 arch_template = os.path.join(
2467 publisher._config.distsroot, series.getSuite(pocket), '%s/%s')2438 publisher._config.distsroot, series.getSuite(pocket), '%s/%s')
@@ -2774,7 +2745,7 @@ class TestUpdateByHash(TestPublisherBase):
2774 # Create the first file.2745 # Create the first file.
2775 with open_for_writing(suite_path('Contents-i386'), 'w') as f:2746 with open_for_writing(suite_path('Contents-i386'), 'w') as f:
2776 f.write('A Contents file\n')2747 f.write('A Contents file\n')
2777 publisher.markPocketDirty(2748 publisher.markSuiteDirty(
2778 self.breezy_autotest, PackagePublishingPocket.RELEASE)2749 self.breezy_autotest, PackagePublishingPocket.RELEASE)
2779 self.runSteps(publisher, step_a=True, step_c=True, step_d=True)2750 self.runSteps(publisher, step_a=True, step_c=True, step_d=True)
2780 flush_database_caches()2751 flush_database_caches()
@@ -2880,7 +2851,7 @@ class TestUpdateByHash(TestPublisherBase):
2880 self.setUpMockTime()2851 self.setUpMockTime()
28812852
2882 # Publish empty index files.2853 # Publish empty index files.
2883 publisher.markPocketDirty(2854 publisher.markSuiteDirty(
2884 self.breezy_autotest, PackagePublishingPocket.RELEASE)2855 self.breezy_autotest, PackagePublishingPocket.RELEASE)
2885 self.runSteps(publisher, step_a=True, step_c=True, step_d=True)2856 self.runSteps(publisher, step_a=True, step_c=True, step_d=True)
2886 suite_path = partial(2857 suite_path = partial(
@@ -2988,7 +2959,7 @@ class TestUpdateByHash(TestPublisherBase):
2988 self.ubuntutest.main_archive)2959 self.ubuntutest.main_archive)
2989 self.runSteps(publisher, step_a2=True, step_c=True, step_d=True)2960 self.runSteps(publisher, step_a2=True, step_c=True, step_d=True)
2990 transaction.commit()2961 transaction.commit()
2991 self.assertEqual(set(), publisher.dirty_pockets)2962 self.assertEqual(set(), publisher.dirty_suites)
2992 # The condemned index files are removed, and no new Release file is2963 # The condemned index files are removed, and no new Release file is
2993 # generated.2964 # generated.
2994 expected_suite_files = (2965 expected_suite_files = (
@@ -3021,7 +2992,7 @@ class TestUpdateByHash(TestPublisherBase):
3021 self.ubuntutest.main_archive)2992 self.ubuntutest.main_archive)
3022 self.runSteps(publisher, step_a2=True, step_c=True, step_d=True)2993 self.runSteps(publisher, step_a2=True, step_c=True, step_d=True)
3023 transaction.commit()2994 transaction.commit()
3024 self.assertEqual(set(), publisher.dirty_pockets)2995 self.assertEqual(set(), publisher.dirty_suites)
3025 self.assertEqual(release_mtime, os.stat(release_path).st_mtime)2996 self.assertEqual(release_mtime, os.stat(release_path).st_mtime)
3026 # The condemned index files are removed, and no new Release file is2997 # The condemned index files are removed, and no new Release file is
3027 # generated.2998 # generated.

Subscribers

People subscribed via source and target branches

to status/vote changes: