Merge lp:~cjwatson/launchpad/ftparchive-cleanup-old-indexes into lp:launchpad

Proposed by Colin Watson on 2016-02-09
Status: Merged
Merged at revision: 17923
Proposed branch: lp:~cjwatson/launchpad/ftparchive-cleanup-old-indexes
Merge into: lp:launchpad
Diff against target: 125 lines (+76/-8)
2 files modified
lib/lp/archivepublisher/model/ftparchive.py (+18/-8)
lib/lp/archivepublisher/tests/test_ftparchive.py (+58/-0)
To merge this branch: bzr merge lp:~cjwatson/launchpad/ftparchive-cleanup-old-indexes
Reviewer Review Type Date Requested Status
William Grant code 2016-02-09 Approve on 2016-02-09
Review via email: mp+285428@code.launchpad.net

Commit message

Remove old indexes before calling apt-ftparchive.

Description of the change

Remove old indexes before calling apt-ftparchive. This copes with the situation where we switch index_compressors to not contain bzip2 any more; without this we'd have outdated Packages.bz2 files left around on the filesystem.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)
Colin Watson (cjwatson) :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'lib/lp/archivepublisher/model/ftparchive.py'
--- lib/lp/archivepublisher/model/ftparchive.py 2016-02-06 03:43:15 +0000
+++ lib/lp/archivepublisher/model/ftparchive.py 2016-02-09 15:58:37 +0000
@@ -50,9 +50,16 @@
50 return (os.path.basename(filename).split("_"))[0]50 return (os.path.basename(filename).split("_"))[0]
5151
5252
53def safe_mkdir(path):53def make_empty_dir(path):
54 """Ensures the path exists, creating it if it doesn't."""54 """Ensure that the path exists and is an empty directory."""
55 if not os.path.exists(path):55 if os.path.isdir(path):
56 for name in os.listdir(path):
57 child_path = os.path.join(path, name)
58 # Directories containing index files should never have
59 # subdirectories. Guard against expensive mistakes by not
60 # recursing here.
61 os.unlink(child_path)
62 else:
56 os.makedirs(path, 0o755)63 os.makedirs(path, 0o755)
5764
5865
@@ -754,17 +761,20 @@
754 distroseries.include_long_descriptions,761 distroseries.include_long_descriptions,
755 distroseries.index_compressors)762 distroseries.index_compressors)
756763
757 # XXX: 2006-08-24 kiko: Why do we do this directory creation here?764 # Make sure all the relevant directories exist and are empty. Each
765 # of these only contains files generated by apt-ftparchive, and may
766 # contain files left over from previous configurations (e.g.
767 # different compressor types).
758 for comp in comps:768 for comp in comps:
759 component_path = os.path.join(769 component_path = os.path.join(
760 self._config.distsroot, suite, comp)770 self._config.distsroot, suite, comp)
761 safe_mkdir(os.path.join(component_path, "source"))771 make_empty_dir(os.path.join(component_path, "source"))
762 if not distroseries.include_long_descriptions:772 if not distroseries.include_long_descriptions:
763 safe_mkdir(os.path.join(component_path, "i18n"))773 make_empty_dir(os.path.join(component_path, "i18n"))
764 for arch in archs:774 for arch in archs:
765 safe_mkdir(os.path.join(component_path, "binary-" + arch))775 make_empty_dir(os.path.join(component_path, "binary-" + arch))
766 for subcomp in self.publisher.subcomponents:776 for subcomp in self.publisher.subcomponents:
767 safe_mkdir(os.path.join(777 make_empty_dir(os.path.join(
768 component_path, subcomp, "binary-" + arch))778 component_path, subcomp, "binary-" + arch))
769779
770 def writeAptConfig(self, apt_config, suite, comps, archs,780 def writeAptConfig(self, apt_config, suite, comps, archs,
771781
=== modified file 'lib/lp/archivepublisher/tests/test_ftparchive.py'
--- lib/lp/archivepublisher/tests/test_ftparchive.py 2016-02-05 20:28:29 +0000
+++ lib/lp/archivepublisher/tests/test_ftparchive.py 2016-02-09 15:58:37 +0000
@@ -33,6 +33,7 @@
33 )33 )
34from lp.soyuz.enums import (34from lp.soyuz.enums import (
35 BinaryPackageFormat,35 BinaryPackageFormat,
36 IndexCompressionType,
36 PackagePublishingPriority,37 PackagePublishingPriority,
37 PackagePublishingStatus,38 PackagePublishingStatus,
38 )39 )
@@ -536,6 +537,63 @@
536 os.path.join(self._distsdir, "hoary-test", "main",537 os.path.join(self._distsdir, "hoary-test", "main",
537 "source", "Sources.gz")))538 "source", "Sources.gz")))
538539
540 def test_generateConfig_index_compressors_changed(self):
541 # If index_compressors changes between runs, then old compressed
542 # files are removed.
543 publisher = Publisher(
544 self._logger, self._config, self._dp, self._archive)
545 fa = FTPArchiveHandler(
546 self._logger, self._config, self._dp, self._distribution,
547 publisher)
548 fa.createEmptyPocketRequests(fullpublish=True)
549 self._publishDefaultOverrides(fa, "main")
550 self._publishDefaultFileLists(fa, "main")
551 self._addRepositoryFile("main", "tiny", "tiny_0.1.dsc")
552 self._addRepositoryFile("main", "tiny", "tiny_0.1.tar.gz")
553 self._addRepositoryFile("main", "tiny", "tiny_0.1_i386.deb")
554 comp_dir = os.path.join(self._distsdir, "hoary-test", "main")
555 os.makedirs(os.path.join(comp_dir, "uefi"))
556 with open(os.path.join(comp_dir, "uefi", "stuff"), "w"):
557 pass
558
559 # Run the publisher once with gzip and bzip2 compressors.
560 apt_conf = fa.generateConfig(fullpublish=True)
561 with open(apt_conf) as apt_conf_file:
562 self.assertIn(
563 'Packages::Compress "gzip bzip2";', apt_conf_file.read())
564 fa.runApt(apt_conf)
565 self.assertContentEqual(
566 ["Packages.gz", "Packages.bz2"],
567 os.listdir(os.path.join(comp_dir, "binary-i386")))
568 self.assertContentEqual(
569 ["Packages.gz", "Packages.bz2"],
570 os.listdir(os.path.join(
571 comp_dir, "debian-installer", "binary-i386")))
572 self.assertContentEqual(
573 ["Sources.gz", "Sources.bz2"],
574 os.listdir(os.path.join(comp_dir, "source")))
575
576 # Try again, this time with gzip and xz compressors. There are no
577 # bzip2 leftovers, but other files are left untouched.
578 self._distribution["hoary-test"].index_compressors = [
579 IndexCompressionType.GZIP, IndexCompressionType.XZ]
580 apt_conf = fa.generateConfig(fullpublish=True)
581 with open(apt_conf) as apt_conf_file:
582 self.assertIn(
583 'Packages::Compress "gzip xz";', apt_conf_file.read())
584 fa.runApt(apt_conf)
585 self.assertContentEqual(
586 ["Packages.gz", "Packages.xz"],
587 os.listdir(os.path.join(comp_dir, "binary-i386")))
588 self.assertContentEqual(
589 ["Packages.gz", "Packages.xz"],
590 os.listdir(os.path.join(
591 comp_dir, "debian-installer", "binary-i386")))
592 self.assertContentEqual(
593 ["Sources.gz", "Sources.xz"],
594 os.listdir(os.path.join(comp_dir, "source")))
595 self.assertEqual(["stuff"], os.listdir(os.path.join(comp_dir, "uefi")))
596
539 def test_cleanCaches_noop_if_recent(self):597 def test_cleanCaches_noop_if_recent(self):
540 # cleanCaches does nothing if it was run recently.598 # cleanCaches does nothing if it was run recently.
541 fa = self._setUpFTPArchiveHandler()599 fa = self._setUpFTPArchiveHandler()