Merge lp:~cjwatson/launchpad/ftparchive-cleanup-old-indexes into lp:launchpad

Proposed by Colin Watson on 2016-02-09
Status: Merged
Merged at revision: 17923
Proposed branch: lp:~cjwatson/launchpad/ftparchive-cleanup-old-indexes
Merge into: lp:launchpad
Diff against target: 125 lines (+76/-8)
2 files modified
lib/lp/archivepublisher/model/ftparchive.py (+18/-8)
lib/lp/archivepublisher/tests/test_ftparchive.py (+58/-0)
To merge this branch: bzr merge lp:~cjwatson/launchpad/ftparchive-cleanup-old-indexes
Reviewer Review Type Date Requested Status
William Grant code 2016-02-09 Approve on 2016-02-09
Review via email: mp+285428@code.launchpad.net

Commit message

Remove old indexes before calling apt-ftparchive.

Description of the change

Remove old indexes before calling apt-ftparchive. This copes with the situation where we switch index_compressors to not contain bzip2 any more; without this we'd have outdated Packages.bz2 files left around on the filesystem.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)
Colin Watson (cjwatson) :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/archivepublisher/model/ftparchive.py'
2--- lib/lp/archivepublisher/model/ftparchive.py 2016-02-06 03:43:15 +0000
3+++ lib/lp/archivepublisher/model/ftparchive.py 2016-02-09 15:58:37 +0000
4@@ -50,9 +50,16 @@
5 return (os.path.basename(filename).split("_"))[0]
6
7
8-def safe_mkdir(path):
9- """Ensures the path exists, creating it if it doesn't."""
10- if not os.path.exists(path):
11+def make_empty_dir(path):
12+ """Ensure that the path exists and is an empty directory."""
13+ if os.path.isdir(path):
14+ for name in os.listdir(path):
15+ child_path = os.path.join(path, name)
16+ # Directories containing index files should never have
17+ # subdirectories. Guard against expensive mistakes by not
18+ # recursing here.
19+ os.unlink(child_path)
20+ else:
21 os.makedirs(path, 0o755)
22
23
24@@ -754,17 +761,20 @@
25 distroseries.include_long_descriptions,
26 distroseries.index_compressors)
27
28- # XXX: 2006-08-24 kiko: Why do we do this directory creation here?
29+ # Make sure all the relevant directories exist and are empty. Each
30+ # of these only contains files generated by apt-ftparchive, and may
31+ # contain files left over from previous configurations (e.g.
32+ # different compressor types).
33 for comp in comps:
34 component_path = os.path.join(
35 self._config.distsroot, suite, comp)
36- safe_mkdir(os.path.join(component_path, "source"))
37+ make_empty_dir(os.path.join(component_path, "source"))
38 if not distroseries.include_long_descriptions:
39- safe_mkdir(os.path.join(component_path, "i18n"))
40+ make_empty_dir(os.path.join(component_path, "i18n"))
41 for arch in archs:
42- safe_mkdir(os.path.join(component_path, "binary-" + arch))
43+ make_empty_dir(os.path.join(component_path, "binary-" + arch))
44 for subcomp in self.publisher.subcomponents:
45- safe_mkdir(os.path.join(
46+ make_empty_dir(os.path.join(
47 component_path, subcomp, "binary-" + arch))
48
49 def writeAptConfig(self, apt_config, suite, comps, archs,
50
51=== modified file 'lib/lp/archivepublisher/tests/test_ftparchive.py'
52--- lib/lp/archivepublisher/tests/test_ftparchive.py 2016-02-05 20:28:29 +0000
53+++ lib/lp/archivepublisher/tests/test_ftparchive.py 2016-02-09 15:58:37 +0000
54@@ -33,6 +33,7 @@
55 )
56 from lp.soyuz.enums import (
57 BinaryPackageFormat,
58+ IndexCompressionType,
59 PackagePublishingPriority,
60 PackagePublishingStatus,
61 )
62@@ -536,6 +537,63 @@
63 os.path.join(self._distsdir, "hoary-test", "main",
64 "source", "Sources.gz")))
65
66+ def test_generateConfig_index_compressors_changed(self):
67+ # If index_compressors changes between runs, then old compressed
68+ # files are removed.
69+ publisher = Publisher(
70+ self._logger, self._config, self._dp, self._archive)
71+ fa = FTPArchiveHandler(
72+ self._logger, self._config, self._dp, self._distribution,
73+ publisher)
74+ fa.createEmptyPocketRequests(fullpublish=True)
75+ self._publishDefaultOverrides(fa, "main")
76+ self._publishDefaultFileLists(fa, "main")
77+ self._addRepositoryFile("main", "tiny", "tiny_0.1.dsc")
78+ self._addRepositoryFile("main", "tiny", "tiny_0.1.tar.gz")
79+ self._addRepositoryFile("main", "tiny", "tiny_0.1_i386.deb")
80+ comp_dir = os.path.join(self._distsdir, "hoary-test", "main")
81+ os.makedirs(os.path.join(comp_dir, "uefi"))
82+ with open(os.path.join(comp_dir, "uefi", "stuff"), "w"):
83+ pass
84+
85+ # Run the publisher once with gzip and bzip2 compressors.
86+ apt_conf = fa.generateConfig(fullpublish=True)
87+ with open(apt_conf) as apt_conf_file:
88+ self.assertIn(
89+ 'Packages::Compress "gzip bzip2";', apt_conf_file.read())
90+ fa.runApt(apt_conf)
91+ self.assertContentEqual(
92+ ["Packages.gz", "Packages.bz2"],
93+ os.listdir(os.path.join(comp_dir, "binary-i386")))
94+ self.assertContentEqual(
95+ ["Packages.gz", "Packages.bz2"],
96+ os.listdir(os.path.join(
97+ comp_dir, "debian-installer", "binary-i386")))
98+ self.assertContentEqual(
99+ ["Sources.gz", "Sources.bz2"],
100+ os.listdir(os.path.join(comp_dir, "source")))
101+
102+ # Try again, this time with gzip and xz compressors. There are no
103+ # bzip2 leftovers, but other files are left untouched.
104+ self._distribution["hoary-test"].index_compressors = [
105+ IndexCompressionType.GZIP, IndexCompressionType.XZ]
106+ apt_conf = fa.generateConfig(fullpublish=True)
107+ with open(apt_conf) as apt_conf_file:
108+ self.assertIn(
109+ 'Packages::Compress "gzip xz";', apt_conf_file.read())
110+ fa.runApt(apt_conf)
111+ self.assertContentEqual(
112+ ["Packages.gz", "Packages.xz"],
113+ os.listdir(os.path.join(comp_dir, "binary-i386")))
114+ self.assertContentEqual(
115+ ["Packages.gz", "Packages.xz"],
116+ os.listdir(os.path.join(
117+ comp_dir, "debian-installer", "binary-i386")))
118+ self.assertContentEqual(
119+ ["Sources.gz", "Sources.xz"],
120+ os.listdir(os.path.join(comp_dir, "source")))
121+ self.assertEqual(["stuff"], os.listdir(os.path.join(comp_dir, "uefi")))
122+
123 def test_cleanCaches_noop_if_recent(self):
124 # cleanCaches does nothing if it was run recently.
125 fa = self._setUpFTPArchiveHandler()