Merge lp:~jelmer/launchpad/55288-publisher-unknown-distroseries into lp:launchpad

Proposed by Jelmer Vernooij
Status: Merged
Approved by: Jelmer Vernooij
Approved revision: no longer in the source branch.
Merged at revision: 11323
Proposed branch: lp:~jelmer/launchpad/55288-publisher-unknown-distroseries
Merge into: lp:launchpad
Diff against target: 236 lines (+67/-28)
4 files modified
lib/lp/archivepublisher/config.py (+19/-14)
lib/lp/archivepublisher/ftparchive.py (+11/-3)
lib/lp/archivepublisher/tests/test_config.py (+19/-9)
lib/lp/archivepublisher/tests/test_ftparchive.py (+18/-2)
To merge this branch: bzr merge lp:~jelmer/launchpad/55288-publisher-unknown-distroseries
Reviewer Review Type Date Requested Status
Leonard Richardson (community) code Approve
Review via email: mp+32232@code.launchpad.net

Commit message

Warn about distroseries without lucilleconfig rather than failing the entire publisher run.

Description of the change

This makes the publisher more robust against distroseries that have not been initialized for soyuz yet.

Rather than having the publisher configuration raise a LucilleConfigError when a distroseries doesn't have a matching entry in the lucille configuration, it will now write a warning to the log file.

To post a comment you must log in.
Revision history for this message
Leonard Richardson (leonardr) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/archivepublisher/config.py'
2--- lib/lp/archivepublisher/config.py 2010-07-07 06:28:03 +0000
3+++ lib/lp/archivepublisher/config.py 2010-08-10 17:43:45 +0000
4@@ -120,18 +120,15 @@
5 config_segment["archtags"].append(
6 dar.architecturetag.encode('utf-8'))
7
8- if not dr.lucilleconfig:
9- raise LucilleConfigError(
10- 'No Lucille configuration section for %s' % dr.name)
11-
12- strio = StringIO(dr.lucilleconfig.encode('utf-8'))
13- config_segment["config"] = ConfigParser()
14- config_segment["config"].readfp(strio)
15- strio.close()
16- config_segment["components"] = config_segment["config"].get(
17- "publishing", "components").split(" ")
18-
19- self._distroseries[distroseries_name] = config_segment
20+ if dr.lucilleconfig:
21+ strio = StringIO(dr.lucilleconfig.encode('utf-8'))
22+ config_segment["config"] = ConfigParser()
23+ config_segment["config"].readfp(strio)
24+ strio.close()
25+ config_segment["components"] = config_segment["config"].get(
26+ "publishing", "components").split(" ")
27+
28+ self._distroseries[distroseries_name] = config_segment
29
30 strio = StringIO(distribution.lucilleconfig.encode('utf-8'))
31 self._distroconfig = ConfigParser()
32@@ -144,11 +141,19 @@
33 # Because dicts iterate for keys only; this works to get dr names
34 return self._distroseries.keys()
35
36+ def series(self, dr):
37+ try:
38+ return self._distroseries[dr]
39+ except KeyError:
40+ raise LucilleConfigError(
41+ 'No Lucille config section for %s in %s' %
42+ (dr, self.distroName))
43+
44 def archTagsForSeries(self, dr):
45- return self._distroseries[dr]["archtags"]
46+ return self.series(dr)["archtags"]
47
48 def componentsForSeries(self, dr):
49- return self._distroseries[dr]["components"]
50+ return self.series(dr)["components"]
51
52 def _extractConfigInfo(self):
53 """Extract configuration information into the attributes we use"""
54
55=== modified file 'lib/lp/archivepublisher/ftparchive.py'
56--- lib/lp/archivepublisher/ftparchive.py 2009-10-26 18:40:04 +0000
57+++ lib/lp/archivepublisher/ftparchive.py 2010-08-10 17:43:45 +0000
58@@ -138,6 +138,14 @@
59 self._config = config
60 self._diskpool = diskpool
61 self.distro = distro
62+ self.distroseries = []
63+ for distroseries in self.distro.series:
64+ if not distroseries.name in self._config.distroSeriesNames():
65+ self.log.warning("Distroseries %s in %s doesn't have "
66+ "a lucille configuration.", distroseries.name,
67+ self.distro.name)
68+ else:
69+ self.distroseries.append(distroseries)
70 self.publisher = publisher
71 self.release_files_needed = {}
72
73@@ -185,7 +193,7 @@
74 # iterate over the pockets, and do the suffix check inside
75 # createEmptyPocketRequest; that would also allow us to replace
76 # the == "" check we do there by a RELEASE match
77- for distroseries in self.distro:
78+ for distroseries in self.distroseries:
79 components = self._config.componentsForSeries(distroseries.name)
80 for pocket, suffix in pocketsuffix.items():
81 if not fullpublish:
82@@ -366,7 +374,7 @@
83
84 def generateOverrides(self, fullpublish=False):
85 """Collect packages that need overrides, and generate them."""
86- for distroseries in self.distro.series:
87+ for distroseries in self.distroseries:
88 for pocket in PackagePublishingPocket.items:
89 if not fullpublish:
90 if not self.publisher.isDirty(distroseries, pocket):
91@@ -629,7 +637,7 @@
92
93 def generateFileLists(self, fullpublish=False):
94 """Collect currently published FilePublishings and write filelists."""
95- for distroseries in self.distro.series:
96+ for distroseries in self.distroseries:
97 for pocket in pocketsuffix:
98 if not fullpublish:
99 if not self.publisher.isDirty(distroseries, pocket):
100
101=== modified file 'lib/lp/archivepublisher/tests/test_config.py'
102--- lib/lp/archivepublisher/tests/test_config.py 2010-07-18 00:24:06 +0000
103+++ lib/lp/archivepublisher/tests/test_config.py 2010-08-10 17:43:45 +0000
104@@ -5,36 +5,48 @@
105
106 __metaclass__ = type
107
108-import unittest
109-
110 from zope.component import getUtility
111
112 from canonical.config import config
113 from canonical.launchpad.interfaces import IDistributionSet
114 from canonical.testing import LaunchpadZopelessLayer
115
116-
117-class TestConfig(unittest.TestCase):
118+from lp.archivepublisher.config import Config, LucilleConfigError
119+from lp.testing import TestCaseWithFactory
120+
121+
122+class TestConfig(TestCaseWithFactory):
123 layer = LaunchpadZopelessLayer
124
125 def setUp(self):
126+ super(TestConfig, self).setUp()
127 self.layer.switchDbUser(config.archivepublisher.dbuser)
128 self.ubuntutest = getUtility(IDistributionSet)['ubuntutest']
129
130+ def testMissingDistroSeries(self):
131+ distroseries = self.factory.makeDistroSeries(
132+ distribution=self.ubuntutest, name="somename")
133+ d = Config(self.ubuntutest)
134+ dsns = d.distroSeriesNames()
135+ self.assertEquals(len(dsns), 2)
136+ self.assertEquals(dsns[0], "breezy-autotest")
137+ self.assertEquals(dsns[1], "hoary-test")
138+ self.assertRaises(LucilleConfigError,
139+ d.archTagsForSeries, "somename")
140+ self.assertRaises(LucilleConfigError,
141+ d.archTagsForSeries, "unknown")
142+
143 def testInstantiate(self):
144 """Config should instantiate"""
145- from lp.archivepublisher.config import Config
146 d = Config(self.ubuntutest)
147
148 def testDistroName(self):
149 """Config should be able to return the distroName"""
150- from lp.archivepublisher.config import Config
151 d = Config(self.ubuntutest)
152 self.assertEqual(d.distroName, "ubuntutest")
153
154 def testDistroSeriesNames(self):
155 """Config should return two distroseries names"""
156- from lp.archivepublisher.config import Config
157 d = Config(self.ubuntutest)
158 dsns = d.distroSeriesNames()
159 self.assertEquals(len(dsns), 2)
160@@ -43,14 +55,12 @@
161
162 def testArchTagsForSeries(self):
163 """Config should have the arch tags for the drs"""
164- from lp.archivepublisher.config import Config
165 d = Config(self.ubuntutest)
166 archs = d.archTagsForSeries("hoary-test")
167 self.assertEquals(len(archs), 2)
168
169 def testDistroConfig(self):
170 """Config should have parsed a distro config"""
171- from lp.archivepublisher.config import Config
172 d = Config(self.ubuntutest)
173 # NOTE: Add checks here when you add stuff in util.py
174 self.assertEquals(d.stayofexecution, 5)
175
176=== modified file 'lib/lp/archivepublisher/tests/test_ftparchive.py'
177--- lib/lp/archivepublisher/tests/test_ftparchive.py 2010-07-18 00:24:06 +0000
178+++ lib/lp/archivepublisher/tests/test_ftparchive.py 2010-08-10 17:43:45 +0000
179@@ -15,7 +15,7 @@
180 from zope.component import getUtility
181
182 from canonical.config import config
183-from canonical.launchpad.scripts.logger import QuietFakeLogger
184+from canonical.launchpad.scripts.logger import BufferLogger, QuietFakeLogger
185 from canonical.testing import LaunchpadZopelessLayer
186 from lp.archivepublisher.config import Config
187 from lp.archivepublisher.diskpool import DiskPool
188@@ -23,6 +23,7 @@
189 from lp.archivepublisher.publishing import Publisher
190 from lp.registry.interfaces.distribution import IDistributionSet
191 from lp.registry.interfaces.pocket import PackagePublishingPocket
192+from lp.testing import TestCaseWithFactory
193
194
195 def sanitize_apt_ftparchive_Sources_output(text):
196@@ -55,10 +56,11 @@
197 return self._result[i:j]
198
199
200-class TestFTPArchive(unittest.TestCase):
201+class TestFTPArchive(TestCaseWithFactory):
202 layer = LaunchpadZopelessLayer
203
204 def setUp(self):
205+ super(TestFTPArchive, self).setUp()
206 self.layer.switchDbUser(config.archivepublisher.dbuser)
207
208 self._distribution = getUtility(IDistributionSet)['ubuntutest']
209@@ -79,6 +81,7 @@
210 self._publisher = SamplePublisher(self._archive)
211
212 def tearDown(self):
213+ super(TestFTPArchive, self).tearDown()
214 shutil.rmtree(self._config.distroroot)
215
216 def _verifyFile(self, filename, directory, output_filter=None):
217@@ -116,6 +119,19 @@
218 self._publisher)
219 return fa
220
221+ def test_NoLucilleConfig(self):
222+ # Distroseries without a lucille configuration get ignored
223+ # and trigger a warning, they don't break the publisher
224+ logger = BufferLogger()
225+ publisher = Publisher(
226+ logger, self._config, self._dp, self._archive)
227+ self.factory.makeDistroSeries(self._distribution, name="somename")
228+ fa = FTPArchiveHandler(logger, self._config, self._dp,
229+ self._distribution, publisher)
230+ fa.createEmptyPocketRequests(fullpublish=True)
231+ self.assertEquals("WARNING: Distroseries somename in ubuntutest doesn't "
232+ "have a lucille configuration.\n", logger.buffer.getvalue())
233+
234 def test_getSourcesForOverrides(self):
235 # getSourcesForOverrides returns a list of tuples containing:
236 # (sourcename, suite, component, section)