Merge lp:~brian-murray/ubuntu-release-upgrader/improve-upgrade-failure-messages into lp:ubuntu-release-upgrader

Proposed by Brian Murray on 2018-12-11
Status: Merged
Merged at revision: 3201
Proposed branch: lp:~brian-murray/ubuntu-release-upgrader/improve-upgrade-failure-messages
Merge into: lp:ubuntu-release-upgrader
Diff against target: 474 lines (+120/-30)
6 files modified
DistUpgrade/DistUpgradeCache.py (+19/-7)
DistUpgrade/DistUpgradeController.py (+25/-5)
DistUpgrade/DistUpgradeFetcherCore.py (+1/-1)
debian/changelog (+14/-2)
do-release-upgrade (+12/-0)
tests/test_sources_list.py (+49/-15)
To merge this branch: bzr merge lp:~brian-murray/ubuntu-release-upgrader/improve-upgrade-failure-messages
Reviewer Review Type Date Requested Status
Ubuntu Core Development Team 2018-12-11 Pending
Review via email: mp+360636@code.launchpad.net
To post a comment you must log in.
3201. By Brian Murray on 2018-12-11

* DistUpgrade/DistUpgradeCache.py: in the event there is a failure to calculate the upgrade provide information about the log files in /var/log/dist-upgrade. (LP: #1771387)

3202. By Brian Murray on 2018-12-11

ppa-purge only works with Launchpad PPAs

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified symlink 'DistUpgrade/DistUpgrade'
2=== target changed u'.' => u'DistUpgrade'
3=== modified file 'DistUpgrade/DistUpgradeCache.py'
4--- DistUpgrade/DistUpgradeCache.py 2018-11-28 19:12:50 +0000
5+++ DistUpgrade/DistUpgradeCache.py 2018-12-11 23:25:23 +0000
6@@ -666,14 +666,23 @@
7 # rationale why it doesn't
8 lock.release()
9 t.join()
10- # FIXME: change the text to something more useful
11+ # the most likely problem is the 3rd party pkgs so don't address
12+ # foreignPkgs and devRelease being True
13 details = _("An unresolvable problem occurred while "
14- "calculating the upgrade.\n\n "
15- "This can be caused by:\n"
16- " * Upgrading to a pre-release version of Ubuntu\n"
17- " * Running the current pre-release version of Ubuntu\n"
18- " * Unofficial software packages not provided by Ubuntu\n"
19- "\n")
20+ "calculating the upgrade.\n\n ")
21+ if self.config.get("Options", "foreignPkgs") == "True":
22+ details += _("This was likely caused by:\n"
23+ " * Unofficial software packages not provided by Ubuntu\n"
24+ "Please use the tool 'ppa-purge' from the ppa-purge \n"
25+ "package to remove software from a Launchpad PPA and \n"
26+ "try the upgrade again.\n"
27+ "\n")
28+ elif self.config.get("Options", "foreignPkgs") == "False" and \
29+ self.config.get("Options", "devRelease") == "True":
30+ details += _("This was caused by:\n"
31+ " * Upgrading to a pre-release version of Ubuntu\n"
32+ "This is most likely a transient problem, \n"
33+ "please try again later.\n")
34 # we never have partialUpgrades (including removes) on a stable system
35 # with only ubuntu sources so we do not recommend reporting a bug
36 if partialUpgrade:
37@@ -682,6 +691,9 @@
38 else:
39 details += _("If none of this applies, then please report this bug using "
40 "the command 'ubuntu-bug ubuntu-release-upgrader-core' in a terminal.")
41+ details += _("If you want to investigate this yourself the log files in "
42+ "'/var/log/dist-upgrade' will contain details about the upgrade. "
43+ "Specifically, look at 'main.log' and 'apt.log'.")
44 # make the error text available again on stdout for the
45 # text frontend
46 self._stopAptResolverLog()
47
48=== modified file 'DistUpgrade/DistUpgradeController.py'
49--- DistUpgrade/DistUpgradeController.py 2018-10-31 22:06:06 +0000
50+++ DistUpgrade/DistUpgradeController.py 2018-12-11 23:25:23 +0000
51@@ -131,6 +131,11 @@
52 # ConfigParser deals only with strings it seems *sigh*
53 self.config.add_section("Options")
54 self.config.set("Options","withNetwork", str(self.useNetwork))
55+ if self.options:
56+ if self.options.devel_release:
57+ self.config.set("Options","devRelease", "True")
58+ else:
59+ self.config.set("Options","devRelease", "False")
60
61 # some constants here
62 self.fromDist = self.config.get("Sources","From")
63@@ -535,6 +540,7 @@
64 distro.get_sources(self.sources)
65 distro.enable_component("main")
66 main_was_missing = True
67+ logging.debug('get_distro().enable_component("main") succeeded')
68 except NoDistroTemplateException as e:
69 logging.exception('NoDistroTemplateException raised: %s' % e)
70 # fallback if everything else does not work,
71@@ -680,8 +686,16 @@
72 foundToDist |= validTo
73 elif entry.dist in fromDists:
74 foundToDist |= validTo
75- entry.dist = toDists[fromDists.index(entry.dist)]
76- logging.debug("entry '%s' updated to new dist" % get_string_with_no_auth_from_source_entry(entry))
77+ # check to see whether the archive provides the new dist
78+ test_entry = copy.copy(entry)
79+ test_entry.dist = self.toDist
80+ if not self._sourcesListEntryDownloadable(test_entry):
81+ entry.disabled = True
82+ self.sources_disabled = True
83+ logging.debug("entry '%s' was disabled (no Release file)" % get_string_with_no_auth_from_source_entry(entry))
84+ else:
85+ entry.dist = toDists[fromDists.index(entry.dist)]
86+ logging.debug("entry '%s' updated to new dist" % get_string_with_no_auth_from_source_entry(entry))
87 elif entry.type == 'deb-src':
88 continue
89 elif validMirror:
90@@ -933,6 +947,12 @@
91 # compare the list after the update again
92 self.obsolete_pkgs = self.cache._getObsoletesPkgs()
93 self.foreign_pkgs = self.cache._getForeignPkgs(self.origin, self.fromDist, self.toDist)
94+ # If a PPA has already been disabled the pkgs won't be considered
95+ # foreign
96+ if len(self.foreign_pkgs) > 0:
97+ self.config.set("Options","foreignPkgs", "True")
98+ else:
99+ self.config.set("Options","foreignPkgs", "False")
100 if self.serverMode:
101 self.tasks = self.cache.installedTasks
102 logging.debug("Foreign: %s" % " ".join(sorted(self.foreign_pkgs)))
103@@ -1486,7 +1506,7 @@
104 self._view.getTerminal().call([script], hidden=True)
105 except Exception as e:
106 logging.error("got error from PostInstallScript %s (%s)" % (script, e))
107-
108+
109 def abort(self):
110 """ abort the upgrade, cleanup (as much as possible) """
111 logging.debug("abort called")
112@@ -1518,7 +1538,7 @@
113 return True
114 logging.error("depends '%s' is not satisfied" % depstr)
115 return False
116-
117+
118 def checkViewDepends(self):
119 " check if depends are satisfied "
120 logging.debug("checkViewDepends()")
121@@ -1537,7 +1557,7 @@
122 _("The required dependency '%s' is not "
123 "installed. " % dep))
124 sys.exit(1)
125- return res
126+ return res
127
128 def _verifyBackports(self):
129 # run update (but ignore errors in case the countrymirror
130
131=== modified file 'DistUpgrade/DistUpgradeFetcherCore.py'
132--- DistUpgrade/DistUpgradeFetcherCore.py 2016-11-29 16:36:37 +0000
133+++ DistUpgrade/DistUpgradeFetcherCore.py 2018-12-11 23:25:23 +0000
134@@ -220,7 +220,7 @@
135 def runDistUpgrader(self):
136 args = [self.script] + self.run_options
137 if os.getuid() != 0:
138- os.execv("/usr/bin/sudo", ["sudo"] + args)
139+ os.execv("/usr/bin/sudo", ["sudo", "-E"] + args)
140 else:
141 os.execv(self.script, args)
142
143
144=== modified file 'debian/changelog'
145--- debian/changelog 2018-11-28 22:11:44 +0000
146+++ debian/changelog 2018-12-11 23:25:23 +0000
147@@ -1,10 +1,22 @@
148-ubuntu-release-upgrader (1:19.04.7) UNRELEASED; urgency=medium
149+ubuntu-release-upgrader (1:19.04.7ubuntu1) UNRELEASED; urgency=medium
150
151 * DistUpgrade/DistUpgradeCache.py: When calculating free space needed for
152 mount points don't use a negative number as the buffer. (LP: #1786484)
153 * fix more typos re RELEASE_UPRADER_ALLOW_THIRD_PARTY
154+ * DistUpgrade/DistUpgradeController.py: When rewriting sources.list entries
155+ check to see if the source provides packages for the release to which the
156+ upgrade is occurring. If it doesn't the entry is disabled thereby
157+ improving upgrades with PPAs. (LP: #1807043)
158+ * do-release-upgrade: add a parameter to allow third party mirrors and
159+ repositories, additionally pass along the environmental variable
160+ RELEASE_UPGRADER_ALLOW_THIRD_PARTY via pkexec and sudo. (LP: #1807032)
161+ * DistUpgrade/DistUpgradeCache.py: improve messaging regarding failures to
162+ calculate the upgrade and suggest using ppa-purge if PPAs are an issue.
163+ * DistUpgrade/DistUpgradeCache.py: in the event there is a failure to
164+ calculate the upgrade provide information about the log files in
165+ /var/log/dist-upgrade. (LP: #1771387)
166
167- -- Brian Murray <brian@ubuntu.com> Wed, 28 Nov 2018 14:11:10 -0800
168+ -- Brian Murray <brian@ubuntu.com> Tue, 11 Dec 2018 15:12:19 -0800
169
170 ubuntu-release-upgrader (1:19.04.6) disco; urgency=medium
171
172
173=== modified file 'do-release-upgrade'
174--- do-release-upgrade 2018-11-28 18:02:37 +0000
175+++ do-release-upgrade 2018-12-11 23:25:23 +0000
176@@ -103,6 +103,11 @@
177 help=_("Check only if a new distribution release is "
178 "available and report the result via the "
179 "exit code"))
180+ parser.add_option ("--allow-third-party", default=False,
181+ action="store_true", dest="allow_third_party",
182+ help=_("Try the upgrade with third party "
183+ "mirrors and repositories enabled "
184+ "instead of commenting them out."))
185 parser.add_option ("-q", "--quiet", default=False, action="store_true",
186 dest="quiet")
187
188@@ -195,12 +200,19 @@
189 "requires a reboot. Please reboot before upgrading."))
190 sys.exit(1)
191
192+ if options.allow_third_party:
193+ # the env variable is used by code in the dist-upgrader tarball
194+ os.environ["RELEASE_UPGRADER_ALLOW_THIRD_PARTY"] = "True"
195+
196 # GTK 3 fetcher starts setting up the the GUI and KDE GUI needs to start GUI
197 # later, too.
198 if (options.frontend in {"DistUpgradeViewGtk3", "DistUpgradeViewKDE"} and
199 os.getuid() != 0):
200 if 'WAYLAND_DISPLAY' in os.environ:
201 subprocess.run(['xhost', '+si:localuser:root'])
202+ # this is needed because pkexec doesn't pass on the env
203+ if 'RELEASE_UPGRADER_ALLOW_THIRD_PARTY' in os.environ:
204+ sys.argv.append('--allow-third-party')
205 os.execv("/usr/bin/pkexec", ["pkexec"] + sys.argv)
206
207 fetcher = get_fetcher(options.frontend, m.new_dist, options.data_dir)
208
209=== modified file 'tests/test_sources_list.py'
210--- tests/test_sources_list.py 2018-11-28 22:11:44 +0000
211+++ tests/test_sources_list.py 2018-12-11 23:25:23 +0000
212@@ -92,8 +92,9 @@
213 deb http://security.ubuntu.com/ubuntu/ gutsy-security main restricted
214 """)
215
216+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
217 @mock.patch("DistUpgrade.DistUpgradeController.get_distro")
218- def test_sources_list_rewrite(self, mock_get_distro):
219+ def test_sources_list_rewrite(self, mock_get_distro, mock_sourcesListEntryDownloadable):
220 """
221 test regular sources.list rewrite
222 """
223@@ -107,8 +108,10 @@
224 "Ubuntu Feisty Fawn",
225 "7.04")
226 d.openCache(lock=False)
227+ mock_sourcesListEntryDownloadable.return_value = True
228 res = d.updateSourcesList()
229 self.assertTrue(mock_get_distro.called)
230+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
231 self.assertTrue(res)
232
233 # now test the result
234@@ -127,7 +130,7 @@
235 apt_pkg.config.find_file("Dir::Etc::sourcelist") + ".in",
236 apt_pkg.config.find_file("Dir::Etc::sourcelist") + ".distUpgrade"
237 ]))
238-
239+
240 @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController.abort")
241 @mock.patch("DistUpgrade.DistUpgradeController.get_distro")
242 def test_double_check_source_distribution_reject(self, mock_abort, mock_get_distro):
243@@ -145,10 +148,10 @@
244 mock_get_distro.return_value = UbuntuDistribution("Ubuntu", "feisty",
245 "Ubuntu Feisty Fawn",
246 "7.04")
247-
248+
249 class AbortException(Exception):
250 """Exception"""
251-
252+
253 mock_abort.side_effect = AbortException
254 d.openCache(lock=False)
255 with self.assertRaises(AbortException):
256@@ -157,8 +160,9 @@
257 self.assertTrue(mock_get_distro.called)
258 self.assertTrue(v.askYesNoQuestion.called)
259
260+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
261 @mock.patch("DistUpgrade.DistUpgradeController.get_distro")
262- def test_double_check_source_distribution_continue(self, mock_get_distro):
263+ def test_double_check_source_distribution_continue(self, mock_get_distro, mock_sourcesListEntryDownloadable):
264 """
265 test that an upgrade from feisty with a sources.list containing
266 hardy asks a question, and if continued, does something.
267@@ -174,13 +178,15 @@
268 "Ubuntu Feisty Fawn",
269 "7.04")
270 d.openCache(lock=False)
271+ mock_sourcesListEntryDownloadable.return_value = True
272 res = d.updateSourcesList()
273 self.assertTrue(mock_get_distro.called)
274+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
275 self.assertTrue(res)
276
277 # now test the result
278 #print(open(os.path.join(self.testdir,"sources.list")).read())
279-
280+
281 # The result here is not really all that helpful, hence we
282 # added the question in the first place. But still better to
283 # check what it does than to not check it.
284@@ -195,6 +201,7 @@
285 # deb http://security.ubuntu.com/ubuntu/ hardy-security universe
286
287 deb http://archive.ubuntu.com/ubuntu/ gutsy main
288+
289 """)
290 # check that the backup file was created correctly
291 self.assertEqual(0, subprocess.call(
292@@ -203,8 +210,9 @@
293 apt_pkg.config.find_file("Dir::Etc::sourcelist") + ".distUpgrade"
294 ]))
295
296+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
297 @mock.patch("DistUpgrade.DistUpgradeController.get_distro")
298- def test_sources_list_inactive_mirror(self, mock_get_distro):
299+ def test_sources_list_inactive_mirror(self, mock_get_distro, mock_sourcesListEntryDownloadable):
300 """
301 test sources.list rewrite of an obsolete mirror
302 """
303@@ -218,8 +226,10 @@
304 "Ubuntu Feisty Fawn",
305 "7.04")
306 d.openCache(lock=False)
307+ mock_sourcesListEntryDownloadable.return_value = True
308 res = d.updateSourcesList()
309 self.assertTrue(mock_get_distro.called)
310+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
311 self.assertTrue(res)
312
313 # now test the result
314@@ -311,8 +321,9 @@
315 deb http://archive.canonical.com/ubuntu gutsy partner
316 """)
317
318+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
319 @mock.patch("DistUpgrade.DistUpgradeController.get_distro")
320- def test_extras_removal(self, mock_get_distro):
321+ def test_extras_removal(self, mock_get_distro, mock_sourcesListEntryDownloadable):
322 """
323 test removal of extras.ubuntu.com archives
324 """
325@@ -328,7 +339,9 @@
326 "Ubuntu Feisty Fawn",
327 "7.04")
328 d.openCache(lock=False)
329+ mock_sourcesListEntryDownloadable.return_value = True
330 res = d.updateSourcesList()
331+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
332 self.assertTrue(res)
333
334 sources_file = apt_pkg.config.find_file("Dir::Etc::sourcelist")
335@@ -338,7 +351,8 @@
336
337 """)
338
339- def test_powerpc_transition(self):
340+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
341+ def test_powerpc_transition(self, mock_sourcesListEntryDownloadable):
342 """
343 test transition of powerpc to ports.ubuntu.com
344 """
345@@ -351,7 +365,9 @@
346 v = DistUpgradeViewNonInteractive()
347 d = DistUpgradeController(v, datadir=self.testdir)
348 d.openCache(lock=False)
349+ mock_sourcesListEntryDownloadable.return_value = True
350 res = d.updateSourcesList()
351+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
352 self.assertTrue(res)
353 # now test the result
354 self._verifySources("""
355@@ -362,7 +378,8 @@
356 """)
357 apt_pkg.config.set("APT::Architecture", arch)
358
359- def test_sparc_transition(self):
360+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
361+ def test_sparc_transition(self, mock_sourcesListEntryDownloadable):
362 """
363 test transition of sparc to ports.ubuntu.com
364 """
365@@ -377,7 +394,9 @@
366 d.fromDist = "gutsy"
367 d.toDist = "hardy"
368 d.openCache(lock=False)
369+ mock_sourcesListEntryDownloadable.return_value = True
370 res = d.updateSourcesList()
371+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
372 self.assertTrue(res)
373 # now test the result
374 self._verifySources("""
375@@ -480,7 +499,8 @@
376 deb http://archive.ubuntu.com/ubuntu precise-security main restricted universe multiverse
377 """)
378
379- def test_partner_update(self):
380+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
381+ def test_partner_update(self, mock_sourcesListEntryDownloadable):
382 """
383 test transition partner repository updates
384 """
385@@ -491,7 +511,9 @@
386 v = DistUpgradeViewNonInteractive()
387 d = DistUpgradeController(v, datadir=self.testdir)
388 d.openCache(lock=False)
389+ mock_sourcesListEntryDownloadable.return_value = True
390 res = d.updateSourcesList()
391+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
392 self.assertTrue(res)
393
394 # now test the result
395@@ -504,7 +526,8 @@
396 deb http://archive.canonical.com/ubuntu gutsy partner
397 """)
398
399- def test_private_ppa_transition(self):
400+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
401+ def test_private_ppa_transition(self, mock_sourcesListEntryDownloadable):
402 if "RELEASE_UPGRADER_ALLOW_THIRD_PARTY" in os.environ:
403 del os.environ["RELEASE_UPGRADER_ALLOW_THIRD_PARTY"]
404 shutil.copy(
405@@ -516,7 +539,9 @@
406 v = DistUpgradeViewNonInteractive()
407 d = DistUpgradeController(v, datadir=self.testdir)
408 d.openCache(lock=False)
409+ mock_sourcesListEntryDownloadable.return_value = True
410 res = d.updateSourcesList()
411+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
412 self.assertTrue(res)
413
414 # now test the result
415@@ -533,7 +558,8 @@
416 deb https://user:pass@private-ppa.launchpad.net/commercial-ppa-uploaders gutsy main
417 """)
418
419- def test_apt_cacher_and_apt_bittorent(self):
420+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
421+ def test_apt_cacher_and_apt_bittorent(self, mock_sourcesListEntryDownloadable):
422 """
423 test transition of apt-cacher/apt-torrent uris
424 """
425@@ -544,7 +570,9 @@
426 v = DistUpgradeViewNonInteractive()
427 d = DistUpgradeController(v, datadir=self.testdir)
428 d.openCache(lock=False)
429+ mock_sourcesListEntryDownloadable.return_value = True
430 res = d.updateSourcesList()
431+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
432 self.assertTrue(res)
433
434 # now test the result
435@@ -593,7 +621,8 @@
436 # deb http://ppa.launchpad.net/random-ppa quantal main # ppa of VĂ­ctor R. Ruiz (vrruiz) disabled on upgrade to gutsy
437 """)
438
439- def test_local_mirror(self):
440+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
441+ def test_local_mirror(self, mock_sourcesListEntryDownloadable):
442 """
443 test that a local mirror with official -backports works (LP:# 1067393)
444 """
445@@ -603,7 +632,9 @@
446 v = DistUpgradeViewNonInteractive()
447 d = DistUpgradeController(v, datadir=self.testdir)
448 d.openCache(lock=False)
449+ mock_sourcesListEntryDownloadable.return_value = True
450 res = d.updateSourcesList()
451+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
452 self.assertTrue(res)
453
454 # verify it
455@@ -614,7 +645,8 @@
456 deb http://archive.ubuntu.com/ubuntu gutsy-backports main restricted universe multiverse
457 """)
458
459- def test_disable_proposed(self):
460+ @mock.patch("DistUpgrade.DistUpgradeController.DistUpgradeController._sourcesListEntryDownloadable")
461+ def test_disable_proposed(self, mock_sourcesListEntryDownloadable):
462 """
463 Test that proposed is disabled when upgrading to a development
464 release.
465@@ -628,7 +660,9 @@
466 options.devel_release = True
467 d = DistUpgradeController(v, options, datadir=self.testdir)
468 d.openCache(lock=False)
469+ mock_sourcesListEntryDownloadable.return_value = True
470 res = d.updateSourcesList()
471+ self.assertTrue(mock_sourcesListEntryDownloadable.called)
472 self.assertTrue(res)
473
474 self._verifySources("""

Subscribers

People subscribed via source and target branches