Merge lp:~cjwatson/launchpad/more-buildd-manager-logging into lp:launchpad

Proposed by Colin Watson
Status: Merged
Merged at revision: 18998
Proposed branch: lp:~cjwatson/launchpad/more-buildd-manager-logging
Merge into: lp:launchpad
Diff against target: 65 lines (+17/-4)
2 files modified
lib/lp/buildmaster/interactor.py (+16/-3)
lib/lp/buildmaster/model/buildfarmjobbehaviour.py (+1/-1)
To merge this branch: bzr merge lp:~cjwatson/launchpad/more-buildd-manager-logging
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+369031@code.launchpad.net

Commit message

Add a bit more logging to buildd-manager when fetching files.

Description of the change

Logging something more specific than Twisted's "Stopping factory" entries makes it easier to see which files have been grabbed and which are still in progress.

I suspect this won't actually help us debug current odd network slowness very much, but it will make me feel better when reading logs. :-)

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/interactor.py'
2--- lib/lp/buildmaster/interactor.py 2018-02-08 09:39:46 +0000
3+++ lib/lp/buildmaster/interactor.py 2019-06-19 10:06:23 +0000
4@@ -238,13 +238,14 @@
5 """Get the URL for a file on the builder with a given SHA-1."""
6 return urlappend(self._file_cache_url, sha1).encode('utf8')
7
8- def getFile(self, sha_sum, file_to_write):
9+ def getFile(self, sha_sum, file_to_write, logger=None):
10 """Fetch a file from the builder.
11
12 :param sha_sum: The sha of the file (which is also its name on the
13 builder)
14 :param file_to_write: A file name or file-like object to write
15 the file to
16+ :param logger: An optional logger.
17 :return: A Deferred that calls back when the download is done, or
18 errback with the error string.
19 """
20@@ -256,19 +257,31 @@
21 response.deliverBody(FileWritingProtocol(finished, file_to_write))
22 return finished
23
24+ def log_success(result):
25+ logger.info("Grabbed %s" % file_url)
26+ return result
27+
28+ def log_failure(failure):
29+ logger.info("Failed to grab %s: %s\n%s" % (
30+ file_url, failure.getErrorMessage(), failure.getTraceback()))
31+ return failure
32+
33 d.addCallback(got_response)
34+ if logger is not None:
35+ d.addCallbacks(log_success, log_failure)
36 return d
37
38- def getFiles(self, files):
39+ def getFiles(self, files, logger=None):
40 """Fetch many files from the builder.
41
42 :param files: A sequence of pairs of the builder file name to
43 retrieve and the file name or file object to write the file to.
44+ :param logger: An optional logger.
45
46 :return: A DeferredList that calls back when the download is done.
47 """
48 dl = defer.gatherResults([
49- self.getFile(builder_file, local_file)
50+ self.getFile(builder_file, local_file, logger=logger)
51 for builder_file, local_file in files])
52 return dl
53
54
55=== modified file 'lib/lp/buildmaster/model/buildfarmjobbehaviour.py'
56--- lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2019-03-13 17:29:11 +0000
57+++ lib/lp/buildmaster/model/buildfarmjobbehaviour.py 2019-06-19 10:06:23 +0000
58@@ -349,7 +349,7 @@
59 raise BuildDaemonError(
60 "Build returned a file named %r." % filename)
61 filenames_to_download.append((sha1, out_file_name))
62- yield self._slave.getFiles(filenames_to_download)
63+ yield self._slave.getFiles(filenames_to_download, logger=logger)
64
65 transaction.commit()
66