Merge lp:~salgado/launchpad/nuke-debug-references into lp:launchpad
- nuke-debug-references
- Merge into devel
Proposed by
Guilherme Salgado
Status: | Merged |
---|---|
Merged at revision: | not available |
Proposed branch: | lp:~salgado/launchpad/nuke-debug-references |
Merge into: | lp:launchpad |
Diff against target: |
376 lines (+1/-305) 4 files modified
lib/canonical/config/schema-lazr.conf (+0/-16) lib/canonical/launchpad/webapp/configure.zcml (+0/-6) lib/canonical/launchpad/webapp/publication.py (+1/-98) lib/lp/bugs/doc/memory-debug.txt (+0/-185) |
To merge this branch: | bzr merge lp:~salgado/launchpad/nuke-debug-references |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Francis J. Lacoste (community) | Approve | ||
Review via email: mp+21673@code.launchpad.net |
Commit message
Description of the change
Remove debugReferences
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/canonical/config/schema-lazr.conf' | |||
2 | --- lib/canonical/config/schema-lazr.conf 2010-03-10 01:52:56 +0000 | |||
3 | +++ lib/canonical/config/schema-lazr.conf 2010-03-18 19:34:18 +0000 | |||
4 | @@ -601,22 +601,6 @@ | |||
5 | 601 | # datatype: integer | 601 | # datatype: integer |
6 | 602 | storm_cache_size: 500 | 602 | storm_cache_size: 500 |
7 | 603 | 603 | ||
8 | 604 | |||
9 | 605 | [debug] | ||
10 | 606 | # Set this to true to compare memory and reference leaking | ||
11 | 607 | # from one request to the next. Turning this to true without | ||
12 | 608 | # setting the number of threads to 1 will give not-so-useful | ||
13 | 609 | # results. | ||
14 | 610 | references: False | ||
15 | 611 | |||
16 | 612 | # File to keep the reference count into. | ||
17 | 613 | references_scoreboard_file: /tmp/refcounts-scoreboard.txt | ||
18 | 614 | |||
19 | 615 | # Log file where leaks will be reported. Actually, this log will contain | ||
20 | 616 | # an entry for each request containing the time, the pageid and the increase | ||
21 | 617 | # in refcounts and memory. | ||
22 | 618 | references_leak_log: /tmp/references-leak.log | ||
23 | 619 | |||
24 | 620 | [diff] | 604 | [diff] |
25 | 621 | # The maximum size in bytes to read from the librarian to make available in | 605 | # The maximum size in bytes to read from the librarian to make available in |
26 | 622 | # the web UI. 512k == 524288 bytes. | 606 | # the web UI. 512k == 524288 bytes. |
27 | 623 | 607 | ||
28 | === modified file 'lib/canonical/launchpad/webapp/configure.zcml' | |||
29 | --- lib/canonical/launchpad/webapp/configure.zcml 2010-03-16 05:08:47 +0000 | |||
30 | +++ lib/canonical/launchpad/webapp/configure.zcml 2010-03-18 19:34:18 +0000 | |||
31 | @@ -753,12 +753,6 @@ | |||
32 | 753 | handler="canonical.launchpad.webapp.adapter.break_main_thread_db_access" | 753 | handler="canonical.launchpad.webapp.adapter.break_main_thread_db_access" |
33 | 754 | /> | 754 | /> |
34 | 755 | 755 | ||
35 | 756 | <!-- Make sure that the config is sane for references debugging. --> | ||
36 | 757 | <subscriber | ||
37 | 758 | for="zope.app.appsetup.IProcessStartingEvent" | ||
38 | 759 | handler="canonical.launchpad.webapp.publication.debug_references_startup_check" | ||
39 | 760 | /> | ||
40 | 761 | |||
41 | 762 | <!-- Set the default timeout function. --> | 756 | <!-- Set the default timeout function. --> |
42 | 763 | <subscriber | 757 | <subscriber |
43 | 764 | for="zope.app.appsetup.IProcessStartingEvent" | 758 | for="zope.app.appsetup.IProcessStartingEvent" |
44 | 765 | 759 | ||
45 | === modified file 'lib/canonical/launchpad/webapp/publication.py' | |||
46 | --- lib/canonical/launchpad/webapp/publication.py 2010-03-04 22:42:07 +0000 | |||
47 | +++ lib/canonical/launchpad/webapp/publication.py 2010-03-18 19:34:18 +0000 | |||
48 | @@ -7,7 +7,6 @@ | |||
49 | 7 | 'LaunchpadBrowserPublication', | 7 | 'LaunchpadBrowserPublication', |
50 | 8 | ] | 8 | ] |
51 | 9 | 9 | ||
52 | 10 | import gc | ||
53 | 11 | import os | 10 | import os |
54 | 12 | import re | 11 | import re |
55 | 13 | import sys | 12 | import sys |
56 | @@ -18,7 +17,6 @@ | |||
57 | 18 | 17 | ||
58 | 19 | from cProfile import Profile | 18 | from cProfile import Profile |
59 | 20 | from datetime import datetime | 19 | from datetime import datetime |
60 | 21 | from time import strftime | ||
61 | 22 | 20 | ||
62 | 23 | import tickcount | 21 | import tickcount |
63 | 24 | import transaction | 22 | import transaction |
64 | @@ -47,9 +45,7 @@ | |||
65 | 47 | import canonical.launchpad.webapp.adapter as da | 45 | import canonical.launchpad.webapp.adapter as da |
66 | 48 | 46 | ||
67 | 49 | from canonical.config import config, dbconfig | 47 | from canonical.config import config, dbconfig |
71 | 50 | from canonical.mem import ( | 48 | from canonical.mem import memory, resident |
69 | 51 | countsByType, deltaCounts, memory, mostRefs, printCounts, readCounts, | ||
70 | 52 | resident) | ||
72 | 53 | from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities | 49 | from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities |
73 | 54 | from canonical.launchpad.readonly import is_read_only | 50 | from canonical.launchpad.readonly import is_read_only |
74 | 55 | from lp.registry.interfaces.person import ( | 51 | from lp.registry.interfaces.person import ( |
75 | @@ -608,9 +604,6 @@ | |||
76 | 608 | 604 | ||
77 | 609 | da.clear_request_started() | 605 | da.clear_request_started() |
78 | 610 | 606 | ||
79 | 611 | if config.debug.references: | ||
80 | 612 | self.debugReferencesLeak(request) | ||
81 | 613 | |||
82 | 614 | # Maintain operational statistics. | 607 | # Maintain operational statistics. |
83 | 615 | if getattr(request, '_wants_retry', False): | 608 | if getattr(request, '_wants_retry', False): |
84 | 616 | OpStats.stats['retries'] += 1 | 609 | OpStats.stats['retries'] += 1 |
85 | @@ -719,101 +712,11 @@ | |||
86 | 719 | vss_start, rss_start, vss_end, rss_end)) | 712 | vss_start, rss_start, vss_end, rss_end)) |
87 | 720 | log.close() | 713 | log.close() |
88 | 721 | 714 | ||
89 | 722 | def debugReferencesLeak(self, request): | ||
90 | 723 | """See what kind of references are increasing. | ||
91 | 724 | |||
92 | 725 | This logs the current RSS and references count by types in a | ||
93 | 726 | scoreboard file. If that file exists, we compare the current stats | ||
94 | 727 | with the previous one and logs the increase along the current page id. | ||
95 | 728 | |||
96 | 729 | Note that this only provides reliable results when only one thread is | ||
97 | 730 | processing requests. | ||
98 | 731 | """ | ||
99 | 732 | gc.collect() | ||
100 | 733 | current_rss = resident() | ||
101 | 734 | current_garbage_count = len(gc.garbage) | ||
102 | 735 | # Convert type to string, because that's what we get when reading | ||
103 | 736 | # the old scoreboard. | ||
104 | 737 | current_refs = [ | ||
105 | 738 | (count, str(ref_type)) for count, ref_type in mostRefs(n=0)] | ||
106 | 739 | # Add G as prefix to types on the garbage list. | ||
107 | 740 | current_garbage = [ | ||
108 | 741 | (count, 'G%s' % str(ref_type)) | ||
109 | 742 | for count, ref_type in countsByType(gc.garbage, n=0)] | ||
110 | 743 | scoreboard_path = config.debug.references_scoreboard_file | ||
111 | 744 | |||
112 | 745 | # Read in previous scoreboard if it exists. | ||
113 | 746 | if os.path.exists(scoreboard_path): | ||
114 | 747 | scoreboard = open(scoreboard_path, 'r') | ||
115 | 748 | try: | ||
116 | 749 | stats = scoreboard.readline().split() | ||
117 | 750 | prev_rss = int(stats[0].strip()) | ||
118 | 751 | prev_garbage_count = int(stats[1].strip()) | ||
119 | 752 | prev_refs = readCounts(scoreboard, '=== GARBAGE ===\n') | ||
120 | 753 | prev_garbage = readCounts(scoreboard) | ||
121 | 754 | finally: | ||
122 | 755 | scoreboard.close() | ||
123 | 756 | mem_leak = current_rss - prev_rss | ||
124 | 757 | garbage_leak = current_garbage_count - prev_garbage_count | ||
125 | 758 | delta_refs = list(deltaCounts(prev_refs, current_refs)) | ||
126 | 759 | delta_refs.extend(deltaCounts(prev_garbage, current_garbage)) | ||
127 | 760 | self.logReferencesLeak(request, mem_leak, delta_refs) | ||
128 | 761 | |||
129 | 762 | # Save the current scoreboard. | ||
130 | 763 | scoreboard = open(scoreboard_path, 'w') | ||
131 | 764 | try: | ||
132 | 765 | scoreboard.write("%d %d\n" % (current_rss, current_garbage_count)) | ||
133 | 766 | printCounts(current_refs, scoreboard) | ||
134 | 767 | scoreboard.write('=== GARBAGE ===\n') | ||
135 | 768 | printCounts(current_garbage, scoreboard) | ||
136 | 769 | finally: | ||
137 | 770 | scoreboard.close() | ||
138 | 771 | |||
139 | 772 | def logReferencesLeak(self, request, mem_leak, delta_refs): | ||
140 | 773 | """Log the time, pageid, increase in RSS and increase in references. | ||
141 | 774 | """ | ||
142 | 775 | log = open(config.debug.references_leak_log, 'a') | ||
143 | 776 | try: | ||
144 | 777 | pageid = request._orig_env.get('launchpad.pageid', 'Unknown') | ||
145 | 778 | # It can happen that the pageid is ''?!? | ||
146 | 779 | if pageid == '': | ||
147 | 780 | pageid = 'Unknown' | ||
148 | 781 | leak_in_mb = float(mem_leak) / (1024*1024) | ||
149 | 782 | formatted_delta = "; ".join( | ||
150 | 783 | "%s=%d" % (ref_type, count) | ||
151 | 784 | for count, ref_type in delta_refs) | ||
152 | 785 | log.write('%s %s %.2fMb %s\n' % ( | ||
153 | 786 | strftime('%Y-%m-%d:%H:%M:%S'), | ||
154 | 787 | pageid, | ||
155 | 788 | leak_in_mb, | ||
156 | 789 | formatted_delta)) | ||
157 | 790 | finally: | ||
158 | 791 | log.close() | ||
159 | 792 | |||
160 | 793 | 715 | ||
161 | 794 | class InvalidThreadsConfiguration(Exception): | 716 | class InvalidThreadsConfiguration(Exception): |
162 | 795 | """Exception thrown when the number of threads isn't set correctly.""" | 717 | """Exception thrown when the number of threads isn't set correctly.""" |
163 | 796 | 718 | ||
164 | 797 | 719 | ||
165 | 798 | def debug_references_startup_check(event): | ||
166 | 799 | """Event handler for IProcessStartingEvent. | ||
167 | 800 | |||
168 | 801 | If debug/references is set to True, we make sure that the number of | ||
169 | 802 | threads is configured to 1. We also delete any previous scoreboard file. | ||
170 | 803 | """ | ||
171 | 804 | if not config.debug.references: | ||
172 | 805 | return | ||
173 | 806 | |||
174 | 807 | if config.threads != 1: | ||
175 | 808 | raise InvalidThreadsConfiguration( | ||
176 | 809 | "Number of threads should be one when debugging references.") | ||
177 | 810 | |||
178 | 811 | # Remove any previous scoreboard, the content is meaningless once | ||
179 | 812 | # the server is restarted. | ||
180 | 813 | if os.path.exists(config.debug.references_scoreboard_file): | ||
181 | 814 | os.remove(config.debug.references_scoreboard_file) | ||
182 | 815 | |||
183 | 816 | |||
184 | 817 | class DefaultPrimaryContext: | 720 | class DefaultPrimaryContext: |
185 | 818 | """The default primary context is the context.""" | 721 | """The default primary context is the context.""" |
186 | 819 | 722 | ||
187 | 820 | 723 | ||
188 | === removed file 'lib/lp/bugs/doc/memory-debug.txt' | |||
189 | --- lib/lp/bugs/doc/memory-debug.txt 2009-06-12 16:36:02 +0000 | |||
190 | +++ lib/lp/bugs/doc/memory-debug.txt 1970-01-01 00:00:00 +0000 | |||
191 | @@ -1,185 +0,0 @@ | |||
192 | 1 | = Memory debugging = | ||
193 | 2 | |||
194 | 3 | Application server can leak references, and this is hard to track down. | ||
195 | 4 | The LaunchpadBrowserPublication contains special purpose memory | ||
196 | 5 | debugging helper to help diagnose such problems. | ||
197 | 6 | |||
198 | 7 | The memory leaking diagnosis is invoked at endRequest() only when the | ||
199 | 8 | debug.references variable is true. It is turned off by default. | ||
200 | 9 | |||
201 | 10 | >>> from canonical.config import config | ||
202 | 11 | >>> config.debug.references | ||
203 | 12 | False | ||
204 | 13 | |||
205 | 14 | Setting it to true, will save in endRequest() a dump of the current | ||
206 | 15 | memory usage and counts in the file specified in | ||
207 | 16 | config.debug.references_scoreboard_file. (We also set the | ||
208 | 17 | config.debug.references_leak_log file which will contain the | ||
209 | 18 | differential diagnostic.) | ||
210 | 19 | |||
211 | 20 | >>> import os | ||
212 | 21 | >>> import tempfile | ||
213 | 22 | >>> from textwrap import dedent | ||
214 | 23 | |||
215 | 24 | >>> tempdir = tempfile.mkdtemp(prefix='memory-debug') | ||
216 | 25 | >>> scoreboard_path = os.path.join(tempdir, 'scoreboard.txt') | ||
217 | 26 | >>> references_log_path = os.path.join(tempdir, 'references-leak.log') | ||
218 | 27 | |||
219 | 28 | >>> config.push('debug-references', dedent("""\ | ||
220 | 29 | ... [debug] | ||
221 | 30 | ... references: True | ||
222 | 31 | ... references_scoreboard_file: %s | ||
223 | 32 | ... references_leak_log: %s | ||
224 | 33 | ... """ % (scoreboard_path, references_log_path))) | ||
225 | 34 | |||
226 | 35 | There is a debug_references_startup_check() that is called when the | ||
227 | 36 | IProcessStartingEvent is fired. It raises an exception if the setting | ||
228 | 37 | is on, but the number of threads is not 1. | ||
229 | 38 | |||
230 | 39 | >>> from canonical.launchpad.webapp.publication import ( | ||
231 | 40 | ... debug_references_startup_check) | ||
232 | 41 | >>> orig_threads = config.threads | ||
233 | 42 | >>> orig_threads > 1 | ||
234 | 43 | True | ||
235 | 44 | >>> from zope.app.appsetup import ProcessStarting | ||
236 | 45 | |||
237 | 46 | # We don't use notify here, because we don't want to invoke the | ||
238 | 47 | # other subscribers. | ||
239 | 48 | >>> debug_references_startup_check(ProcessStarting()) | ||
240 | 49 | Traceback (most recent call last): | ||
241 | 50 | ... | ||
242 | 51 | InvalidThreadsConfiguration: ... | ||
243 | 52 | |||
244 | 53 | This handler also deletes any previous scoreboard file that might be | ||
245 | 54 | present (left by a previous run.) | ||
246 | 55 | |||
247 | 56 | >>> config.threads = 1 | ||
248 | 57 | >>> open(scoreboard_path, 'w').close() | ||
249 | 58 | >>> debug_references_startup_check(ProcessStarting()) | ||
250 | 59 | >>> os.path.exists(scoreboard_path) | ||
251 | 60 | False | ||
252 | 61 | |||
253 | 62 | The memory monitoring code also watches the garbage list. Make sure that | ||
254 | 63 | there is some available: | ||
255 | 64 | |||
256 | 65 | >>> class garbage(object): | ||
257 | 66 | ... """Uncollectable objects involved in a cycle.""" | ||
258 | 67 | ... def __init__(self): | ||
259 | 68 | ... self.cycle = self | ||
260 | 69 | ... def __del__(self): | ||
261 | 70 | ... """The presence of this method makes us garbage.""" | ||
262 | 71 | >>> trash = garbage() | ||
263 | 72 | >>> del trash | ||
264 | 73 | |||
265 | 74 | So the first time endRequest() is called, the references count is dumped | ||
266 | 75 | to the scoreboard file. | ||
267 | 76 | |||
268 | 77 | >>> from canonical.launchpad.webapp.servers import LaunchpadTestRequest | ||
269 | 78 | >>> from canonical.launchpad.webapp.adapter import set_request_started | ||
270 | 79 | >>> def create_request(pageid='View:+template'): | ||
271 | 80 | ... """Returns a LaunchpadTestRequest with appropriate fixture. | ||
272 | 81 | ... | ||
273 | 82 | ... Sets some attributes on the request expected by the | ||
274 | 83 | ... publication, and that are usually set by previous publication | ||
275 | 84 | ... steps. | ||
276 | 85 | ... """ | ||
277 | 86 | ... request = LaunchpadTestRequest() | ||
278 | 87 | ... # Adds some expected requests variables by endRequest(). | ||
279 | 88 | ... request._orig_env['launchpad.pageid'] = pageid | ||
280 | 89 | ... request.oopsid = None | ||
281 | 90 | ... # Prevents warnings when endRequest() is called. | ||
282 | 91 | ... set_request_started() | ||
283 | 92 | ... return request | ||
284 | 93 | |||
285 | 94 | >>> from canonical.launchpad.webapp.publication import ( | ||
286 | 95 | ... LaunchpadBrowserPublication) | ||
287 | 96 | >>> publication = LaunchpadBrowserPublication(None) | ||
288 | 97 | >>> publication.endRequest(create_request(), None) | ||
289 | 98 | |||
290 | 99 | The first line of the scoreboard is the number of bytes of RSS used and | ||
291 | 100 | the number of items on the garbage list. | ||
292 | 101 | |||
293 | 102 | >>> import re | ||
294 | 103 | >>> scoreboard = open(scoreboard_path, 'r') | ||
295 | 104 | >>> lines = scoreboard.readlines() | ||
296 | 105 | >>> re.match('\d+ \d+$', lines[0].strip()) is not None | ||
297 | 106 | True | ||
298 | 107 | |||
299 | 108 | Lines up to the '=== GARBAGE ===' marker contains a count and a type | ||
300 | 109 | name. This is the number of references for that type. | ||
301 | 110 | |||
302 | 111 | >>> re.match('\d+ <type.*>$', lines[1].strip()) is not None | ||
303 | 112 | True | ||
304 | 113 | |||
305 | 114 | After the '=== GARBAGE ===' marker, again we find a count and type name | ||
306 | 115 | on each subsequent lines. | ||
307 | 116 | |||
308 | 117 | >>> garbage_marker = lines.index('=== GARBAGE ===\n') | ||
309 | 118 | >>> garbage_marker > 1 | ||
310 | 119 | True | ||
311 | 120 | >>> print "\n".join(lines[garbage_marker:]) | ||
312 | 121 | === GARBAGE === | ||
313 | 122 | ...1 G<class 'garbage'>... | ||
314 | 123 | |||
315 | 124 | On the second request, the scoreboard will be compared to the previous | ||
316 | 125 | one and differences will be reported in the references leak log file. | ||
317 | 126 | The format is: | ||
318 | 127 | |||
319 | 128 | date pageid increase_in_RSS increase_in_garbage type=increase; \ | ||
320 | 129 | type=increase; G<type>=increase; ... | ||
321 | 130 | |||
322 | 131 | Type names prefixed by G means that a number of instances of that type | ||
323 | 132 | was added to the garbage list. | ||
324 | 133 | |||
325 | 134 | Example: | ||
326 | 135 | |||
327 | 136 | 2008-04-03:18:15:45 Question:+index 0.57Mb <type 'tuple'>=6419; | ||
328 | 137 | <type 'dict'>=2744; <type 'list'>=1942; <type 'weakref'>=544; | ||
329 | 138 | G<class 'garbage'>=1 | ||
330 | 139 | |||
331 | 140 | >>> os.path.exists(references_log_path) | ||
332 | 141 | False | ||
333 | 142 | |||
334 | 143 | (Add some garbage) | ||
335 | 144 | |||
336 | 145 | >>> trash = garbage() | ||
337 | 146 | >>> del trash | ||
338 | 147 | |||
339 | 148 | >>> publication.endRequest(create_request(), None) | ||
340 | 149 | >>> references_log = open(references_log_path, 'r') | ||
341 | 150 | >>> references_log.read() | ||
342 | 151 | "... View:+template ...Mb... <type 'list'>=...; | ||
343 | 152 | ...G<class 'garbage'>=1...\n" | ||
344 | 153 | |||
345 | 154 | Of course when the tool is off, none of the above processing takes | ||
346 | 155 | place: | ||
347 | 156 | |||
348 | 157 | >>> config.push('turn-off-references-debug', dedent("""\ | ||
349 | 158 | ... [debug] | ||
350 | 159 | ... references: False""")) | ||
351 | 160 | >>> scoreboard.close() | ||
352 | 161 | >>> references_log.close() | ||
353 | 162 | >>> os.remove(scoreboard_path) | ||
354 | 163 | >>> os.remove(references_log_path) | ||
355 | 164 | |||
356 | 165 | >>> publication.endRequest(create_request(), None) | ||
357 | 166 | >>> os.path.exists(config.debug.references_scoreboard_file) | ||
358 | 167 | False | ||
359 | 168 | >>> os.path.exists(config.debug.references_leak_log) | ||
360 | 169 | False | ||
361 | 170 | |||
362 | 171 | (Clean up) | ||
363 | 172 | |||
364 | 173 | >>> ignored = config.pop('debug-references') | ||
365 | 174 | >>> config.threads = orig_threads | ||
366 | 175 | |||
367 | 176 | >>> import gc | ||
368 | 177 | >>> new_garbage = [] | ||
369 | 178 | >>> for g in gc.garbage: | ||
370 | 179 | ... # We only know how to clean our own garbage. | ||
371 | 180 | ... if isinstance(g, garbage): | ||
372 | 181 | ... del g.cycle | ||
373 | 182 | ... else: | ||
374 | 183 | ... new_garbage.append(g) | ||
375 | 184 | >>> gc.garbage[:] = new_garbage | ||
376 | 185 |
Nuke, nuke, nuke!