Merge lp:~salgado/launchpad/nuke-debug-references into lp:launchpad

[Guilherme Salgado]

Remove debugReferencesLeak() and co. No longer needed as we'll be using meliae to get memory dumps which can later be analyzed: lp:~salgado/launchpad/use-meliae

[Francis J. Lacoste (flacoste)]

Nuke, nuke, nuke!

review: Approve

=== modified file 'lib/canonical/config/schema-lazr.conf'
--- lib/canonical/config/schema-lazr.conf	2010-03-10 01:52:56 +0000
+++ lib/canonical/config/schema-lazr.conf	2010-03-18 19:34:18 +0000
@@ -601,22 +601,6 @@
 # datatype: integer
 storm_cache_size: 500
 
-
-[debug]
-# Set this to true to compare memory and reference leaking
-# from one request to the next. Turning this to true without
-# setting the number of threads to 1 will give not-so-useful
-# results.
-references: False
-
-# File to keep the reference count into.
-references_scoreboard_file: /tmp/refcounts-scoreboard.txt
-
-# Log file where leaks will be reported. Actually, this log will contain
-# an entry for each request containing the time, the pageid and the increase
-# in refcounts and memory.
-references_leak_log: /tmp/references-leak.log
-
 [diff]
 # The maximum size in bytes to read from the librarian to make available in
 # the web UI.  512k == 524288 bytes.

=== modified file 'lib/canonical/launchpad/webapp/configure.zcml'
--- lib/canonical/launchpad/webapp/configure.zcml	2010-03-16 05:08:47 +0000
+++ lib/canonical/launchpad/webapp/configure.zcml	2010-03-18 19:34:18 +0000
@@ -753,12 +753,6 @@
         handler="canonical.launchpad.webapp.adapter.break_main_thread_db_access"
         />
 
-    <!-- Make sure that the config is sane for references debugging. -->
-    <subscriber
-        for="zope.app.appsetup.IProcessStartingEvent"
-        handler="canonical.launchpad.webapp.publication.debug_references_startup_check"
-        />
-
     <!-- Set the default timeout function. -->
     <subscriber
         for="zope.app.appsetup.IProcessStartingEvent"

=== modified file 'lib/canonical/launchpad/webapp/publication.py'
--- lib/canonical/launchpad/webapp/publication.py	2010-03-04 22:42:07 +0000
+++ lib/canonical/launchpad/webapp/publication.py	2010-03-18 19:34:18 +0000
@@ -7,7 +7,6 @@
     'LaunchpadBrowserPublication',
     ]
 
-import gc
 import os
 import re
 import sys
@@ -18,7 +17,6 @@
 
 from cProfile import Profile
 from datetime import datetime
-from time import strftime
 
 import tickcount
 import transaction
@@ -47,9 +45,7 @@
 import canonical.launchpad.webapp.adapter as da
 
 from canonical.config import config, dbconfig
-from canonical.mem import (
-    countsByType, deltaCounts, memory, mostRefs, printCounts, readCounts,
-    resident)
+from canonical.mem import memory, resident
 from canonical.launchpad.interfaces.launchpad import ILaunchpadCelebrities
 from canonical.launchpad.readonly import is_read_only
 from lp.registry.interfaces.person import (
@@ -608,9 +604,6 @@
 
         da.clear_request_started()
 
-        if config.debug.references:
-            self.debugReferencesLeak(request)
-
         # Maintain operational statistics.
         if getattr(request, '_wants_retry', False):
             OpStats.stats['retries'] += 1
@@ -719,101 +712,11 @@
                 vss_start, rss_start, vss_end, rss_end))
             log.close()
 
-    def debugReferencesLeak(self, request):
-        """See what kind of references are increasing.
-
-        This logs the current RSS and references count by types in a
-        scoreboard file. If that file exists, we compare the current stats
-        with the previous one and logs the increase along the current page id.
-
-        Note that this only provides reliable results when only one thread is
-        processing requests.
-        """
-        gc.collect()
-        current_rss = resident()
-        current_garbage_count = len(gc.garbage)
-        # Convert type to string, because that's what we get when reading
-        # the old scoreboard.
-        current_refs = [
-            (count, str(ref_type)) for count, ref_type in mostRefs(n=0)]
-        # Add G as prefix to types on the garbage list.
-        current_garbage = [
-            (count, 'G%s' % str(ref_type))
-            for count, ref_type in countsByType(gc.garbage, n=0)]
-        scoreboard_path = config.debug.references_scoreboard_file
-
-        # Read in previous scoreboard if it exists.
-        if os.path.exists(scoreboard_path):
-            scoreboard = open(scoreboard_path, 'r')
-            try:
-                stats = scoreboard.readline().split()
-                prev_rss = int(stats[0].strip())
-                prev_garbage_count = int(stats[1].strip())
-                prev_refs = readCounts(scoreboard, '=== GARBAGE ===\n')
-                prev_garbage = readCounts(scoreboard)
-            finally:
-                scoreboard.close()
-            mem_leak = current_rss - prev_rss
-            garbage_leak = current_garbage_count - prev_garbage_count
-            delta_refs = list(deltaCounts(prev_refs, current_refs))
-            delta_refs.extend(deltaCounts(prev_garbage, current_garbage))
-            self.logReferencesLeak(request, mem_leak, delta_refs)
-
-        # Save the current scoreboard.
-        scoreboard = open(scoreboard_path, 'w')
-        try:
-            scoreboard.write("%d %d\n" % (current_rss, current_garbage_count))
-            printCounts(current_refs, scoreboard)
-            scoreboard.write('=== GARBAGE ===\n')
-            printCounts(current_garbage, scoreboard)
-        finally:
-            scoreboard.close()
-
-    def logReferencesLeak(self, request, mem_leak, delta_refs):
-        """Log the time, pageid, increase in RSS and increase in references.
-        """
-        log = open(config.debug.references_leak_log, 'a')
-        try:
-            pageid = request._orig_env.get('launchpad.pageid', 'Unknown')
-            # It can happen that the pageid is ''?!?
-            if pageid == '':
-                pageid = 'Unknown'
-            leak_in_mb = float(mem_leak) / (1024*1024)
-            formatted_delta = "; ".join(
-                "%s=%d" % (ref_type, count)
-                for count, ref_type in delta_refs)
-            log.write('%s %s %.2fMb %s\n' % (
-                strftime('%Y-%m-%d:%H:%M:%S'),
-                pageid,
-                leak_in_mb,
-                formatted_delta))
-        finally:
-            log.close()
-
 
 class InvalidThreadsConfiguration(Exception):
     """Exception thrown when the number of threads isn't set correctly."""
 
 
-def debug_references_startup_check(event):
-    """Event handler for IProcessStartingEvent.
-
-    If debug/references is set to True, we make sure that the number of
-    threads is configured to 1. We also delete any previous scoreboard file.
-    """
-    if not config.debug.references:
-        return
-
-    if config.threads != 1:
-        raise InvalidThreadsConfiguration(
-            "Number of threads should be one when debugging references.")
-
-    # Remove any previous scoreboard, the content is meaningless once
-    # the server is restarted.
-    if os.path.exists(config.debug.references_scoreboard_file):
-        os.remove(config.debug.references_scoreboard_file)
-
-
 class DefaultPrimaryContext:
     """The default primary context is the context."""
 

=== removed file 'lib/lp/bugs/doc/memory-debug.txt'
--- lib/lp/bugs/doc/memory-debug.txt	2009-06-12 16:36:02 +0000
+++ lib/lp/bugs/doc/memory-debug.txt	1970-01-01 00:00:00 +0000
@@ -1,185 +0,0 @@
-= Memory debugging =
-
-Application server can leak references, and this is hard to track down.
-The LaunchpadBrowserPublication contains special purpose memory
-debugging helper to help diagnose such problems.
-
-The memory leaking diagnosis is invoked at endRequest() only when the
-debug.references variable is true. It is turned off by default.
-
-    >>> from canonical.config import config
-    >>> config.debug.references
-    False
-
-Setting it to true, will save in endRequest() a dump of the current
-memory usage and counts in the file specified in
-config.debug.references_scoreboard_file.  (We also set the
-config.debug.references_leak_log file which will contain the
-differential diagnostic.)
-
-    >>> import os
-    >>> import tempfile
-    >>> from textwrap import dedent
-
-    >>> tempdir = tempfile.mkdtemp(prefix='memory-debug')
-    >>> scoreboard_path = os.path.join(tempdir, 'scoreboard.txt')
-    >>> references_log_path = os.path.join(tempdir, 'references-leak.log')
-
-    >>> config.push('debug-references', dedent("""\
-    ...     [debug]
-    ...     references: True
-    ...     references_scoreboard_file: %s
-    ...     references_leak_log: %s
-    ...     """ % (scoreboard_path, references_log_path)))
-
-There is a debug_references_startup_check() that is called when the
-IProcessStartingEvent is fired. It raises an exception if the setting
-is on, but the number of threads is not 1.
-
-    >>> from canonical.launchpad.webapp.publication import (
-    ...     debug_references_startup_check)
-    >>> orig_threads = config.threads
-    >>> orig_threads > 1
-    True
-    >>> from zope.app.appsetup import ProcessStarting
-
-    # We don't use notify here, because we don't want to invoke the
-    # other subscribers.
-    >>> debug_references_startup_check(ProcessStarting())
-    Traceback (most recent call last):
-      ...
-    InvalidThreadsConfiguration: ...
-
-This handler also deletes any previous scoreboard file that might be
-present (left by a previous run.)
-
-    >>> config.threads = 1
-    >>> open(scoreboard_path, 'w').close()
-    >>> debug_references_startup_check(ProcessStarting())
-    >>> os.path.exists(scoreboard_path)
-    False
-
-The memory monitoring code also watches the garbage list. Make sure that
-there is some available:
-
-    >>> class garbage(object):
-    ...     """Uncollectable objects involved in a cycle."""
-    ...     def __init__(self):
-    ...         self.cycle = self
-    ...     def __del__(self):
-    ...         """The presence of this method makes us garbage."""
-    >>> trash = garbage()
-    >>> del trash
-
-So the first time endRequest() is called, the references count is dumped
-to the scoreboard file.
-
-    >>> from canonical.launchpad.webapp.servers import LaunchpadTestRequest
-    >>> from canonical.launchpad.webapp.adapter import set_request_started
-    >>> def create_request(pageid='View:+template'):
-    ...     """Returns a LaunchpadTestRequest with appropriate fixture.
-    ...
-    ...     Sets some attributes on the request expected by the
-    ...     publication, and that are usually set by previous publication
-    ...     steps.
-    ...     """
-    ...     request = LaunchpadTestRequest()
-    ...     # Adds some expected requests variables by endRequest().
-    ...     request._orig_env['launchpad.pageid'] = pageid
-    ...     request.oopsid = None
-    ...     # Prevents warnings when endRequest() is called.
-    ...     set_request_started()
-    ...     return request
-
-    >>> from canonical.launchpad.webapp.publication import (
-    ...     LaunchpadBrowserPublication)
-    >>> publication = LaunchpadBrowserPublication(None)
-    >>> publication.endRequest(create_request(), None)
-
-The first line of the scoreboard is the number of bytes of RSS used and
-the number of items on the garbage list.
-
-    >>> import re
-    >>> scoreboard = open(scoreboard_path, 'r')
-    >>> lines = scoreboard.readlines()
-    >>> re.match('\d+ \d+$', lines[0].strip()) is not None
-    True
-
-Lines up to the '=== GARBAGE ===' marker contains a count and a type
-name. This is the number of references for that type.
-
-    >>> re.match('\d+ <type.*>$', lines[1].strip()) is not None
-    True
-
-After the '=== GARBAGE ===' marker, again we find a count and type name
-on each subsequent lines.
-
-    >>> garbage_marker = lines.index('=== GARBAGE ===\n')
-    >>> garbage_marker > 1
-    True
-    >>> print "\n".join(lines[garbage_marker:])
-    === GARBAGE ===
-    ...1 G<class 'garbage'>...
-
-On the second request, the scoreboard will be compared to the previous
-one and differences will be reported in the references leak log file.
-The format is:
-
-    date pageid increase_in_RSS increase_in_garbage type=increase; \
-    type=increase; G<type>=increase; ...
-
-Type names prefixed by G means that a number of instances of that type
-was added to the garbage list.
-
-Example:
-
-    2008-04-03:18:15:45 Question:+index 0.57Mb <type 'tuple'>=6419;
-        <type 'dict'>=2744; <type 'list'>=1942; <type 'weakref'>=544;
-        G<class 'garbage'>=1
-
-    >>> os.path.exists(references_log_path)
-    False
-
-(Add some garbage)
-
-    >>> trash = garbage()
-    >>> del trash
-
-    >>> publication.endRequest(create_request(), None)
-    >>> references_log = open(references_log_path, 'r')
-    >>> references_log.read()
-    "... View:+template ...Mb... <type 'list'>=...;
-    ...G<class 'garbage'>=1...\n"
-
-Of course when the tool is off, none of the above processing takes
-place:
-
-    >>> config.push('turn-off-references-debug', dedent("""\
-    ...     [debug]
-    ...     references: False"""))
-    >>> scoreboard.close()
-    >>> references_log.close()
-    >>> os.remove(scoreboard_path)
-    >>> os.remove(references_log_path)
-
-    >>> publication.endRequest(create_request(), None)
-    >>> os.path.exists(config.debug.references_scoreboard_file)
-    False
-    >>> os.path.exists(config.debug.references_leak_log)
-    False
-
-(Clean up)
-
-    >>> ignored = config.pop('debug-references')
-    >>> config.threads = orig_threads
-
-    >>> import gc
-    >>> new_garbage = []
-    >>> for g in gc.garbage:
-    ...     # We only know how to clean our own garbage.
-    ...     if isinstance(g, garbage):
-    ...         del g.cycle
-    ...     else:
-    ...         new_garbage.append(g)
-    >>> gc.garbage[:] = new_garbage
-