Merge lp:~salgado/launchpad/bug-481375 into lp:launchpad

[Guilherme Salgado]

Fix bug 481375 by getting the real (i.e. uncompressed) size of gzipped log files

[Guilherme Salgado (salgado)]

Fix bug 481375 by getting the real (i.e. uncompressed) size of gzipped files

[Brad Crittenden (bac)]

review: Approve (code)

[Francis J. Lacoste (flacoste)]

review: Approve (release-critical)

=== modified file 'lib/lp/services/apachelogparser/base.py'
--- lib/lp/services/apachelogparser/base.py	2009-08-27 03:27:46 +0000
+++ lib/lp/services/apachelogparser/base.py	2009-11-17 18:45:29 +0000
@@ -33,11 +33,7 @@
     store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
     for file_name in file_names:
         file_path = os.path.join(root, file_name)
-        file_size = os.path.getsize(file_path)
-        if file_name.endswith('.gz'):
-            fd = gzip.open(file_path)
-        else:
-            fd = open(file_path)
+        fd, file_size = get_fd_and_file_size(file_path)
         first_line = unicode(fd.readline())
         parsed_file = store.find(ParsedApacheLog, first_line=first_line).one()
         position = 0
@@ -57,6 +53,29 @@
     return files_to_parse
 
 
+def get_fd_and_file_size(file_path):
+    """Return a file descriptor and the file size for the given file path.
+
+    The file descriptor will have the default mode ('r') and will be seeked to
+    the beginning.
+
+    The file size returned is that of the uncompressed file, in case the given
+    file_path points to a gzipped file.
+    """
+    if file_path.endswith('.gz'):
+        fd = gzip.open(file_path)
+        # There doesn't seem to be a better way of figuring out the
+        # uncompressed size of a file, so we'll read the whole file here.
+        file_size = len(fd.read())
+        # Seek back to the beginning of the file as if we had just opened
+        # it.
+        fd.seek(0)
+    else:
+        fd = open(file_path)
+        file_size = os.path.getsize(file_path)
+    return fd, file_size
+
+
 def parse_file(fd, start_position, logger, get_download_key):
     """Parse the given file starting on the given position.
 

=== modified file 'lib/lp/services/apachelogparser/tests/test_apachelogparser.py'
--- lib/lp/services/apachelogparser/tests/test_apachelogparser.py	2009-08-31 15:04:47 +0000
+++ lib/lp/services/apachelogparser/tests/test_apachelogparser.py	2009-11-17 18:45:29 +0000
@@ -17,7 +17,7 @@
 from canonical.launchpad.scripts.librarian_apache_log_parser import DBUSER
 from lp.services.apachelogparser.base import (
     create_or_update_parsedlog_entry, get_day, get_files_to_parse,
-    get_host_date_status_and_request, parse_file)
+    get_fd_and_file_size, get_host_date_status_and_request, parse_file)
 from lp.services.apachelogparser.model.parsedapachelog import ParsedApacheLog
 from lp.testing import TestCase
 
@@ -59,6 +59,30 @@
         self.assertEqual(get_day(date), datetime(2008, 6, 13))
 
 
+class Test_get_fd_and_file_size(TestCase):
+
+    def _ensureFileSizeIsCorrect(self, file_path):
+        """Ensure the file size returned is correct.
+
+        Also ensure that the file descriptors returned where seek()ed to the
+        very beginning.
+        """
+        fd, file_size = get_fd_and_file_size(file_path)
+        self.assertEqual(fd.tell(), 0)
+        self.assertEqual(len(fd.read()), file_size)
+
+    def test_regular_file(self):
+        file_path = os.path.join(
+            here, 'apache-log-files', 'librarian-oneline.log')
+        self._ensureFileSizeIsCorrect(file_path)
+
+    def test_gzip_file(self):
+        file_path = os.path.join(
+            here, 'apache-log-files',
+            'launchpadlibrarian.net.access-log.1.gz')
+        self._ensureFileSizeIsCorrect(file_path)
+
+
 def get_path_download_key(path):
     return path