Merge lp:~notmyname/swift/stats_log_lines into lp:~hudson-openstack/swift/trunk

[John Dickinson]

Stats system access log processor now allows for extra log fields to be at the end of the access log lines.

This allows the proxy server to freely append logged fields to log messages without breaking the stats system.

[gholt (gholt)]

review: Approve

=== modified file 'swift/stats/access_processor.py'
--- swift/stats/access_processor.py	2011-01-14 08:45:39 +0000
+++ swift/stats/access_processor.py	2011-01-17 18:38:51 +0000
@@ -40,7 +40,7 @@
         '''given a raw access log line, return a dict of the good parts'''
         d = {}
         try:
-            (_,
+            (unused,
             server,
             client_ip,
             lb_ip,
@@ -57,7 +57,8 @@
             etag,
             trans_id,
             headers,
-            processing_time) = (unquote(x) for x in raw_log[16:].split(' '))
+            processing_time) = (unquote(x) for x in
+                                raw_log[16:].split(' ')[:18])
         except ValueError:
             self.logger.debug(_('Bad line data: %s') % repr(raw_log))
             return {}

=== modified file 'test/unit/stats/test_access_processor.py'
--- test/unit/stats/test_access_processor.py	2011-01-04 23:34:43 +0000
+++ test/unit/stats/test_access_processor.py	2011-01-17 18:38:51 +0000
@@ -21,8 +21,49 @@
 
 class TestAccessProcessor(unittest.TestCase):
 
-    def test_placeholder(self):
-        pass
+    def test_log_line_parser_field_count(self):
+        p = access_processor.AccessLogProcessor({})
+        # too few fields
+        log_line = [str(x) for x in range(17)]
+        log_line[1] = 'proxy-server'
+        log_line[4] = '1/Jan/3/4/5/6'
+        log_line[6] = '/v1/a/c/o'
+        log_line = 'x'*16 + ' '.join(log_line)
+        res = p.log_line_parser(log_line)
+        expected = {}
+        self.assertEquals(res, expected)
+        # right amount of fields
+        log_line = [str(x) for x in range(18)]
+        log_line[1] = 'proxy-server'
+        log_line[4] = '1/Jan/3/4/5/6'
+        log_line[6] = '/v1/a/c/o'
+        log_line = 'x'*16 + ' '.join(log_line)
+        res = p.log_line_parser(log_line)
+        expected = {'code': 8, 'processing_time': '17', 'auth_token': '11',
+                    'month': '01', 'second': '6', 'year': '3', 'tz': '+0000',
+                    'http_version': '7', 'object_name': 'o', 'etag': '14',
+                    'method': '5', 'trans_id': '15', 'client_ip': '2',
+                    'bytes_out': 13, 'container_name': 'c', 'day': '1',
+                    'minute': '5', 'account': 'a', 'hour': '4',
+                    'referrer': '9', 'request': '/v1/a/c/o',
+                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
+        self.assertEquals(res, expected)
+        # too many fields
+        log_line = [str(x) for x in range(19)]
+        log_line[1] = 'proxy-server'
+        log_line[4] = '1/Jan/3/4/5/6'
+        log_line[6] = '/v1/a/c/o'
+        log_line = 'x'*16 + ' '.join(log_line)
+        res = p.log_line_parser(log_line)
+        expected = {'code': 8, 'processing_time': '17', 'auth_token': '11',
+                    'month': '01', 'second': '6', 'year': '3', 'tz': '+0000',
+                    'http_version': '7', 'object_name': 'o', 'etag': '14',
+                    'method': '5', 'trans_id': '15', 'client_ip': '2',
+                    'bytes_out': 13, 'container_name': 'c', 'day': '1',
+                    'minute': '5', 'account': 'a', 'hour': '4',
+                    'referrer': '9', 'request': '/v1/a/c/o',
+                    'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
+        self.assertEquals(res, expected)
 
 
 if __name__ == '__main__':