OpenStack Object Storage (swift)

Merge lp:~notmyname/swift/fix_query_parsing into lp:~hudson-openstack/swift/trunk

  1. fix_query_parsing
  2. Merge into trunk
Proposed by John Dickinson
Status: Merged
Approved by: gholt
Approved revision: 252
Merged at revision: 251
Proposed branch: lp:~notmyname/swift/fix_query_parsing
Merge into: lp:~hudson-openstack/swift/trunk
Diff against target: 69 lines (+28/-2)
3 files modified
swift/stats/access_processor.py (+4/-1)
test/unit/stats/test_access_processor.py (+24/-0)
test/unit/stats/test_log_processor.py (+0/-1)
To merge this branch: bzr merge lp:~notmyname/swift/fix_query_parsing
Reviewer Review Type Date Requested Status
Swift Core security contacts Pending
Review via email: mp+53917@code.launchpad.net

Description of the change

makes the query parameter parsing explicit in the access log parser

To post a comment you must log in.
Revision history for this message
OpenStack Infra (hudson-openstack) wrote :
Download full text (72.6 KiB)

The attempt to merge lp:~notmyname/swift/fix_query_parsing into lp:swift failed. Below is the output from the failed tests.

running test
running egg_info
creating swift.egg-info
writing swift.egg-info/PKG-INFO
writing top-level names to swift.egg-info/top_level.txt
writing dependency_links to swift.egg-info/dependency_links.txt
writing entry points to swift.egg-info/entry_points.txt
writing manifest file 'swift.egg-info/SOURCES.txt'
reading manifest file 'swift.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
warning: no files found matching 'ChangeLog'
writing manifest file 'swift.egg-info/SOURCES.txt'
running build_ext

UNABLE TO READ FUNCTIONAL TESTS CONFIG FILE
UNABLE TO READ FUNCTIONAL TESTS CONFIG FILE
SKIPPING FUNCTIONAL TESTS DUE TO NO CONFIG
testAccountHead (test.functional.tests.TestAccount) ... SKIP
testContainerListing (test.functional.tests.TestAccount) ... SKIP
testContainerSerializedInfo (test.functional.tests.TestAccount) ... SKIP
testContainersOrderedByName (test.functional.tests.TestAccount) ... SKIP
testInvalidAuthToken (test.functional.tests.TestAccount) ... SKIP
testInvalidPath (test.functional.tests.TestAccount) ... SKIP
testInvalidUTF8Path (test.functional.tests.TestAccount) ... SKIP
testLastContainerMarker (test.functional.tests.TestAccount) ... SKIP
testListingLimit (test.functional.tests.TestAccount) ... SKIP
testMarkerLimitContainerList (test.functional.tests.TestAccount) ... SKIP
testNoAuthToken (test.functional.tests.TestAccount) ... SKIP
testPUT (test.functional.tests.TestAccount) ... SKIP
testVersionOnlyPath (test.functional.tests.TestAccount) ... SKIP
testGetRequest (test.functional.tests.TestAccountNoContainers) ... SKIP
testGetRequest (test.functional.tests.TestAccountNoContainersUTF8) ... SKIP
testAccountHead (test.functional.tests.TestAccountUTF8) ... SKIP
testContainerListing (test.functional.tests.TestAccountUTF8) ... SKIP
testContainerSerializedInfo (test.functional.tests.TestAccountUTF8) ... SKIP
testContainersOrderedByName (test.functional.tests.TestAccountUTF8) ... SKIP
testInvalidAuthToken (test.functional.tests.TestAccountUTF8) ... SKIP
testInvalidPath (test.functional.tests.TestAccountUTF8) ... SKIP
testInvalidUTF8Path (test.functional.tests.TestAccountUTF8) ... SKIP
testLastContainerMarker (test.functional.tests.TestAccountUTF8) ... SKIP
testListingLimit (test.functional.tests.TestAccountUTF8) ... SKIP
testMarkerLimitContainerList (test.functional.tests.TestAccountUTF8) ... SKIP
testNoAuthToken (test.functional.tests.TestAccountUTF8) ... SKIP
testPUT (test.functional.tests.TestAccountUTF8) ... SKIP
testVersionOnlyPath (test.functional.tests.TestAccountUTF8) ... SKIP
testContainerExistenceCachingProblem (test.functional.tests.TestContainer) ... SKIP
testContainerFileList (test.functional.tests.TestContainer) ... SKIP
testContainerFileListOnContainerThatDoesNotExist (test.functional.tests.TestContainer) ... SKIP
testContainerFileListWithLimit (test.functional.tests.TestContainer) ... SKIP
testContainerInfo (test.functional.tests.TestContainer) ... SKIP
testContainerInfoOnContainerThatDoesNotExist (test.functional.tests.TestContainer) ... SKIP
testContainerNameLimit (test.functional.tes...

lp:~notmyname/swift/fix_query_parsing updated
252. By John Dickinson

fixed tests

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'swift/stats/access_processor.py'
--- swift/stats/access_processor.py 2011-02-02 21:39:08 +0000
+++ swift/stats/access_processor.py 2011-03-17 22:24:35 +0000
@@ -20,6 +20,8 @@
20from swift.common.utils import split_path, get_logger20from swift.common.utils import split_path, get_logger
2121
22month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()22month_map = '_ Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec'.split()
23LISTING_PARAMS = set(
24 'path limit format delimiter marker end_marker prefix'.split())
2325
2426
25class AccessLogProcessor(object):27class AccessLogProcessor(object):
@@ -95,7 +97,8 @@
95 # (format, path, delimiter, etc.). Save a "1" here97 # (format, path, delimiter, etc.). Save a "1" here
96 # to indicate that this request is 1 request for98 # to indicate that this request is 1 request for
97 # its respective key.99 # its respective key.
98 d[k] = 1100 if k in LISTING_PARAMS:
101 d[k] = 1
99 d['client_ip'] = client_ip102 d['client_ip'] = client_ip
100 d['lb_ip'] = lb_ip103 d['lb_ip'] = lb_ip
101 d['method'] = method104 d['method'] = method
102105
=== modified file 'test/unit/stats/test_access_processor.py'
--- test/unit/stats/test_access_processor.py 2011-01-17 17:07:58 +0000
+++ test/unit/stats/test_access_processor.py 2011-03-17 22:24:35 +0000
@@ -21,6 +21,30 @@
2121
22class TestAccessProcessor(unittest.TestCase):22class TestAccessProcessor(unittest.TestCase):
2323
24 def test_log_line_parser_query_args(self):
25 p = access_processor.AccessLogProcessor({})
26 log_line = [str(x) for x in range(18)]
27 log_line[1] = 'proxy-server'
28 log_line[4] = '1/Jan/3/4/5/6'
29 query = 'foo'
30 for param in access_processor.LISTING_PARAMS:
31 query += '&%s=blah' % param
32 log_line[6] = '/v1/a/c/o?%s' % query
33 log_line = 'x'*16 + ' '.join(log_line)
34 res = p.log_line_parser(log_line)
35 expected = {'code': 8, 'processing_time': '17', 'auth_token': '11',
36 'month': '01', 'second': '6', 'year': '3', 'tz': '+0000',
37 'http_version': '7', 'object_name': 'o', 'etag': '14',
38 'method': '5', 'trans_id': '15', 'client_ip': '2',
39 'bytes_out': 13, 'container_name': 'c', 'day': '1',
40 'minute': '5', 'account': 'a', 'hour': '4',
41 'referrer': '9', 'request': '/v1/a/c/o',
42 'user_agent': '10', 'bytes_in': 12, 'lb_ip': '3'}
43 for param in access_processor.LISTING_PARAMS:
44 expected[param] = 1
45 expected['query'] = query
46 self.assertEquals(res, expected)
47
24 def test_log_line_parser_field_count(self):48 def test_log_line_parser_field_count(self):
25 p = access_processor.AccessLogProcessor({})49 p = access_processor.AccessLogProcessor({})
26 # too few fields50 # too few fields
2751
=== modified file 'test/unit/stats/test_log_processor.py'
--- test/unit/stats/test_log_processor.py 2011-02-16 22:29:59 +0000
+++ test/unit/stats/test_log_processor.py 2011-03-17 22:24:35 +0000
@@ -131,7 +131,6 @@
131 'http_version': 'HTTP/1.0',131 'http_version': 'HTTP/1.0',
132 'object_name': 'bar',132 'object_name': 'bar',
133 'etag': '-',133 'etag': '-',
134 'foo': 1,
135 'method': 'GET',134 'method': 'GET',
136 'trans_id': 'txfa431231-7f07-42fd-8fc7-7da9d8cc1f90',135 'trans_id': 'txfa431231-7f07-42fd-8fc7-7da9d8cc1f90',
137 'client_ip': '1.2.3.4',136 'client_ip': '1.2.3.4',