Merge lp:~notmyname/swift/stats_improvement into lp:~hudson-openstack/swift/trunk

Proposed by John Dickinson
Status: Merged
Approved by: Chuck Thier
Approved revision: 133
Merged at revision: 135
Proposed branch: lp:~notmyname/swift/stats_improvement
Merge into: lp:~hudson-openstack/swift/trunk
Prerequisite: lp:~notmyname/swift/end_marker
Diff against target: 109 lines (+27/-36)
2 files modified
swift/stats/log_processor.py (+18/-31)
test/unit/stats/test_log_processor.py (+9/-5)
To merge this branch: bzr merge lp:~notmyname/swift/stats_improvement
Reviewer Review Type Date Requested Status
Chuck Thier (community) Approve
clayg Approve
Review via email: mp+41199@code.launchpad.net

Commit message

improved the container listings for the stats processor

Description of the change

improved the container listings for the stats processor

This fixes the issue where the stats system wouldn't process the last hour in the lookback window

To post a comment you must log in.
132. By John Dickinson

merged with trunk

133. By John Dickinson

merged with trunk

Revision history for this message
clayg (clay-gerrard) wrote :

I like the strptime stuff - much cleaner, I'm going to approve this just to get that in ;)

review: Approve
Revision history for this message
Chuck Thier (cthier) wrote :

I agree with clayg

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'swift/stats/log_processor.py'
--- swift/stats/log_processor.py 2010-11-12 17:01:46 +0000
+++ swift/stats/log_processor.py 2010-11-22 23:34:25 +0000
@@ -113,48 +113,35 @@
113 listing_filter=None):113 listing_filter=None):
114 '''114 '''
115 Get a container listing, filtered by start_date, end_date, and115 Get a container listing, filtered by start_date, end_date, and
116 listing_filter. Dates, if given, should be in YYYYMMDDHH format116 listing_filter. Dates, if given, must be in YYYYMMDDHH format
117 '''117 '''
118 search_key = None118 search_key = None
119 if start_date is not None:119 if start_date is not None:
120 date_parts = []
121 try:120 try:
122 year, start_date = start_date[:4], start_date[4:]121 parsed_date = time.strptime(start_date, '%Y%m%d%H')
123 if year:122 except ValueError:
124 date_parts.append(year)
125 month, start_date = start_date[:2], start_date[2:]
126 if month:
127 date_parts.append(month)
128 day, start_date = start_date[:2], start_date[2:]
129 if day:
130 date_parts.append(day)
131 hour, start_date = start_date[:2], start_date[2:]
132 if hour:
133 date_parts.append(hour)
134 except IndexError:
135 pass123 pass
136 else:124 else:
137 search_key = '/'.join(date_parts)125 year = '%04d' % parsed_date.tm_year
126 month = '%02d' % parsed_date.tm_mon
127 day = '%02d' % parsed_date.tm_mday
128 hour = '%02d' % parsed_date.tm_hour
129 search_key = '/'.join([year, month, day, hour])
138 end_key = None130 end_key = None
139 if end_date is not None:131 if end_date is not None:
140 date_parts = []
141 try:132 try:
142 year, end_date = end_date[:4], end_date[4:]133 parsed_date = time.strptime(end_date, '%Y%m%d%H')
143 if year:134 except ValueError:
144 date_parts.append(year)
145 month, end_date = end_date[:2], end_date[2:]
146 if month:
147 date_parts.append(month)
148 day, end_date = end_date[:2], end_date[2:]
149 if day:
150 date_parts.append(day)
151 hour, end_date = end_date[:2], end_date[2:]
152 if hour:
153 date_parts.append(hour)
154 except IndexError:
155 pass135 pass
156 else:136 else:
157 end_key = '/'.join(date_parts)137 year = '%04d' % parsed_date.tm_year
138 month = '%02d' % parsed_date.tm_mon
139 day = '%02d' % parsed_date.tm_mday
140 hour = '%02d' % parsed_date.tm_hour
141 # Since the end_marker filters by <=, we need to add something
142 # to then end_key to make sure we get all the data under the
143 # last hour. Adding '/\x7f' should be all inclusive.
144 end_key = '/'.join([year, month, day, hour]) + '/\x7f'
158 container_listing = self.internal_proxy.get_container_list(145 container_listing = self.internal_proxy.get_container_list(
159 swift_account,146 swift_account,
160 container_name,147 container_name,
161148
=== modified file 'test/unit/stats/test_log_processor.py'
--- test/unit/stats/test_log_processor.py 2010-11-19 22:52:31 +0000
+++ test/unit/stats/test_log_processor.py 2010-11-22 23:34:25 +0000
@@ -37,10 +37,11 @@
37 end_marker=None):37 end_marker=None):
38 n = '2010/03/14/13/obj1'38 n = '2010/03/14/13/obj1'
39 if marker is None or n > marker:39 if marker is None or n > marker:
40 if end_marker and n <= end_marker:40 if end_marker:
41 return [{'name': n}]41 if n <= end_marker:
42 elif end_marker:42 return [{'name': n}]
43 return []43 else:
44 return []
44 return [{'name': n}]45 return [{'name': n}]
45 return []46 return []
4647
@@ -178,6 +179,10 @@
178 end_date='2010031412')179 end_date='2010031412')
179 expected = []180 expected = []
180 self.assertEquals(result, expected)181 self.assertEquals(result, expected)
182 result = p.get_container_listing('a', 'foo', start_date='2010031412',
183 end_date='2010031413')
184 expected = ['2010/03/14/13/obj1']
185 self.assertEquals(result, expected)
181186
182 def test_get_object_data(self):187 def test_get_object_data(self):
183 p = log_processor.LogProcessor(self.proxy_config, DumbLogger())188 p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
@@ -212,7 +217,6 @@
212 p = log_processor.LogProcessor(self.proxy_config, DumbLogger())217 p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
213 result = p.generate_keylist_mapping()218 result = p.generate_keylist_mapping()
214 expected = {}219 expected = {}
215 print p.plugins
216 self.assertEquals(result, expected)220 self.assertEquals(result, expected)
217221
218 def test_generate_keylist_mapping_with_dummy_plugins(self):222 def test_generate_keylist_mapping_with_dummy_plugins(self):