Merge lp:~notmyname/swift/stats_improvement into lp:~hudson-openstack/swift/trunk

[John Dickinson]

improved the container listings for the stats processor

improved the container listings for the stats processor

This fixes the issue where the stats system wouldn't process the last hour in the lookback window

[lp:~notmyname/swift/stats_improvement]

132. By John Dickinson on 2010-11-19

merged with trunk

133. By John Dickinson on 2010-11-22

merged with trunk

[clayg (clay-gerrard)]

I like the strptime stuff - much cleaner, I'm going to approve this just to get that in ;)

review: Approve

[Chuck Thier (cthier)]

I agree with clayg

review: Approve

=== modified file 'swift/stats/log_processor.py'
--- swift/stats/log_processor.py	2010-11-12 17:01:46 +0000
+++ swift/stats/log_processor.py	2010-11-22 23:34:25 +0000
@@ -113,48 +113,35 @@
                               listing_filter=None):
         '''
         Get a container listing, filtered by start_date, end_date, and
-        listing_filter. Dates, if given, should be in YYYYMMDDHH format
+        listing_filter. Dates, if given, must be in YYYYMMDDHH format
         '''
         search_key = None
         if start_date is not None:
-            date_parts = []
             try:
-                year, start_date = start_date[:4], start_date[4:]
-                if year:
-                    date_parts.append(year)
-                    month, start_date = start_date[:2], start_date[2:]
-                    if month:
-                        date_parts.append(month)
-                        day, start_date = start_date[:2], start_date[2:]
-                        if day:
-                            date_parts.append(day)
-                            hour, start_date = start_date[:2], start_date[2:]
-                            if hour:
-                                date_parts.append(hour)
-            except IndexError:
+                parsed_date = time.strptime(start_date, '%Y%m%d%H')
+            except ValueError:
                 pass
             else:
-                search_key = '/'.join(date_parts)
+                year = '%04d' % parsed_date.tm_year
+                month = '%02d' % parsed_date.tm_mon
+                day = '%02d' % parsed_date.tm_mday
+                hour = '%02d' % parsed_date.tm_hour
+                search_key = '/'.join([year, month, day, hour])
         end_key = None
         if end_date is not None:
-            date_parts = []
             try:
-                year, end_date = end_date[:4], end_date[4:]
-                if year:
-                    date_parts.append(year)
-                    month, end_date = end_date[:2], end_date[2:]
-                    if month:
-                        date_parts.append(month)
-                        day, end_date = end_date[:2], end_date[2:]
-                        if day:
-                            date_parts.append(day)
-                            hour, end_date = end_date[:2], end_date[2:]
-                            if hour:
-                                date_parts.append(hour)
-            except IndexError:
+                parsed_date = time.strptime(end_date, '%Y%m%d%H')
+            except ValueError:
                 pass
             else:
-                end_key = '/'.join(date_parts)
+                year = '%04d' % parsed_date.tm_year
+                month = '%02d' % parsed_date.tm_mon
+                day = '%02d' % parsed_date.tm_mday
+                hour = '%02d' % parsed_date.tm_hour
+                # Since the end_marker filters by <=, we need to add something
+                # to then end_key to make sure we get all the data under the
+                # last hour. Adding '/\x7f' should be all inclusive.
+                end_key = '/'.join([year, month, day, hour]) + '/\x7f'
         container_listing = self.internal_proxy.get_container_list(
                                     swift_account,
                                     container_name,

=== modified file 'test/unit/stats/test_log_processor.py'
--- test/unit/stats/test_log_processor.py	2010-11-19 22:52:31 +0000
+++ test/unit/stats/test_log_processor.py	2010-11-22 23:34:25 +0000
@@ -37,10 +37,11 @@
                            end_marker=None):
         n = '2010/03/14/13/obj1'
         if marker is None or n > marker:
-            if end_marker and n <= end_marker:
-                return [{'name': n}]
-            elif end_marker:
-                return []
+            if end_marker:
+                if n <= end_marker:
+                    return [{'name': n}]
+                else:
+                    return []
             return [{'name': n}]
         return []
 
@@ -178,6 +179,10 @@
                                             end_date='2010031412')
         expected = []
         self.assertEquals(result, expected)
+        result = p.get_container_listing('a', 'foo', start_date='2010031412',
+                                            end_date='2010031413')
+        expected = ['2010/03/14/13/obj1']
+        self.assertEquals(result, expected)
 
     def test_get_object_data(self):
         p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
@@ -212,7 +217,6 @@
         p = log_processor.LogProcessor(self.proxy_config, DumbLogger())
         result = p.generate_keylist_mapping()
         expected = {}
-        print p.plugins
         self.assertEquals(result, expected)
 
     def test_generate_keylist_mapping_with_dummy_plugins(self):