Merge lp:~david-goetz/swift/zero_logger into lp:~hudson-openstack/swift/trunk

[David Goetz]

Changing auditor logging to show zero byte files rate and amount of time doing list dirs verses actual auditing.

[Jay Payne (letterj)]

Looks good but one note. It doesn't display the number of quarantined files when it finishes.

Just something to think about since this will be used in cluster with many different use cases.

review: Approve

[lp:~david-goetz/swift/zero_logger]

299. By David Goetz on 2011-05-23

adding total quarantine and error to logs

[Jay Payne (letterj)]

Looks great. Thanks for adding the totals to the final numbers

review: Approve

[gholt (gholt)]

Should this be linked to https://bugs.launchpad.net/swift/+bug/782328 ?

[gholt (gholt)]

review: Approve

=== modified file 'swift/obj/auditor.py'
--- swift/obj/auditor.py	2011-03-16 14:55:07 +0000
+++ swift/obj/auditor.py	2011-05-23 20:21:11 +0000
@@ -63,46 +63,57 @@
         begin = reported = time.time()
         self.total_bytes_processed = 0
         self.total_files_processed = 0
+        total_quarantines = 0
+        total_errors = 0
         files_running_time = 0
+        time_auditing = 0
         all_locs = audit_location_generator(self.devices,
                                             object_server.DATADIR,
                                             mount_check=self.mount_check,
                                             logger=self.logger)
         for path, device, partition in all_locs:
+            loop_time = time.time()
             self.object_audit(path, device, partition)
             self.files_running_time = ratelimit_sleep(
                 self.files_running_time, self.max_files_per_second)
             self.total_files_processed += 1
-            if time.time() - reported >= self.log_time:
+            now = time.time()
+            if now - reported >= self.log_time:
                 self.logger.info(_(
                     'Object audit (%(type)s). '
                     'Since %(start_time)s: Locally: %(passes)d passed, '
                     '%(quars)d quarantined, %(errors)d errors '
-                    'files/sec: %(frate).2f , bytes/sec: %(brate).2f') % {
+                    'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
+                    'Total time: %(total).2f, Auditing time: %(audit).2f, '
+                    'Rate: %(audit_rate).2f') % {
                             'type': self.auditor_type,
                             'start_time': time.ctime(reported),
-                            'passes': self.passes,
-                            'quars': self.quarantines,
+                            'passes': self.passes, 'quars': self.quarantines,
                             'errors': self.errors,
-                            'frate': self.passes / (time.time() - reported),
-                            'brate': self.bytes_processed /
-                                     (time.time() - reported)})
-                reported = time.time()
+                            'frate': self.passes / (now - reported),
+                            'brate': self.bytes_processed / (now - reported),
+                            'total': (now - begin), 'audit': time_auditing,
+                            'audit_rate': time_auditing / (now - begin)})
+                reported = now
+                total_quarantines += self.quarantines
+                total_errors += self.errors
                 self.passes = 0
                 self.quarantines = 0
                 self.errors = 0
                 self.bytes_processed = 0
+            time_auditing += (now - loop_time)
         elapsed = time.time() - begin
         self.logger.info(_(
-                'Object audit (%(type)s) "%(mode)s" mode '
-                'completed: %(elapsed).02fs. '
-                'Total files/sec: %(frate).2f , '
-                'Total bytes/sec: %(brate).2f ') % {
-                    'type': self.auditor_type,
-                    'mode': mode,
-                    'elapsed': elapsed,
-                    'frate': self.total_files_processed / elapsed,
-                    'brate': self.total_bytes_processed / elapsed})
+            'Object audit (%(type)s) "%(mode)s" mode '
+            'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
+            'Total errors: %(errors)d, Total files/sec: %(frate).2f , '
+            'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
+            'Rate: %(audit_rate).2f') % {
+                'type': self.auditor_type, 'mode': mode, 'elapsed': elapsed,
+                'quars': total_quarantines, 'errors': total_errors,
+                'frate': self.total_files_processed / elapsed,
+                'brate': self.total_bytes_processed / elapsed,
+                'audit': time_auditing, 'audit_rate': time_auditing / elapsed})
 
     def object_audit(self, path, device, partition):
         """
@@ -133,6 +144,7 @@
             except DiskFileNotExist:
                 return
             if self.zero_byte_only_at_fps and obj_size:
+                self.passes += 1
                 return
             for chunk in df:
                 self.bytes_running_time = ratelimit_sleep(