Merge lp:~thomir-deactivatedaccount/core-result-checker/trunk-check-swift-better into lp:core-result-checker

[Thomi Richards]

Wait for swift to be consistent.

Add a check for swift to become consistent before we try and make the container public.

[Celso Providelo (cprov)]

Right, let's see the SwiftManager in action.

I just think that 10 minutes spinning message waiting for swift seems excessive (I can see the logging storm already), but we shall see ...

For situations like this I'd prefer moving jobs quickly (< 30 s) to the deadletter queue and have proper monitoring and replay scripts.

review: Approve

[lp:~thomir-deactivatedaccount/core-result-checker/trunk-check-swift-better]

15. By Thomi Richards on 2015-04-07

Set default timeout to 60 seconds, not 600.

=== modified file 'core_result_checker/__init__.py'
--- core_result_checker/__init__.py	2015-04-07 00:06:54 +0000
+++ core_result_checker/__init__.py	2015-04-07 04:03:38 +0000
@@ -17,6 +17,7 @@
 
 import argparse
 import configparser
+from datetime import datetime
 import logging
 import os
 
@@ -37,9 +38,9 @@
 
 class Worker(object):
 
-    def __init__(self, swift_publisher, retry_publisher):
+    def __init__(self, swift_manager, retry_publisher):
         get_logger(__name__).info("Service Started.")
-        self.swift_publisher = swift_publisher
+        self.swift_manager = swift_manager
         self.retry_publisher = retry_publisher
 
     def __call__(self, message):
@@ -54,6 +55,7 @@
             logger.error("Unable to unpack incoming message: %s", str(e))
             return MessageActions.Retry
 
+        # check to see if adt-run reported infrastructure failures:
         if int(exit_code) in (16, 20, 100):
             logger.info(
                 "Test run infrastructure failed (exit code %s), retrying.",
@@ -66,8 +68,18 @@
             device,
             image_name,
         )
+
+        # check to see if swift is consistent yet:
+        if not self.swift_manager.container_contains_file(
+            container_name,
+            "results.tgz"
+        ):
+            logger.warning("Swift container not yet consistent.")
+            self.retry_publisher.retry_for_swift(message)
+            return MessageActions.Acknowledge
+
         try:
-            self.swift_publisher.make_container_public(container_name)
+            self.swift_manager.make_container_public(container_name)
         except RuntimeError as e:
             logger.error(
                 "Unable to publish swift container '%s'. Error: %s.",
@@ -105,9 +117,9 @@
     return config
 
 
-class SwiftPublisher(object):
+class SwiftManager(object):
 
-    """A class that knows how to make a swift container public."""
+    """A class that knows how to interact with swift."""
 
     def __init__(self, nova_config):
         self.config = nova_config
@@ -131,6 +143,11 @@
                 )
             )
 
+    def container_contains_file(self, container_name, file_name):
+        swift = SwiftService(self.config)
+        result = next(swift.stat(container_name, [file_name]))
+        return result['success']
+
 
 class RetryPublisher(object):
 
@@ -139,9 +156,10 @@
     def __init__(self, connection, max_retries):
         self.connection = connection
         self.max_retries = max_retries
+        self.swift_retry_seconds = 60
 
     def retry_test_run(self, payload):
-        """Maybe retry a test payload.
+        """Maybe retry a test payload. Due to adt-run reported failure.
 
         This will look at it's retry count, and either requeue it in the
         test queue, or insert it into the dead letter queue.
@@ -149,23 +167,65 @@
         """
         retry_count = int(payload.get('test_run_retry_count', '0'))
         if retry_count < self.max_retries:
-            q = self.connection.SimpleQueue(
-                "core.tests.{}".format(constants.API_VERSION)
-            )
             payload['test_run_retry_count'] = retry_count + 1
-            q.put(payload)
-            q.close()
+            self._insert_payload_into_queue(
+                "core.tests.{}".format(constants.API_VERSION),
+                payload
+            )
         else:
             logger = get_logger(__name__, payload)
             logger.error(
                 "Test retry count exceeds maximum. Inserting into dead "
                 "letter queue"
             )
-            q = self.connection.SimpleQueue(
-                "core.deadletters.{}".format(constants.API_VERSION)
-            )
-            q.put(payload)
-            q.close()
+            self._insert_payload_into_queue(
+                "core.deadletters.{}".format(constants.API_VERSION),
+                payload
+            )
+
+    def retry_for_swift(self, payload):
+        """Maybe retry a test payload while waiting for swift.
+
+        This will look at the first time the message was checked, and give a
+        certain grace period. After that period is over, we will put the
+        payload into the dead letter queue.
+
+        """
+        swift_check_key = 'swift_first_check_time'
+        first_swift_check_time = payload.get(swift_check_key)
+        if first_swift_check_time is None:
+            first_swift_check_time = datetime.utcnow()
+            payload[swift_check_key] = str(first_swift_check_time.timestamp())
+        else:
+            first_swift_check_time = datetime.fromtimestamp(
+                float(first_swift_check_time)
+            )
+        now = datetime.utcnow()
+        time_since_first_check = now - first_swift_check_time
+        logger = get_logger(__name__, payload)
+        if time_since_first_check.total_seconds() < self.swift_retry_seconds:
+            logger.info(
+                "Requeueing message since swift is not yet consistent"
+            )
+            self._insert_payload_into_queue(
+                "core.result.{}".format(constants.API_VERSION),
+                payload
+            )
+        else:
+            logger.error(
+                "swift is still not consistent after %d seconds. Inserting "
+                "message into dead letter queue.",
+                self.swift_retry_seconds
+            )
+            self._insert_payload_into_queue(
+                "core.deadletters.{}".format(constants.API_VERSION),
+                payload
+            )
+
+    def _insert_payload_into_queue(self, queue, payload):
+        q = self.connection.SimpleQueue(queue)
+        q.put(payload)
+        q.close()
 
 
 def main():
@@ -184,9 +244,9 @@
     )
     try:
         with kombu.Connection(amqp_uris) as connection:
-            swift_publisher = SwiftPublisher(config['nova'])
+            swift_manager = SwiftManager(config['nova'])
             retry_publisher = RetryPublisher(connection, 3)
-            worker = Worker(swift_publisher, retry_publisher)
+            worker = Worker(swift_manager, retry_publisher)
             queue_monitor = SimpleRabbitQueueWorker(
                 connection,
                 "core.result.{}".format(constants.API_VERSION),