Merge bootstack-ops:plus65_retry into bootstack-ops:master

[Giuseppe Petralia]

Add sleep retry on MAAS connection errors

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[David O Neill (dmzoneill)]

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 58aaa04c3d7d1b2e8486ed007f41af74e3936542

diff --git a/bootstack-ops/cloud_report.py b/bootstack-ops/cloud_report.py
index df3fdc6..f13a5d3 100644
--- a/bootstack-ops/cloud_report.py
+++ b/bootstack-ops/cloud_report.py
@@ -50,6 +50,7 @@ from gevent.pywsgi import WSGIServer
 from maas.client.viscera.controllers import RackController, RegionController
 from oauthlib import oauth1
 import requests
+from aiohttp.client_exceptions import ServerDisconnectedError
 
 
 JUJU_DATA = os.environ.get('JUJU_DATA')
@@ -65,6 +66,9 @@ API_PORT = os.environ.get("API_PORT", 5000)
 
 UNKNOWN_VALUE = "unknown"
 
+RETRY = 10
+SLEEP = 2
+
 
 def get_timestamp():
     timestamp = time.time()
@@ -87,6 +91,15 @@ def url_join(base, uri):
         return base + uri
 
 
+def retry(call_to_retry, num=RETRY):
+    for i in range(0, num):
+        try:
+           return call_to_retry()
+        except Exception:
+            time.sleep(SLEEP * i)
+            print("Error: failed to retrieve machines from maas [{}/{}]".format(i+1,num))
+
+
 class MaasV2(object):
     def __init__(self):
         self.client = None
@@ -117,46 +130,57 @@ class MaasV2(object):
         else:
             self.maas_endpoint = MAAS_ENDPOINT
 
-        try:
-            if MAAS_PASSWORD:
-                self.client = login(
-                    self.maas_endpoint,
-                    username=MAAS_USERNAME, password=MAAS_PASSWORD,
-                )
-            elif MAAS_APIKEY:
-                self.client = connect(
-                    self.maas_endpoint,
-                    apikey=MAAS_APIKEY
-                )
-        except Exception as e:
-            print("Error connecting to MAAS endpoint: {}. {}".format(self.maas_endpoint, e))
-
-        return self.client is not None
+        for i in range(0, RETRY):
+            try:
+                if MAAS_PASSWORD:
+                    self.client = login(
+                        self.maas_endpoint,
+                        username=MAAS_USERNAME, password=MAAS_PASSWORD,
+                    )
+                elif MAAS_APIKEY:
+                    self.client = connect(
+                        self.maas_endpoint,
+                        apikey=MAAS_APIKEY
+                    )
+                return self.client is not None
+            except Exception as e:
+                print("Error connecting to MAAS endpoint: {}. {}".format(self.maas_endpoint, e))
+                time.sleep(SLEEP * i)
 
     def disconnect(self):
         pass
 
     def _list_machines(self):
         try:
-            return self.client.machines.list()
+            return retry(self.client.machines.list) or []
         except Exception as e:
             print("Error retrieving machines list from MAAS endpoint: {}. {}".format(self.maas_endpoint, e))
         return []
 
     def _list_rack_controllers(self):
         try:
-            return self.client.rack_controllers.list()
+            return retry(self.client.rack_controllers.list) or []
         except Exception as e:
             print("Error retrieving rack controllers list from MAAS endpoint: {}. {}".format(self.maas_endpoint, e))
         return []
 
     def _list_region_controllers(self):
         try:
-            return self.client.region_controllers.list()
+            return retry(self.client.region_controllers.list) or []
         except Exception as e:
             print("Error retrieving region controllers list from MAAS endpoint: {}. {}".format(self.maas_endpoint, e))
         return []
 
+    def _get_maas_version(self):
+        version = "None"
+        report_version = retry(self.client.version.get)
+        if report_version:
+            version = report_version.version
+        return version
+
+    def _get_power_parameters(self, m):
+        return retry(m.get_power_parameters) or {}
+
     def _serialize_machine(self, m):
         """
         Convert a Maas Machine into a dict.
@@ -172,7 +196,7 @@ class MaasV2(object):
         else:
             status = m.status.name.lower()
             power_type = m.power_type.lower()
-            power_address = m.get_power_parameters().get('power_address', UNKNOWN_VALUE)
+            power_address = self._get_power_parameters(m).get('power_address', UNKNOWN_VALUE)
 
         mac_addresses = [net_if.mac_address.lower() for net_if in m.interfaces]
 
@@ -198,7 +222,7 @@ class MaasV2(object):
         :return: json report
         """
 
-        maas_version = self.client.version.get().version
+        maas_version = self._get_maas_version()
 
         machines = list(map(self._serialize_machine, self._list_machines())) + \
                    list(map(self._serialize_machine, self._list_rack_controllers())) + \
@@ -274,8 +298,8 @@ class MaasV1(object):
                                    signature_method=oauth1.SIGNATURE_PLAINTEXT)
 
             url = url_join(self.maas_endpoint, uri)
-            uri, headers, body = client.sign(url)
-            response = requests.get(uri, headers=headers)
+            uri, headers, body = retry(client.sign(url)) or [None, None, None]
+            response = retry(requests.get(uri, headers=headers)) or ""
             if response.status_code == 200:
                 return response.json()
         except Exception as e:
@@ -374,13 +398,16 @@ class Juju(object):
         else:
             cmd = ['snap run juju', 'status', '--format=yaml']
 
-        try:
-            output = subprocess.check_output(cmd, env=env)
-            parsed = yaml.safe_load(output)
-            machines = parsed['machines']
-        except subprocess.CalledProcessError as e:
-            print("Exception calling juju status command. {}".format(e))
-
+        for i in range(0, RETRY):
+            try:
+                output = subprocess.check_output(cmd, timeout=30, env=env)
+                parsed = yaml.safe_load(output)
+                machines = parsed['machines']
+                i = RETRY + 1
+            except Exception as e:
+                print("Error: failed to retrieve machines from juju [{}/{}]\n{}".format(i+1,RETRY,e))
+                time.sleep(SLEEP * i)
+    
         return machines
 
     def connect(self):