Merge lp:~thomir-deactivatedaccount/adt-continuous-deployer/trunk-fix-reaper into lp:adt-continuous-deployer

[Thomi Richards]

Several improvements to repaer.py to try and make it more reliable.

Several improvements to repaer.py to try and make it more reliable.

[Thomi Richards (thomir-deactivatedaccount)]

This has been tested on wendigo, and from my small sample size (3), appears to solve the issue we're seeing.

[Celso Providelo (cprov)]

Nice! way more robust for destroying envs, `nova delete` will never fail us as juju does.

review: Approve

=== modified file 'ci_automation/branch.py'
--- ci_automation/branch.py	2015-06-04 23:09:56 +0000
+++ ci_automation/branch.py	2015-06-25 23:23:48 +0000
@@ -26,6 +26,7 @@
 
 from . import config
 
+
 def get_revno(url):
     '''Get the current revision for a branch URL.
 
@@ -139,4 +140,3 @@
     """
     identifiers = ["{}\t{}".format(k, v) for k, v in revno_dict.items()]
     return '\n'.join(identifiers)
-

=== modified file 'ci_automation/juju.py'
--- ci_automation/juju.py	2015-05-28 00:21:30 +0000
+++ ci_automation/juju.py	2015-06-25 23:23:48 +0000
@@ -112,9 +112,16 @@
         # No juju environment here.
         pass
     else:
-        # FIXME or leave alone?
         args = ['juju', 'destroy-environment', '-y', '--force', name]
-        subprocess.check_call(args, env=env)
+        try:
+            utils.check_call(args, env=env)
+        except subprocess.CalledProcessError as e:
+            print("juju destroy-environment failed with %s" % e)
+            print("Manually deleting left-over instances...")
+            # juju sometimes fails, so simulate what it does ourselves.
+            remaining = nova.get_instances_for_environment(name)
+            nova.delete_instances(remaining)
+            print("Instances have been manually deleted")
 
     # Ensure all related security-groups left behind by `juju` are removed,
     # otherwise they will remain consuming resources (and quota).
@@ -125,26 +132,29 @@
         if name in line
     ]
 
-    for secgroup in secgroups:
-        secgroup = secgroup.strip()
-        # Try harder because sometimes the instance retaining the secgroup
-        # takes sometime to be purged.
-        for i in range(3):
+    # Try harder because sometimes the instance retaining the secgroup
+    # takes sometime to be purged.
+    for i in range(3):
+        for secgroup in secgroups.copy():
             try:
                 subprocess.check_call(
                     ['nova', 'secgroup-delete', secgroup],
                     stdout=subprocess.DEVNULL,
                     stderr=subprocess.DEVNULL
                 )
+                secgroups.remove(secgroup)
             except subprocess.CalledProcessError:
-                time.sleep(1)
-            else:
-                break
-        else:
-            print(
-                'The security-group "{}" could not be removed. '
-                'Please remove it manually.'.format(secgroup)
-            )
+                print(
+                    "Error removing secgroup '{}'. "
+                    "It will be retried".format(secgroup)
+                )
+        time.sleep(1)
+
+    if secgroups:
+        print(
+            'The security groups "{}" could not be removed. '
+            'Please remove them manually.'.format(', '.join(secgroups))
+        )
 
     shutil.rmtree(env_dir)
 

=== modified file 'ci_automation/nova.py'
--- ci_automation/nova.py	2015-06-19 09:33:20 +0000
+++ ci_automation/nova.py	2015-06-25 23:23:48 +0000
@@ -21,6 +21,8 @@
 import stat
 import subprocess
 
+from . import utils
+
 
 def nova_list():
     """Default 'nova list' accessor."""
@@ -36,6 +38,29 @@
     return output
 
 
+def get_list_of_nova_instances(nova_output=None):
+    """A generator that yields all the current nova instances."""
+    if nova_output is None:
+        nova_output = nova_list()
+
+    for line in io.StringIO(nova_output).readlines()[3:-1]:
+        yield line.split('|')[2].strip()
+
+
+def get_instances_for_environment(env_name):
+    """get a list of all instnaces whose names start with 'env_name'.
+
+    This is a hacky way of getting all the nova instances that are part of
+    a given juju environment.
+
+    """
+    matched = []
+    for instance_name in get_list_of_nova_instances():
+        if instance_name.strip().startswith(env_name):
+            matched.append(instance_name)
+    return matched
+
+
 def get_services_map(base_dir, nova_output=None):
     """Return a services map dictionary.
 
@@ -146,3 +171,12 @@
     # If there is no network key (unlikely), treat it as not having a floating
     # ip:
     return False
+
+
+def delete_instances(instance_list):
+    for i in instance_list:
+        delete_instance(i)
+
+
+def delete_instance(instance_name):
+    utils.check_call(['nova', 'delete', instance_name])