Merge lp:~tvansteenburgh/juju-deployer/1269783-force-terminate into lp:juju-deployer

[Tim Van Steenburgh]

Add force-termination option using -TT

[David Britton (dpb)]

excellent.

review: Approve

[lp:~tvansteenburgh/juju-deployer/1269783-force-terminate]

158. By Tim Van Steenburgh on 2015-09-25

Add force-termination option using -TT

=== modified file 'HACKING'
--- HACKING	2014-01-23 13:30:32 +0000
+++ HACKING	2015-09-23 15:36:10 +0000
@@ -10,10 +10,19 @@
   $ PYTHONPATH=. python deployer/cli.py
 
 
-Running tests
--------------
+Running unit tests
+------------------
 
 Tests are compatible with nose and can be run using, e.g.:
 
   $ nosetests -s --verbosity=2 deployer/tests
 
+
+Running live environment tests
+------------------------------
+
+  $ juju bootstrap -e local
+  $ JUJU_HOME=~/.juju \
+      JUJU_ENV=local \
+      TEST_ENDPOINT=`juju api-endpoints -e local` \
+      nosetests -s --verbosity=2 deployer/tests/test_goenv.py

=== modified file 'deployer/cli.py'
--- deployer/cli.py	2014-10-01 10:18:36 +0000
+++ deployer/cli.py	2015-09-23 15:36:10 +0000
@@ -50,10 +50,10 @@
         default=False)
     parser.add_argument(
         '-T', '--terminate-machines',
-        help=('Terminate all machines but the bootstrap node.  '
-              'Destroy any services that exist on each'),
-        dest="terminate_machines", action="store_true",
-        default=False)
+        help=('Terminate all machines but the bootstrap node. '
+              'Destroy any services that exist on each. '
+              'Use -TT to forcefully terminate.'),
+        dest="terminate_machines", action="count", default=0)
     parser.add_argument(
         '-t', '--timeout',
         help='Timeout (sec) for entire deployment (45min default)',
@@ -171,9 +171,10 @@
     if options.destroy_services or options.terminate_machines:
         log.info("Resetting environment...")
         env.connect()
-        env.reset(terminate_machines=options.terminate_machines,
+        env.reset(terminate_machines=bool(options.terminate_machines),
                   terminate_delay=options.deploy_delay,
-                  watch=options.watch)
+                  watch=options.watch,
+                  force_terminate=options.terminate_machines > 1)
         log.info("Environment reset in %0.2f", time.time() - start_time)
         sys.exit(0)
 

=== modified file 'deployer/env/base.py'
--- deployer/env/base.py	2015-08-04 12:47:05 +0000
+++ deployer/env/base.py	2015-09-23 15:36:10 +0000
@@ -111,13 +111,14 @@
         self._check_call(
             params, self.log, "Error exposing service %r", name)
 
-    def terminate_machine(self, mid, wait=False):
+    def terminate_machine(self, mid, wait=False, force=False):
         """Terminate a machine.
 
-        The machine can't have any running units, after removing the units or
-        destroying the service, use wait_for_units to know when its safe to
-        delete the machine (ie units have finished executing stop hooks and are
-        removed)
+        Unless ``force=True``, the machine can't have any running units.
+        After removing the units or destroying the service, use wait_for_units
+        to know when its safe to delete the machine (i.e., units have finished
+        executing stop hooks and are removed).
+
         """
         if ((isinstance(mid, int) and mid == 0) or
            (mid.isdigit() and int(mid) == 0)):
@@ -125,6 +126,8 @@
             raise RuntimeError("Can't terminate machine 0")
         params = self._named_env(["juju", "terminate-machine"])
         params.append(mid)
+        if force:
+            params.append('--force')
         try:
             self._check_call(
                 params, self.log, "Error terminating machine %r" % mid)

=== modified file 'deployer/env/go.py'
--- deployer/env/go.py	2015-03-12 18:41:43 +0000
+++ deployer/env/go.py	2015-09-23 15:36:10 +0000
@@ -27,18 +27,18 @@
         self.api_endpoint = endpoint
         self.client = None
 
-    def add_machine(self, series="",constraints={}):
+    def add_machine(self, series="", constraints={}):
         """Add a top level machine to the Juju environment.
 
         Use the given series and constraints.
         Return the machine identifier (e.g. "1").
-        
+
         series: a string such as 'precise' or 'trusty'.
         constraints: a map of constraints (such as mem, arch, etc.) which
           can be parsed by utils.parse_constraints
         """
         return self.client.add_machine(
-            series=series, 
+            series=series,
             constraints=parse_constraints(constraints))['Machine']
 
     def add_unit(self, service_name, machine_spec):
@@ -82,7 +82,9 @@
     def reset(self,
               terminate_machines=False,
               terminate_delay=0,
-              timeout=360, watch=False):
+              timeout=360,
+              watch=False,
+              force_terminate=False):
         """Destroy/reset the environment."""
         status = self.status()
         destroyed = False
@@ -95,8 +97,10 @@
             # Mark any errors as resolved so destruction can proceed.
             self.resolve_errors()
 
-            # Wait for units
-            self.wait_for_units(timeout, goal_state='removed', watch=watch)
+            if not (terminate_machines and force_terminate):
+                # Wait for units to be removed. Only necessary if we're not
+                # force-terminating machines.
+                self.wait_for_units(timeout, goal_state='removed', watch=watch)
 
         # The only value to not terminating is keeping the data on the
         # machines around.
@@ -104,13 +108,15 @@
             self.log.info(
                 " warning: juju-core machines are not reusable for units")
             return
-        self._terminate_machines(status, watch, terminate_delay)
+        self._terminate_machines(
+            status, watch, terminate_delay, force_terminate)
 
-    def _terminate_machines(self, status, watch, terminate_wait):
+    def _terminate_machines(self, status, watch, terminate_wait, force):
         """Terminate all machines, optionally wait for termination.
         """
         # Terminate machines
-        self.log.debug(" Terminating machines")
+        self.log.debug(
+            " Terminating machines %s", 'forcefully' if force else '')
 
         # Don't bother if there are no service unit machines
         if len(status['machines']) == 1:
@@ -137,7 +143,7 @@
         machines.sort(machine_sort)
 
         for mid in machines:
-            self._terminate_machine(mid, container_hosts)
+            self._terminate_machine(mid, container_hosts, force=force)
 
         if containers:
             watch = self.client.get_watch(120)
@@ -145,20 +151,20 @@
                 watch, containers).run(self._delta_event_log)
 
         for mid in container_hosts:
-            self._terminate_machine(mid)
+            self._terminate_machine(mid, force=force)
 
         if terminate_wait:
             self.log.info("  Waiting for machine termination")
             callback = watch and self._delta_event_log or None
-            self.client.wait_for_no_machines(None, callback)
+            self.client.wait_for_no_machines(terminate_wait, callback)
 
-    def _terminate_machine(self, mid, container_hosts=()):
+    def _terminate_machine(self, mid, container_hosts=(), force=False):
         if mid == "0":
             return
         if mid in container_hosts:
             return
         self.log.debug("  Terminating machine %s", mid)
-        self.terminate_machine(mid)
+        self.terminate_machine(mid, force=force)
 
     def _check_timeout(self, etime):
         w_timeout = etime - time.time()

=== modified file 'deployer/tests/test_goenv.py'
--- deployer/tests/test_goenv.py	2015-03-12 18:41:43 +0000
+++ deployer/tests/test_goenv.py	2015-09-23 15:36:10 +0000
@@ -15,6 +15,24 @@
     "Test env must be defined: TEST_ENDPOINT")
 class LiveEnvironmentTest(Base):
 
+    @classmethod
+    def setUpClass(cls):
+        """Base class sets JUJU_HOME to a new tmp dir, but for these
+        tests we need a real JUJU_HOME so that calls to
+        ``juju api-endpoints`` work properly.
+
+        """
+        if not os.environ.get('JUJU_HOME'):
+            raise RuntimeError('JUJU_HOME must be set')
+
+    @classmethod
+    def tearDownClass(cls):
+        """Base class deletes the tmp JUJU_HOME dir it created.
+        Override so we don't delete our real JUJU_HOME!
+
+        """
+        pass
+
     def setUp(self):
         self.endpoint = os.environ.get("TEST_ENDPOINT")
         self.output = self.capture_logging(
@@ -25,10 +43,14 @@
         status = self.env.status()
         self.assertFalse(status.get('services'))
         # Destroy everything.. consistent baseline
-        self.env.reset(terminate_machines=len(status['machines'].keys()) > 1, terminate_delay=240)
+        self.env.reset(
+            terminate_machines=len(status['machines'].keys()) > 1,
+            terminate_delay=240)
 
     def tearDown(self):
-        self.env.reset(terminate_machines=True, terminate_delay=240)
+        self.env.reset(
+            terminate_machines=True, terminate_delay=240,
+            force_terminate=True)
         self.env.close()
 
     def test_env(self):
@@ -54,7 +76,7 @@
 
     def test_add_machine(self):
         machine_name = self.env.add_machine()
-        
+
         # Sleep because juju core watches are eventually consistent (5s window)
         # and status rpc is broken (http://pad.lv/1203105)
         time.sleep(6)
@@ -64,12 +86,13 @@
 
     def test_set_annotation(self):
         machine_name = self.env.add_machine()
-        self.env.set_annotation(machine_name, {'foo': 'bar'}, entity_type='machine')
-        
+        self.env.set_annotation(
+            machine_name, {'foo': 'bar'}, entity_type='machine')
+
         # Sleep because juju core watches are eventually consistent (5s window)
         # and status rpc is broken (http://pad.lv/1203105)
         time.sleep(6)
-        status = self.env.status()
+        self.env.status()
 
         self.assertIn('foo', self.env.client.get_annotation(
             machine_name, 'machine')['Annotations'])

=== modified file 'deployer/tests/test_importer.py'
--- deployer/tests/test_importer.py	2015-08-06 08:40:16 +0000
+++ deployer/tests/test_importer.py	2015-09-23 15:36:10 +0000
@@ -4,7 +4,6 @@
 
 from deployer.config import ConfigStack
 from deployer.action.importer import Importer
-from deployer.utils import yaml_dump, yaml_load
 
 from base import (
     Base,
@@ -62,7 +61,9 @@
         config = {'name': '$hi_world _are_you_there? {guess_who}'}
         self.assertEqual(
             env.method_calls[3], mock.call.deploy(
-                'wiki', 'cs:precise/mediawiki', '', config, None, 1, None))
+                'wiki', 'cs:precise/mediawiki',
+                os.environ.get("JUJU_REPOSITORY", ""),
+                config, None, 1, None))
         env.add_relation.assert_called_once_with('wiki', 'db')
 
     @skip_if_offline

=== modified file 'deployer/utils.py'
--- deployer/utils.py	2015-09-03 14:25:56 +0000
+++ deployer/utils.py	2015-09-23 15:36:10 +0000
@@ -380,6 +380,7 @@
             raise ValueError("No Environment specified")
         return conf['default']
 
+
 def x_in_y(x, y):
     """Check to see if the second argument is named in the first
     argument's unit placement spec.