Merge ~pguimaraes/maas:kvm-overcommit-fix-lp1842896 into ~blake-rouse/maas:fix-1843493

[Pedro Guimarães]

Fixes LP: #1842896. Following changes made on Pods form and model:
* created machine inherits Pod zone instead of Default value, since we are filtering Pods by zone just before calling get_allocated_composed_machine
* Pod model's create_machine now checks for overcommit ratios just before running the creation process, failing with PodProblem if overcommit limits arenot respected
* create_and_sync runs create_machine within transactional() method to avoid concurrency where multiple allocate requests ends up landing on same Pod and not respecting overcommit limits for that Pod

Fixes LP: #1842896. Following changes made on Pods form and model:
* created machine inherits Pod zone instead of Default value, since we are filtering Pods by zone just before calling get_allocated_composed_machine
* Pod model's create_machine now checks for overcommit ratios just before running the creation process, failing with PodProblem if overcommit limits arenot respected
* create_and_sync runs create_machine within transactional() method to avoid concurrency where multiple allocate requests ends up landing on same Pod and not respecting overcommit limits for that Pod

[Blake Rouse (blake-rouse)]

I am actively working on proper fix for this, jsyk

Unmerged commits

8aad7b4... by Pedro Guimarães on 2019-09-16

Fixes LP: #1842896. Following changes made on Pods form and model:
* created machine inherits Pod zone instead of Default value, since we are filtering Pods by zone just before calling get_allocated_composed_machine
* Pod model's create_machine now checks for overcommit ratios just before running the creation process, failing with PodProblem if overcommit limits arenot respected
* create_and_sync runs create_machine within transactional() method to avoid concurrency where multiple allocate requests ends up landing on same Pod and not respecting overcommit limits for that Pod

diff --git a/src/maasserver/forms/pods.py b/src/maasserver/forms/pods.py
index 85c1da2..54dc71f 100644
--- a/src/maasserver/forms/pods.py
+++ b/src/maasserver/forms/pods.py
@@ -443,7 +443,7 @@ class ComposeMachineForm(forms.Form):
         self.initial['domain'] = Domain.objects.get_default_domain()
         self.fields['zone'] = ModelChoiceField(
             required=False, queryset=Zone.objects.all())
-        self.initial['zone'] = Zone.objects.get_default_zone()
+        self.initial['zone'] = self.pod.zone
         self.fields['pool'] = ModelChoiceField(
             required=False, queryset=ResourcePool.objects.all())
         self.initial['pool'] = self.pod.pool
@@ -639,6 +639,59 @@ class ComposeMachineForm(forms.Form):
                 self.pod.sync_hints(pod_hints)
             return created_machine
 
+        def create_and_sync_nonIOThread(result, post_pod_sync=False):
+            requested_machine, result = result
+            discovered_machine, pod_hints = result
+
+            @asynchronous
+            def wrap_create_machine(func,
+                                    discovered_machine,
+                                    commissioning_user,
+                                    skip_commissioning,
+                                    creation_type,
+                                    interfaces,requested_machine,
+                                    domain,pool,zone):
+                d = deferToDatabase(transactional(
+                    partial(
+                        func,
+                        discovered_machine=discovered_machine,
+                        commissioning_user=commissioning_user,
+                        skip_commissioning=skip_commissioning,
+                        creation_type=creation_type,
+                        interfaces=interfaces,
+                        requested_machine=requested_machine,
+                        domain=domain,
+                        pool=pool,
+                        zone=zone,
+                    )
+                ))
+                return d
+            try:
+                created_machine = wrap_create_machine(
+                    self.pod.create_machine,
+                    discovered_machine=discovered_machine,
+                    commissioning_user=self.request.user,
+                    skip_commissioning=skip_commissioning,
+                    creation_type=creation_type,
+                    interfaces=self.get_value_for('interfaces'),
+                    requested_machine=requested_machine,
+                    domain=self.get_value_for('domain'),
+                    pool=self.get_value_for('pool'),
+                    zone=self.get_value_for('zone'),
+                ).wait(timeout)
+            except crochet.TimeoutError:
+                log.debug("Compose: wrap_create_machine timed out")
+                raise Exception(
+                    "Unable to compose a machine because database driver "
+                    "timed out after %d seconds." % (
+                        timeout))
+            
+            post_commit_do(
+                deferToDatabase,
+                transactional(self.pod.sync_hints), pod_hints)
+            log.info("Composition: Result received is: {}".format(created_machine))
+            return created_machine
+
         @inlineCallbacks
         def async_compose_machine(
                 result, power_type, power_paramaters, **kwargs):
@@ -706,7 +759,7 @@ class ComposeMachineForm(forms.Form):
                     "timed out after %d seconds." % (
                         self.pod.power_type, timeout))
             try:
-                return create_and_sync(
+                return create_and_sync_nonIOThread(
                     (requested_machine, result), post_pod_sync=True)
             except Exception:
                 # Issue saving the machine into the database. The composed
diff --git a/src/maasserver/models/bmc.py b/src/maasserver/models/bmc.py
index 5b76b6c..b312dbf 100644
--- a/src/maasserver/models/bmc.py
+++ b/src/maasserver/models/bmc.py
@@ -783,6 +783,23 @@ class Pod(BMC):
             skip_commissioning=False,
             creation_type=NODE_CREATION_TYPE.PRE_EXISTING, interfaces=None,
             requested_machine=None, **kwargs):
+
+        # TODO: Both check_over_commit_ratios and following code to create
+        #       machine should be run within same DB atomic transaction.
+        #       that will ensure that a single thread is adding machines
+        #       to this KVM host.
+        #       TODO action is to try detect if we're under
+        #       transaction.atomic()
+        #       if not, generate LOG warning in case with:
+        #       "models.bmc.pod.create_machine should always run within 
+        #        transaction.atomic(): <ADD STACK TRACE>"
+        msg = self.check_over_commit_ratios(discovered_machine.cores,
+                                            discovered_machine.memory)
+        if msg:
+            raise PodProblem(
+                "Unable to compose KVM instance in '%s'. %s" % (
+                    self.name, msg))
+
         """Create's a `Machine` from `discovered_machines` for this pod."""
         if skip_commissioning:
             status = NODE_STATUS.READY