Merge lp:~ricardokirkner/isitdeployable/k8s-job-for-schema-migration into lp:isitdeployable

[Ricardo Kirkner]

use jobs to run db schema migrations before deployments

By using a job instead of an initContainer it's possible to ensure db migrations are run before the deployment is started. This enables us to remove the throttling on the rolling update process thus improving overall deployment time.

[Guillermo Gonzalez (verterok)]

LGTM

review: Approve

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable-docker/16/

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable-docker/17/

[Otto Co-Pilot (otto-copilot)]

Running landing tests failed
https://jenkins.ols.canonical.com/online-services/job/isitdeployable-docker/18/

=== modified file 'deployment/Makefile'
--- deployment/Makefile	2017-11-13 14:28:56 +0000
+++ deployment/Makefile	2017-11-15 15:54:57 +0000
@@ -34,7 +34,15 @@
 deploy: render-config  ## Deploy stack (from scratch)
 	@$(check-env-vars)
 	@$(check-image-vars)
-	kubectl apply -f tmp -f secrets.yaml --record
+	kubectl apply --record \
+		-f tmp/configmaps.yaml \
+		-f secrets.yaml \
+		-f tmp/services.yaml \
+		-f tmp/ingress.yaml
+	$(MAKE) migrate-db
+	kubectl apply --record -f tmp/deployments.yaml
+	kubectl -n $(NAMESPACE) rollout status deployment/app
+	$(MAKE) clean-migrate-db
 
 .PHONY: update-deploy
 update-deploy: FILENAME ?= tmp/deployments.yaml
@@ -42,6 +50,7 @@
 	@$(check-env-vars)
 	@$(check-image-vars)
 	kubectl replace -f $(FILENAME) --record
+	kubectl -n $(NAMESPACE) rollout status deployment/app
 
 .PHONY: update-config
 update-config: tmp/configmaps.yaml  ## Update config options
@@ -60,13 +69,26 @@
 	kubectl -n $(NAMESPACE) set image deployment/app app=$(DOCKER_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG) worker=$(DOCKER_REGISTRY)/$(IMAGE_NAME):$(IMAGE_TAG) --record
 	kubectl -n $(NAMESPACE) rollout status deployment/app
 
+.PHONY: migrate-db
+migrate-db: tmp/migrate-db-job.yaml  ## Run schema migrations
+	@$(check-env-vars)
+	kubectl apply -f tmp/migrate-db-job.yaml --record
+	until kubectl -n $(NAMESPACE) get jobs migrate-db -o jsonpath='{.status.conditions[?(@.type=="Complete")].status}' | grep True ; do sleep 1 ; done
+
+.PHONY: clean-migrate-db
+clean-migrate-db: tmp/migrate-db-job.yaml  ## Remove job for schema migrations
+	@$(check-env-vars)
+	kubectl delete -f tmp/migrate-db-job.yaml
+
 .PHONY: release
 release:  ## Cut a new release
 	@$(check-env-vars)
 	@$(check-image-vars)
 	$(MAKE) clean-config
 	$(MAKE) render-config
+	$(MAKE) migrate-db
 	$(MAKE) update-deploy
+	$(MAKE) clean-migrate-db
 
 .PHONY: rollback
 rollback:  ## Undo latest release

=== modified file 'deployment/config/deployments.yaml.tpl'
--- deployment/config/deployments.yaml.tpl	2017-11-10 16:02:18 +0000
+++ deployment/config/deployments.yaml.tpl	2017-11-15 15:54:57 +0000
@@ -11,9 +11,6 @@
   revisionHistoryLimit: 2
   strategy:
     type: RollingUpdate
-    rollingUpdate:
-      maxSurge: 1
-      maxUnavailable: 0
   minReadySeconds: 5
   selector:
     matchLabels:
@@ -38,31 +35,6 @@
           - key: "id_rsa.pub"
             path: ./id_rsa.pub
             mode: 0644
-      initContainers:
-      - name: migrate-db
-        image: ${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}
-        imagePullPolicy: Always
-        env:
-          # config
-          - name: DJANGO_SETTINGS_MODULE
-            value: django_project.settings_base
-          # secrets
-          - name: ADMIN_DATABASE_URL
-            valueFrom:
-              secretKeyRef:
-                name: app
-                key: ADMIN_DATABASE_URL
-          - name: DATABASE_URL
-            valueFrom:
-              secretKeyRef:
-                name: app
-                key: DATABASE_URL
-          - name: SECRET_KEY
-            valueFrom:
-              secretKeyRef:
-                name: app
-                key: SECRET_KEY
-        command: ["make", "app-update-db"]
       containers:
       - name: app
         image: ${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}

=== added file 'deployment/config/migrate-db-job.yaml.tpl'
--- deployment/config/migrate-db-job.yaml.tpl	1970-01-01 00:00:00 +0000
+++ deployment/config/migrate-db-job.yaml.tpl	2017-11-15 15:54:57 +0000
@@ -0,0 +1,41 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: migrate-db
+  namespace: ${NAMESPACE}
+  labels:
+    revision: "${REVISION}"
+spec:
+  template: 
+    metadata:
+      name: migrate-db
+      namespace: ${NAMESPACE}
+      labels:
+        revision: "${REVISION}"
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: migrate-db
+          image: ${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}
+          imagePullPolicy: Always
+          env:
+            # config
+            - name: DJANGO_SETTINGS_MODULE
+              value: django_project.settings_base
+            # secrets
+            - name: ADMIN_DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: app
+                  key: ADMIN_DATABASE_URL
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: app
+                  key: DATABASE_URL
+            - name: SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: app
+                  key: SECRET_KEY
+          command: ["make", "app-update-db"]