Merge ~mpontillo/maas:enable-ping--js--bug-1802325 into maas:master

[Mike Pontillo]

Add connectivity check and retry behavior to web UI.

 * Periodically send WebSocket requests to ensure connectivity.
 * Attempt to retry connection if requests are not answered in a timely manner.
 * Fix tests to ensure real WebSocket connections are not attempted.
 * Change .gitattributes to ensure built JavaScript and bundles and CSS are not included in text-based diff output.
 * Enable cache to make webpack builds faster.

Partial fix for bug #1802325.

[MAAS Lander (maas-lander)]

UNIT TESTS
-b enable-ping--js--bug-1802325 lp:~mpontillo/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: a354b3b71ad82561206dab15cf455cca55ce3430

review: Approve

[Blake Rouse (blake-rouse)]

Looks good. Like the ES6 class, so much nicer than that old JS crap.

review: Approve

e9712be... by Mike Pontillo on 2018-11-20

Remove excessive debug logging.

1d2b634... by Mike Pontillo on 2018-11-20

Clear out event handlers so we don't get events from stale websockets.

diff --git a/.gitattributes b/.gitattributes
index 5c54628..7d16138 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -7,3 +7,8 @@
 /services export-ignore
 .gitignore export-ignore
 .gitattributes export-ignore
+
+# Don't include generated JavaScript or CSS in diff output.
+# (They will still show up as binary files that differ.)
+/src/maasserver/static/js/bundle/** -diff
+/src/maasserver/static/css/build.css -diff
diff --git a/src/maasserver/static/js/angular/directives/tests/test_boot_images.js b/src/maasserver/static/js/angular/directives/tests/test_boot_images.js
index c569220..91471c8 100644
--- a/src/maasserver/static/js/angular/directives/tests/test_boot_images.js
+++ b/src/maasserver/static/js/angular/directives/tests/test_boot_images.js
@@ -24,6 +24,11 @@ describe("maasBootImages", function() {
         BootResourcesManager = $injector.get('BootResourcesManager');
         UsersManager = $injector.get('UsersManager');
         ManagerHelperService = $injector.get('ManagerHelperService');
+        // Mock buildSocket so an actual connection is not made.
+        let RegionConnection = $injector.get("RegionConnection");
+        let webSocket = new MockWebSocket();
+        spyOn(RegionConnection, "buildSocket").and.returnValue(webSocket);
+
     }));
 
     // Create a new scope before each test.
diff --git a/src/maasserver/static/js/angular/directives/tests/test_ipranges.js b/src/maasserver/static/js/angular/directives/tests/test_ipranges.js
index db8f812..327f3f8 100644
--- a/src/maasserver/static/js/angular/directives/tests/test_ipranges.js
+++ b/src/maasserver/static/js/angular/directives/tests/test_ipranges.js
@@ -24,6 +24,10 @@ describe("maasIPRanges", function() {
         IPRangesManager = $injector.get('IPRangesManager');
         UsersManager = $injector.get('UsersManager');
         ManagerHelperService = $injector.get('ManagerHelperService');
+        // Mock buildSocket so an actual connection is not made.
+        let RegionConnection = $injector.get("RegionConnection");
+        let webSocket = new MockWebSocket();
+        spyOn(RegionConnection, "buildSocket").and.returnValue(webSocket);
     }));
 
     // Create a new scope before each test.
diff --git a/src/maasserver/static/js/angular/directives/tests/test_notifications.js b/src/maasserver/static/js/angular/directives/tests/test_notifications.js
index 22c7532..e7704b8 100644
--- a/src/maasserver/static/js/angular/directives/tests/test_notifications.js
+++ b/src/maasserver/static/js/angular/directives/tests/test_notifications.js
@@ -62,9 +62,13 @@ describe("maasNotifications", function() {
     var theNotificationsManager;
     var $scope;
 
-    beforeEach(inject(function($rootScope, NotificationsManager) {
+    beforeEach(inject(function($rootScope, NotificationsManager, $injector) {
         theNotificationsManager = NotificationsManager;
         $scope = $rootScope.$new();
+        // Mock buildSocket so an actual connection is not made.
+        let RegionConnection = $injector.get("RegionConnection");
+        let webSocket = new MockWebSocket();
+        spyOn(RegionConnection, "buildSocket").and.returnValue(webSocket);
     }));
 
     describe("maas-notifications", function() {
diff --git a/src/maasserver/static/js/angular/directives/tests/test_pod_parameters.js b/src/maasserver/static/js/angular/directives/tests/test_pod_parameters.js
index 618bb5c..e1f1774 100644
--- a/src/maasserver/static/js/angular/directives/tests/test_pod_parameters.js
+++ b/src/maasserver/static/js/angular/directives/tests/test_pod_parameters.js
@@ -15,6 +15,10 @@ describe("maasPodParameters", function() {
         PodsManager = $injector.get("PodsManager");
         GeneralManager = $injector.get("GeneralManager");
         ManagerHelperService = $injector.get("ManagerHelperService");
+        // Mock buildSocket so an actual connection is not made.
+        let RegionConnection = $injector.get("RegionConnection");
+        let webSocket = new MockWebSocket();
+        spyOn(RegionConnection, "buildSocket").and.returnValue(webSocket);
     }));
 
     // Create a new scope before each test.
diff --git a/src/maasserver/static/js/angular/directives/tests/test_ssh_keys.js b/src/maasserver/static/js/angular/directives/tests/test_ssh_keys.js
index 8783f61..d886f5d 100644
--- a/src/maasserver/static/js/angular/directives/tests/test_ssh_keys.js
+++ b/src/maasserver/static/js/angular/directives/tests/test_ssh_keys.js
@@ -25,6 +25,10 @@ describe("maasSshKeys", function() {
         SSHKeysManager = $injector.get('SSHKeysManager');
         JSONService = $injector.get('JSONService');
         ManagerHelperService = $injector.get('ManagerHelperService');
+        // Mock buildSocket so an actual connection is not made.
+        let RegionConnection = $injector.get("RegionConnection");
+        let webSocket = new MockWebSocket();
+        spyOn(RegionConnection, "buildSocket").and.returnValue(webSocket);
     }));
 
     // Create a new scope before each test.
diff --git a/src/maasserver/static/js/angular/factories/region.js b/src/maasserver/static/js/angular/factories/region.js
index 9e1e8a9..eaa78ca 100644
--- a/src/maasserver/static/js/angular/factories/region.js
+++ b/src/maasserver/static/js/angular/factories/region.js
@@ -1,4 +1,4 @@
-/* Copyright 2015-2016 Canonical Ltd.  This software is licensed under the
+/* Copyright 2015-2018 Canonical Ltd.  This software is licensed under the
  * GNU Affero General Public License version 3 (see the file LICENSE).
  *
  * MAAS Region Connection
@@ -7,34 +7,58 @@
  * service.
  */
 
+// Message types
+const MSG_TYPE = {
+    REQUEST: 0,
+    RESPONSE: 1,
+    NOTIFY: 2,
+    PING: 3,
+    PING_REPLY: 4
+};
+
+// Response types
+const RESPONSE_TYPE = {
+    SUCCESS: 0,
+    ERROR: 1
+};
+
+const REGION_STATE = {
+    DOWN: 0,
+    UP: 1,
+    RETRY: 2
+};
+
 angular.module('MAAS').factory(
-    'RegionConnection',
-    ['$q', '$rootScope', '$timeout', '$window', '$cookies', function(
-        $q, $rootScope, $timeout, $window, $cookies) {
-
-        // Message types
-        var MSG_TYPE = {
-            REQUEST: 0,
-            RESPONSE: 1,
-            NOTIFY: 2
-        };
-
-        // Response types
-        var RESPONSE_TYPE = {
-            SUCCESS: 0,
-            ERROR: 1
-        };
-
-        // Constructor
-        function RegionConnection() {
+    'RegionConnection', [
+        '$q', '$rootScope', '$timeout', '$window', '$cookies', 'LogService',
+        function() { return new RegionConnection(...arguments); }
+    ]
+);
+
+    class RegionConnection {
+
+        constructor($q, $rootScope, $timeout, $window, $cookies, LogService) {
+            this.$q = $q;
+            this.$rootScope = $rootScope;
+            this.$timeout = $timeout;
+            this.$window = $window;
+            this.$cookies = $cookies;
+            this.log = LogService;
+
+            // Expose this constant so the tests can access it.
+            this.STATE = REGION_STATE;
+
             this.callbacks = {};
             this.requests = {};
+            this.pingsInFlight = new Set();
             this.requestId = 0;
             this.url = null;
             this.websocket = null;
-            this.connected = false;
-            this.autoReconnect = true;
-            this.retryTimeout = 5000;
+            this.state = REGION_STATE.DOWN;
+            this.ensureConnectionPromise = null;
+            this.connectionCheckInterval = 5000;
+            this.maxMissedPings = 5;
+            this.maxPatience = 5;
             this.error = null;
 
             // Defer used for defaultConnect. If defaultConnect is called
@@ -58,13 +82,13 @@ angular.module('MAAS').factory(
         }
 
         // Return a new request id.
-        RegionConnection.prototype.newRequestId = function() {
+        newRequestId() {
             this.requestId += 1;
             return this.requestId;
-        };
+        }
 
         // Register event handler.
-        RegionConnection.prototype.registerHandler = function (name, func) {
+        registerHandler(name, func) {
             if(!angular.isDefined(this.handlers[name])) {
                 throw new Error("Invalid handler: " + name);
             }
@@ -72,21 +96,21 @@ angular.module('MAAS').factory(
                 throw new Error("Requires a function to register a handler.");
             }
             this.handlers[name].push(func);
-        };
+        }
 
         // Unregister event handler.
-        RegionConnection.prototype.unregisterHandler = function (name, func) {
+        unregisterHandler(name, func) {
             if(!angular.isDefined(this.handlers[name])) {
                 throw new Error("Invalid handler: " + name);
             }
-            var idx = this.handlers[name].indexOf(func);
+            let idx = this.handlers[name].indexOf(func);
             if(idx >= 0) {
                 this.handlers[name].splice(idx, 1);
             }
-        };
+        }
 
         // Register notification handler.
-        RegionConnection.prototype.registerNotifier = function(name, func) {
+        registerNotifier(name, func) {
             if(!angular.isFunction(func)) {
                 throw new Error("Requires a function to register a notifier.");
             }
@@ -94,96 +118,173 @@ angular.module('MAAS').factory(
                 this.notifiers[name] = [];
             }
             this.notifiers[name].push(func);
-        };
+        }
 
         // Unregister notification handler.
-        RegionConnection.prototype.unregisterNotifier = function(name, func) {
+        unregisterNotifier(name, func) {
             if(angular.isUndefined(this.notifiers[name])) {
                 return;
             }
-            var idx = this.notifiers[name].indexOf(func);
+            let idx = this.notifiers[name].indexOf(func);
             if(idx >= 0) {
                 this.notifiers[name].splice(idx, 1);
             }
-        };
+        }
 
         // Return True if currently connected to region.
-        RegionConnection.prototype.isConnected = function() {
-            return this.connected;
-        };
+        isConnected() {
+            return this.state === REGION_STATE.UP;
+        }
 
         // Builds the websocket connection.
-        RegionConnection.prototype.buildSocket = function(url) {
+        buildSocket(url) {
             return new WebSocket(url);
-        };
+        }
+
+        send(request) {
+            if(this.state !== REGION_STATE.UP) {
+                this.log.warn(
+                    "Sent request while region connection not available:",
+                    request);
+            }
+            // XXX mpontillo 2018-11-19: really we should handle errors here
+            // in a more robust way, but that's a bigger change.
+            this.websocket.send(angular.toJson(request));
+        }
+
+        ping() {
+            let request_id = this.newRequestId();
+            let request = {
+                type: MSG_TYPE.PING,
+                request_id: request_id,
+            };
+            this.pingsInFlight.add(request_id);
+            this.send(request);
+        }
+
+        scheduleEnsureConnection() {
+            if(this.ensureConnectionPromise) {
+                this.$timeout.cancel(this.ensureConnectionPromise);
+                this.ensureConnectionPromise = null;
+            }
+            this.ensureConnectionPromise = this.$timeout(
+                this.ensureConnection.bind(this),
+                this.connectionCheckInterval);
+        }
+
+        ensureConnection() {
+            this.scheduleEnsureConnection();
+            if(this.state === REGION_STATE.UP) {
+                let missedPings = this.pingsInFlight.size;
+                let outstandingRequests = Object.keys(this.requests).length;
+                let impatienceFactor = missedPings + outstandingRequests;
+                if(missedPings > 0) {
+                    this.log.debug(
+                        "Still waiting for ping replies: ",
+                        this.pingsInFlight);
+                }
+                if(missedPings >= this.maxMissedPings) {
+                    // Assume the connection has timed out.
+                    this.log.warn("Closed connection: ping timeout.");
+                    this.retry();
+                } else if(missedPings > 0 &&
+                        impatienceFactor >= this.maxPatience) {
+                    this.log.warn(
+                        "Closed connection: impatience factor exceeded.");
+                    this.retry();
+                }
+                else {
+                    this.ping();
+                }
+            } else if(this.state === REGION_STATE.RETRY) {
+                // We know we're already in RETRY state, but call retry()
+                // again to ensure we don't continue to create additional
+                // WebSocket objects (without closing them).
+                this.retry();
+                this.log.debug("Attempting to reconnect...");
+                this.connect();
+            }
+        }
 
         // Opens the websocket connection.
-        RegionConnection.prototype.connect = function() {
+        connect() {
             this.url = this._buildUrl();
-            this.autoReconnect = true;
             this.websocket = this.buildSocket(this.url);
 
-            var self = this;
-            this.websocket.onopen = function(evt) {
-                self.connected = true;
-                angular.forEach(self.handlers.open, function(func) {
+            this.websocket.onopen = (evt) => {
+                this.state = REGION_STATE.UP;
+                this.pingsInFlight.clear();
+                this.scheduleEnsureConnection();
+                angular.forEach(this.handlers.open, (func) => {
                     func(evt);
                 });
+                this.log.debug("WebSocket connection established.");
             };
-            this.websocket.onerror = function(evt) {
-                angular.forEach(self.handlers.error, function(func) {
+            this.websocket.onerror = (evt) => {
+                this.log.error("WebSocket error: ", evt);
+                angular.forEach(this.handlers.error, (func) => {
                     func(evt);
                 });
             };
-            this.websocket.onclose = function(evt) {
-                self.connected = false;
-                self.error = "Unable to connect to: " + self.url.split("?")[0];
-                angular.forEach(self.handlers.close, function(func) {
+            this.websocket.onclose = (evt) => {
+                let url = this.url.split("?")[0];
+                this.log.warn("WebSocket connection closed: " + url);
+                angular.forEach(this.handlers.close, (func) => {
                     func(evt);
                 });
-                if(self.autoReconnect) {
-                    $timeout(function() {
-                        self.connect();
-                    }, self.retryTimeout);
-                }
+                this.websocket = null;
+                this.retry();
             };
-            this.websocket.onmessage = function(evt) {
-                self.onMessage(angular.fromJson(evt.data));
+            this.websocket.onmessage = (evt) => {
+                this.onMessage(angular.fromJson(evt.data));
             };
-        };
+        }
 
-        // Closes the websocket connection.
-        RegionConnection.prototype.close = function() {
-            this.autoReconnect = false;
-            this.websocket.close();
-            this.websocket = null;
-        };
+        // Closes the websocket connection and begin to retry the connection.
+        retry() {
+            // Ensure the socket is closed, if applicable. If the WebSocket
+            // is already null (such as what might happen if this method is
+            // called from the onclose() hanlder), no need to close it again.
+            if(this.websocket) {
+                // Clear out event handlers. By now we already know the
+                // websocket is closed; we don't want stray events from the
+                // now-dead websocket to affect the existing connection.
+                this.websocket.onopen = null;
+                this.websocket.onerror = null;
+                this.websocket.onclose = null;
+                this.websocket.close();
+                this.websocket = null;
+            }
+            // Set the state to RETRY so that ensureConnection will attempt
+            // to reconnect the next time it runs.
+            this.state = REGION_STATE.RETRY;
+        }
 
         // Return the protocol used for the websocket connection.
-        RegionConnection.prototype._getProtocol = function() {
-            return $window.location.protocol;
-        };
+        _getProtocol() {
+            return this.$window.location.protocol;
+        }
 
         // Return connection url to websocket from current location and
         // html options.
-        RegionConnection.prototype._buildUrl = function() {
-            var host = $window.location.hostname;
-            var port = $window.location.port;
-            var path = $window.location.pathname;
-            var proto = 'ws';
+        _buildUrl() {
+            let host = this.$window.location.hostname;
+            let port = this.$window.location.port;
+            let path = this.$window.location.pathname;
+            let proto = 'ws';
             if (this._getProtocol() === 'https:') {
                 proto = 'wss';
             }
 
             // Path and port can be overridden by href and data-websocket-port
             // in the base element respectively.
-            var base = angular.element("base");
+            let base = angular.element("base");
             if(angular.isDefined(base)) {
-                var newPath = base.attr("href");
+                let newPath = base.attr("href");
                 if(angular.isDefined(newPath)) {
                     path = newPath;
                 }
-                var newPort = base.data("websocket-port");
+                let newPort = base.data("websocket-port");
                 if(angular.isDefined(newPort)) {
                     port = newPort;
                 }
@@ -195,122 +296,130 @@ angular.module('MAAS').factory(
             }
 
             // Build the URL. Include the :port only if it has a value.
-            url = proto + "://" + host;
-            if(angular.isString(port) && port.length > 0){
+            let url = proto + "://" + host;
+            if(angular.isString(port) && port.length > 0) {
                 url += ":" + port;
             }
             url += path + "ws";
 
             // Include the csrftoken in the URL if it's defined.
-            var csrftoken;
-            if(angular.isFunction($cookies.get)) {
-                csrftoken = $cookies.get('csrftoken');
+            let csrftoken;
+            if(angular.isFunction(this.$cookies.get)) {
+                csrftoken = this.$cookies.get('csrftoken');
             } else {
-                csrftoken = $cookies.csrftoken;
+                csrftoken = this.$cookies.csrftoken;
             }
             if(angular.isDefined(csrftoken)) {
                 url += '?csrftoken=' + encodeURIComponent(csrftoken);
             }
 
             return url;
-        };
+        }
 
         // Opens the default websocket connection.
-        RegionConnection.prototype.defaultConnect = function() {
+        defaultConnect() {
             // Already been called but the connection has not been completed.
             if(angular.isObject(this.defaultConnectDefer)) {
                 return this.defaultConnectDefer.promise;
             }
 
             // Already connected.
-            var defer;
+            let defer;
             if(this.isConnected()) {
                 // Create a new defer as the defaultConnectDefer would
                 // have already been resolved.
-                defer = $q.defer();
+                defer = this.$q.defer();
 
                 // Cannot resolve the defer inline as it hasn't been given
                 // back to the caller. It will be called in the next loop.
-                $timeout(defer.resolve);
+                this.$timeout(defer.resolve);
                 return defer.promise;
             }
 
             // Start the connection.
-            var self = this, opened, errored;
-            defer = this.defaultConnectDefer = $q.defer();
-            opened = function(evt) {
+            defer = this.defaultConnectDefer = this.$q.defer();
+            let opened = (evt) => {
                 this.defaultConnectDefer = null;
-                self.unregisterHandler("open", opened);
-                self.unregisterHandler("error", errored);
-                $rootScope.$apply(defer.resolve(evt));
+                this.unregisterHandler("open", opened);
+                this.unregisterHandler("error", errored);
+                this.$rootScope.$apply(defer.resolve(evt));
             };
-            errored = function(evt) {
+            let errored = (evt) => {
                 this.defaultConnectDefer = null;
-                self.unregisterHandler("open", opened);
-                self.unregisterHandler("error", errored);
-                $rootScope.$apply(defer.reject(evt));
+                this.unregisterHandler("open", opened);
+                this.unregisterHandler("error", errored);
+                this.$rootScope.$apply(defer.reject(evt));
             };
             this.registerHandler("open", opened);
             this.registerHandler("error", errored);
             this.connect();
             return defer.promise;
-        };
+        }
 
         // Called when a message is received.
-        RegionConnection.prototype.onMessage = function(msg) {
-            // Response
+        onMessage(msg) {
+            // Synchronous response
             if(msg.type === MSG_TYPE.RESPONSE) {
                 this.onResponse(msg);
-            // Notify
+            // Asynchronous notification
             } else if(msg.type === MSG_TYPE.NOTIFY) {
                 this.onNotify(msg);
+            // Reply to connectivity check
+            } else if(msg.type === MSG_TYPE.PING_REPLY) {
+                this.onPingReply(msg);
             }
-        };
+        }
 
         // Called when a response message is recieved.
-        RegionConnection.prototype.onResponse = function(msg) {
+        onResponse(msg) {
             // Grab the registered defer from the callbacks list.
-            var defer = this.callbacks[msg.request_id];
-            var remembered_request = this.requests[msg.request_id];
+            let defer = this.callbacks[msg.request_id];
+            let remembered_request = this.requests[msg.request_id];
             if(angular.isDefined(defer)) {
                 if(msg.rtype === RESPONSE_TYPE.SUCCESS) {
                     // Resolve the defer inside of the digest cycle, so any
                     // update to an object or collection will trigger a
                     // watcher.
-                    $rootScope.$apply(defer.resolve(msg.result));
+                    this.$rootScope.$apply(defer.resolve(msg.result));
                 } else if(msg.rtype === RESPONSE_TYPE.ERROR) {
                     // Reject the defer since an error occurred.
                     if(angular.isObject(remembered_request)) {
-                        $rootScope.$apply(defer.reject({
+                        this.$rootScope.$apply(defer.reject({
                             "error": msg.error,
                             "request": remembered_request
                         }));
                     } else {
-                        $rootScope.$apply(defer.reject(msg.error));
+                        this.$rootScope.$apply(defer.reject(msg.error));
                     }
                 }
                 // Remove the defer from the callback list.
                 delete this.callbacks[msg.request_id];
                 delete this.requests[msg.request_id];
             }
-        };
+        }
 
         // Called when a notify response is recieved.
-        RegionConnection.prototype.onNotify = function(msg) {
-            var handlers = this.notifiers[msg.name];
+        onNotify(msg) {
+            let handlers = this.notifiers[msg.name];
             if(angular.isArray(handlers)) {
                 angular.forEach(handlers, function(handler) {
                     handler(msg.action, msg.data);
                 });
             }
-        };
+        }
+
+        onPingReply(msg) {
+            // Note: The msg.result at this point contains the last sequence
+            // number received, but it isn't really relevant for us. It could
+            // be helpful for debug logging, if necessary.
+            this.pingsInFlight.delete(msg.request_id);
+        }
 
         // Call method on the region.
-        RegionConnection.prototype.callMethod = function(
-                method, params, remember) {
-            var defer = $q.defer();
-            var request_id = this.newRequestId();
-            var request = {
+        callMethod(method, params, remember) {
+            let defer = this.$q.defer();
+            let request_id = this.newRequestId();
+            let request = {
                 type: MSG_TYPE.REQUEST,
                 request_id: request_id,
                 method: method,
@@ -322,9 +431,9 @@ angular.module('MAAS').factory(
             if (remember) {
                 this.requests[request_id] = request;
             }
-            this.websocket.send(angular.toJson(request));
+            this.send(request);
+            // Uncomment this to log every WebSocket method call.
+            // this.log.debug("callMethod(): ", request, remember);
             return defer.promise;
-        };
-
-        return new RegionConnection();
-    }]);
+        }
+    }
diff --git a/src/maasserver/static/js/angular/factories/tests/test_region.js b/src/maasserver/static/js/angular/factories/tests/test_region.js
index c5e6d8c..57db53a 100644
--- a/src/maasserver/static/js/angular/factories/tests/test_region.js
+++ b/src/maasserver/static/js/angular/factories/tests/test_region.js
@@ -20,7 +20,7 @@ describe("RegionConnection", function() {
     }));
 
     // Load the RegionConnection factory.
-    var RegionConnection, webSocket;
+    let RegionConnection, webSocket;
     beforeEach(inject(function($injector) {
         RegionConnection = $injector.get("RegionConnection");
 
@@ -186,17 +186,93 @@ describe("RegionConnection", function() {
 
     describe("isConnected", function() {
 
-        it("returns true", function() {
-            RegionConnection.connected = true;
+        it("returns true in STATE.UP", function() {
+            RegionConnection.state = RegionConnection.STATE.UP;
             expect(RegionConnection.isConnected()).toBe(true);
         });
 
-        it("returns false", function() {
-            RegionConnection.connected = false;
+        it("returns false in STATE.DOWN", function() {
+            RegionConnection.state = RegionConnection.STATE.DOWN;
+            expect(RegionConnection.isConnected()).toBe(false);
+        });
+
+        it("returns false in STATE.RETRY", function() {
+            RegionConnection.state = RegionConnection.STATE.RETRY;
             expect(RegionConnection.isConnected()).toBe(false);
         });
     });
 
+    describe("scheduleEnsureConnection", function() {
+
+        it("schedules ensureConnection to run", function() {
+            spyOn(RegionConnection, "ensureConnection");
+            RegionConnection.scheduleEnsureConnection();
+            $timeout.flush();
+            expect(RegionConnection.ensureConnection).toHaveBeenCalled();
+        });
+
+        it("does not schedule itself multiple times", function() {
+            spyOn(RegionConnection, "ensureConnection");
+            RegionConnection.scheduleEnsureConnection();
+            RegionConnection.scheduleEnsureConnection();
+            RegionConnection.scheduleEnsureConnection();
+            RegionConnection.scheduleEnsureConnection();
+            RegionConnection.scheduleEnsureConnection();
+            $timeout.flush();
+            expect(RegionConnection.ensureConnection).toHaveBeenCalledTimes(1);
+        });
+
+    });
+
+    describe("ensureConnection", function() {
+
+        beforeEach(function() {
+            spyOn(RegionConnection, "scheduleEnsureConnection");
+            spyOn(RegionConnection, "retry");
+            spyOn(RegionConnection, "connect");
+            spyOn(RegionConnection, "ping");
+        });
+
+        it("schedules itself to run again later", function() {
+            RegionConnection.ensureConnection();
+            expect(
+                RegionConnection.scheduleEnsureConnection).toHaveBeenCalled();
+        });
+
+        it("calls retry() if in UP state with missed pings", function() {
+            RegionConnection.state = RegionConnection.STATE.UP;
+            for(let i = 0; i < RegionConnection.maxMissedPings ; i++) {
+                RegionConnection.pingsInFlight.add(i);
+            }
+            RegionConnection.ensureConnection();
+            expect(RegionConnection.retry).toHaveBeenCalled();
+        });
+
+        it("calls retry() if UP with missed requests and pings", function() {
+            RegionConnection.state = RegionConnection.STATE.UP;
+            RegionConnection.pingsInFlight.add(1);
+            for(let i = 0; i < RegionConnection.maxPatience - 1 ; i++) {
+                RegionConnection.requests[i] = {};
+            }
+            RegionConnection.ensureConnection();
+            expect(RegionConnection.retry).toHaveBeenCalled();
+        });
+
+        it("calls ping() if in UP state", function() {
+            RegionConnection.state = RegionConnection.STATE.UP;
+            RegionConnection.ensureConnection();
+            expect(RegionConnection.ping).toHaveBeenCalled();
+        });
+
+        it("calls retry() and connect() if in RETRY state", function() {
+            RegionConnection.state = RegionConnection.STATE.RETRY;
+            RegionConnection.ensureConnection();
+            expect(RegionConnection.retry).toHaveBeenCalled();
+            expect(RegionConnection.connect).toHaveBeenCalled();
+        });
+
+    });
+
     describe("connect", function() {
 
         var url = "http://test-url", buildUrlSpy;
@@ -210,12 +286,6 @@ describe("RegionConnection", function() {
             expect(RegionConnection.url).toBe(url);
         });
 
-        it("sets autoReconnect to true", function() {
-            RegionConnection.autoReconnect = false;
-            RegionConnection.connect();
-            expect(RegionConnection.autoReconnect).toBe(true);
-        });
-
         it("calls buildSocket with url", function() {
             RegionConnection.connect();
             expect(RegionConnection.buildSocket).toHaveBeenCalledWith(url);
@@ -231,7 +301,7 @@ describe("RegionConnection", function() {
         it("sets connect to true when onopen called", function() {
             RegionConnection.connect();
             webSocket.onopen({});
-            expect(RegionConnection.connected).toBe(true);
+            expect(RegionConnection.isConnected()).toBe(true);
         });
 
         it("calls error handler when onerror called", function(done) {
@@ -244,39 +314,20 @@ describe("RegionConnection", function() {
             webSocket.onerror(evt_obj);
         });
 
-        it("sets connect to false when onclose called", function() {
-            RegionConnection.autoReconnect = false;
+        it("isConnected() is false when onclose called", function() {
             RegionConnection.connect();
             webSocket.onclose({});
-            expect(RegionConnection.connected).toBe(false);
+            expect(RegionConnection.isConnected()).toBe(false);
         });
 
         it("calls close handler when onclose called", function(done) {
             var evt_obj = {};
-            RegionConnection.autoReconnect = false;
-            RegionConnection.connect();
             RegionConnection.registerHandler("close", function(evt) {
                 expect(evt).toBe(evt_obj);
                 done();
             });
-            webSocket.onclose(evt_obj);
-        });
-
-        it("onclose calls connect when autoReconnect is true", function() {
-            RegionConnection.connect();
-            var new_url = "http://new-url";
-            buildUrlSpy.and.returnValue(new_url);
-            webSocket.onclose({});
-            $timeout.flush();
-            expect(RegionConnection.url).toBe(new_url);
-        });
-
-        it("onclose sets error", function() {
             RegionConnection.connect();
-            webSocket.onclose();
-            $timeout.flush();
-            expect(RegionConnection.error).toBe(
-                "Unable to connect to: " + url);
+            webSocket.onclose(evt_obj);
         });
 
         it("calls onMessage when onmessage called", function() {
@@ -289,29 +340,38 @@ describe("RegionConnection", function() {
         });
     });
 
-    describe("close", function() {
+    describe("retry", function() {
 
         beforeEach(function() {
             spyOn(webSocket, "close");
         });
 
-        it("sets autoReconnect to false", function() {
+        it("sets state to RETRY", function() {
             RegionConnection.connect("");
-            RegionConnection.close();
-            expect(RegionConnection.autoReconnect).toBe(false);
+            RegionConnection.retry();
+            expect(RegionConnection.state).toBe(RegionConnection.STATE.RETRY);
         });
 
         it("calls close on websocket", function() {
             RegionConnection.connect("");
-            RegionConnection.close();
+            RegionConnection.retry();
             expect(webSocket.close).toHaveBeenCalled();
         });
 
         it("sets websocket to null", function() {
             RegionConnection.connect("");
-            RegionConnection.close();
+            RegionConnection.retry();
             expect(RegionConnection.websocket).toBeNull();
         });
+
+        it("clears out event handlers", function() {
+            RegionConnection.connect("");
+            let ws = RegionConnection.websocket;
+            RegionConnection.retry();
+            expect(ws.onopen).toBeNull();
+            expect(ws.onerror).toBeNull();
+            expect(ws.onclose).toBeNull();
+        });
     });
 
     describe("_getProtocol", function() {
@@ -413,7 +473,7 @@ describe("RegionConnection", function() {
     describe("defaultConnect", function() {
 
         it("resolve defer if already connected", function(done) {
-            RegionConnection.connected = true;
+            RegionConnection.state = RegionConnection.STATE.UP;
             RegionConnection.defaultConnect().then(function() {
                 done();
             });
diff --git a/src/maasserver/static/js/angular/services/log.js b/src/maasserver/static/js/angular/services/log.js
index f2378ab..894ae03 100644
--- a/src/maasserver/static/js/angular/services/log.js
+++ b/src/maasserver/static/js/angular/services/log.js
@@ -51,7 +51,7 @@ angular.module('MAAS').service('LogService', [
                 // Add time index to the beginning of the log.
                 Array.prototype.unshift.call(args,
                     "[" + self.formatMilliseconds(self.now()) + "]");
-                destination.apply(self, args);
+                destination.apply(console, args);
             }
         };
 
diff --git a/src/maasserver/static/js/angular/testing/websocket.js b/src/maasserver/static/js/angular/testing/websocket.js
index f32fd42..54aeeb1 100644
--- a/src/maasserver/static/js/angular/testing/websocket.js
+++ b/src/maasserver/static/js/angular/testing/websocket.js
@@ -13,9 +13,13 @@ var READY_STATES = {
     CLOSED: 3
 };
 
-var MSG_TYPE = {
+// Message types (copied from region.js)
+const MSG_TYPE = {
     REQUEST: 0,
-    RESPONSE: 1
+    RESPONSE: 1,
+    NOTIFY: 2,
+    PING: 3,
+    PING_REPLY: 4
 };
 
 function MockWebSocket(url) {
@@ -26,12 +30,16 @@ function MockWebSocket(url) {
     this.onmessage = null;
     this.onopen = null;
 
+    this.replyToPing = true;
+
     this.sentData = [];
     this.returnData = [];
     this.receivedData = [];
 
+    this.sequenceNum = 0;
+
     // Simulate the connection opening.
-    var self = this;
+    let self = this;
     setTimeout(function() {
         self.readyState = READY_STATES.OPEN;
         if(angular.isFunction(self.onopen)) {
@@ -44,7 +52,7 @@ MockWebSocket.prototype.close = function() {
     this.readyState = READY_STATES.CLOSING;
 
     // Simulate the connection closing.
-    var self = this;
+    let self = this;
     setTimeout(function() {
         self.readyState = READY_STATES.CLOSED;
         if(angular.isFunction(self.onclose)) {
@@ -54,6 +62,26 @@ MockWebSocket.prototype.close = function() {
 };
 
 MockWebSocket.prototype.send = function(data) {
+    let sentObject = angular.fromJson(data);
+    let sentType = sentObject.type;
+    let sentId = sentObject.request_id;
+    let self = this;
+
+    // Special case for PING type; just reply and return.
+    if(self.replyToPing && angular.isNumber(sentType) &&
+       sentType === MSG_TYPE.PING) {
+        setTimeout(function() {
+            self.sequenceNum += 1;
+            let pingResultData = angular.toJson({
+                type: MSG_TYPE.PING_REPLY,
+                request_id: sentId,
+                result: self.sequenceNum
+            });
+            self.onmessage({ data: pingResultData });
+        });
+        return;
+    }
+
     this.sentData.push(data);
 
     // Exit early if no fake data to return.
@@ -68,13 +96,9 @@ MockWebSocket.prototype.send = function(data) {
         receivedData = [receivedData];
     }
 
-    var self = this;
 
     // Send the response
     setTimeout(function() {
-        var sentObject = angular.fromJson(data);
-        var sentType = sentObject.type;
-        var sentId = sentObject.request_id;
         if(angular.isNumber(sentType) && angular.isNumber(sentId)) {
             // Patch the request_id so the response is the
             // same as the request.
diff --git a/webpack.config.js b/webpack.config.js
index 9880aff..b02d7ce 100644
--- a/webpack.config.js
+++ b/webpack.config.js
@@ -27,6 +27,7 @@ module.exports = {
     plugins: [
         new UglifyJSPlugin({
             parallel: true,
+            cache: true,
             sourceMap: true,
             uglifyOptions: {
                 // Using the 'mangle' option breaks the Angular injector.