Merge lp:~mpontillo/maas/add-logging-for-bug-1627362 into lp:~maas-committers/maas/trunk

[Mike Pontillo]

Add logging for bug #1627362, so we can attempt to determine the root cause.

[lp:~mpontillo/maas/add-logging-for-bug-1627362]

5431. By Mike Pontillo on 2016-10-04

Fix lint.

[Gavin Panella (allenap)]

Looks good. Some comments but no blockers.

review: Approve

[Mike Pontillo (mpontillo)]

We may want to come back to this if we have trouble finding the root cause, but for now it won't be needed. The problem seems to be in the websocket handler's batch handling.

Unmerged revisions

5432. By Mike Pontillo on 2016-10-05

Add debug logging.

5431. By Mike Pontillo on 2016-10-04

Fix lint.

5430. By Mike Pontillo on 2016-10-04

Add logging for bug #1627362.

=== modified file 'src/maasserver/monkey.py'
--- src/maasserver/monkey.py	2016-08-19 10:22:29 +0000
+++ src/maasserver/monkey.py	2016-10-04 14:12:19 +0000
@@ -7,14 +7,69 @@
 
 __all__ = [
     "add_patches_to_django",
+    "orig_parse_datetime"
 ]
 
+import datetime
 import re
 
+from provisioningserver.logger import get_maas_logger
+
+
+maaslog = get_maas_logger("maassever.monkey")
 
 fixed_re = re.compile(r"^([a-z0-9.-]+|\[[a-f0-9]*:[a-f0-9:\.]+\])(:\d+)?$")
 
 
+def _parse_datetime_init(value, *args, **kwargs):
+    """Initializer for the orig_parse_datetime global.
+
+    This is just in case this method somehow gets called before it's
+    initialized, which should never happen.
+    """
+    raise NotImplementedError("Not patched yet.")
+
+# Global variable which will be assigned to the original Django parse_datetime.
+orig_parse_datetime = _parse_datetime_init
+
+
+def maas_parse_datetime(value, *args, **kwargs):
+    """Patch for bug #1627362.
+
+    Also logs the error so we can attempt to determine the root cause.
+    """
+    try:
+        orig_parse_datetime(value, args, kwargs)
+    except TypeError as e:
+        # Check if this is the known-problematic TypeError.
+        if str(e) == "expected string or bytes-like object":
+            maaslog.warning(
+                "[Bug #1627362] Django parse_datetime expected bytes/str; got "
+                "'%s': %r. (If you see this message, please leave a comment "
+                "at https://launchpad.net/bugs/1627362 if it has not yet been "
+                "resolved.)" % (str(type(value), value)))
+            # This may or may not work around the issue, but it's going to be
+            # better than crashing. In MAAS, DateTimeField is use for created
+            # and updated values (on EVERY model), commissioning results
+            # timestamps, the Django `auth_user` model (date_joined,
+            # last_login), and the Discovery model (last_seen and first_seen).
+            # Although None is a valid return value for this function, other
+            # code expects these values to be NOT NULL. Setting this to 1970 so
+            # that users who encounter this bug may notice something strange,
+            # but will not necessarily crash their MAAS.
+            return datetime.datetime(1970, 1, 1, 1, 0)
+        else:
+            # If TypeError occurred for some other reason, just return None
+            # here so we can investigate the crash. This is actually a
+            # different bug.
+            maaslog.warning(
+                "[Bug #1627362] TypeError (%s); got '%s': %r. (If you see this"
+                "message, please leave a comment at "
+                "https://launchpad.net/bugs/1627362 if it has not yet been "
+                "resolved.)" % (str(type(value), value)))
+            return None
+
+
 def fix_django_http_request():
     """Add support for ipv6-formatted ipv4 addresses to django requests.
 
@@ -25,5 +80,18 @@
         django.http.request.host_validation_re = fixed_re
 
 
+def fix_django_parse_datetime():
+    import django.utils.dateparse
+    try:
+        django.utils.dateparse.parse_datetime(None)
+    except TypeError:
+        # The API indicates that None should be returned in the event of an
+        # error. (A TypeError here is incorrect.)
+        global orig_parse_datetime
+        orig_parse_datetime = django.utils.dateparse.parse_datetime
+        django.utils.dateparse.parse_datetime = maas_parse_datetime
+
+
 def add_patches_to_django():
     fix_django_http_request()
+    fix_django_parse_datetime()