Merge lp:~danilo/maas/django-upgrade-method-fixes into lp:~maas-committers/maas/trunk

[Данило Шеган]

Stop using settings, methods, functions and attributes that have been dropped between Django 1.8 and 1.11, fix up issues in our code that surfaced since 1.11 is stricter.

Django has gotten stricter between 1.8 and 1.11, which highlighted a few things in our code:

 - we were passing path to BlockDevice/PhysicalBlockDevice models in tests, yet this was a read-only property on the model for some time already
 - PackageRepository has long lost the "description" field, yet Django silently ignored us setting it in the forms

The rest of the changes are basically following the suggested upgrade paths for Django code.

Dependencies
============

To test with Django 1.11, you need to add the following two PPAs:
  sudo apt-add-repository ppa:danilo/django-1.11
  sudo apt-add-repository ppa:danilo/django-piston3

[Mike Pontillo (mpontillo)]

Looks good. One minor nit below that I won't block you on.

I tested this branch with my local MAAS (on Django 1.8), so I'm pretty sure this doesn't cause a regression.

review: Approve

=== modified file 'docs/development/notifications.rst'
--- docs/development/notifications.rst	2017-02-15 11:24:50 +0000
+++ docs/development/notifications.rst	2017-06-16 07:52:25 +0000
@@ -184,7 +184,7 @@
 
 should be done with ``find_for_user``:
 
-  >>> Notification.objects.find_for_user(user)
+  >>> list(Notification.objects.find_for_user(user))
   [<Notification ...]
 
 

=== modified file 'src/apiclient/testing/django.py'
--- src/apiclient/testing/django.py	2015-12-01 18:12:59 +0000
+++ src/apiclient/testing/django.py	2017-06-16 07:52:25 +0000
@@ -23,7 +23,8 @@
 
 
 # Django 1.6 needs to be configured before we can import WSGIRequest.
-settings.configure(DEBUG=True)
+if not settings.configured:
+    settings.configure(DEBUG=True)
 
 
 class Base64Decoder:
@@ -65,6 +66,10 @@
     """
     handler = MemoryFileUploadHandler()
     meta = {
+        "CONTENT_TYPE": headers["Content-Type"],
+        "CONTENT_LENGTH": headers["Content-Length"],
+        # To make things even more entertaining, 1.8 prefixed meta vars
+        # with "HTTP_" and 1.11 does not.
         "HTTP_CONTENT_TYPE": headers["Content-Type"],
         "HTTP_CONTENT_LENGTH": headers["Content-Length"],
         }

=== modified file 'src/maasserver/api/doc_handler.py'
--- src/maasserver/api/doc_handler.py	2017-01-28 00:51:47 +0000
+++ src/maasserver/api/doc_handler.py	2017-06-16 07:52:25 +0000
@@ -66,8 +66,7 @@
 from textwrap import dedent
 
 from django.http import HttpResponse
-from django.shortcuts import render_to_response
-from django.template import RequestContext
+from django.shortcuts import render
 from docutils import core
 from maasserver.api.doc import (
     describe_api,
@@ -172,10 +171,9 @@
     # Generate the documentation and keep it cached.  Note that we can't do
     # that at the module level because the API doc generation needs Django
     # fully initialized.
-    return render_to_response(
-        'maasserver/api_doc.html',
-        {'doc': reST_to_html_fragment(render_api_docs())},
-        context_instance=RequestContext(request))
+    return render(
+        request, 'maasserver/api_doc.html',
+        {'doc': reST_to_html_fragment(render_api_docs())})
 
 
 def describe(request):

=== modified file 'src/maasserver/api/tests/test_doc.py'
--- src/maasserver/api/tests/test_doc.py	2017-01-28 00:51:47 +0000
+++ src/maasserver/api/tests/test_doc.py	2017-06-16 07:52:25 +0000
@@ -15,7 +15,6 @@
 
 from django.conf.urls import (
     include,
-    patterns,
     url,
 )
 from django.core.exceptions import ImproperlyConfigured
@@ -84,7 +83,7 @@
     def test_urlpatterns_empty(self):
         # No resources are found in empty modules.
         module = self.make_module()
-        module.urlpatterns = patterns("")
+        module.urlpatterns = []
         self.assertSetEqual(set(), find_api_resources(module))
 
     def test_urlpatterns_not_present(self):
@@ -95,7 +94,7 @@
     def test_urlpatterns_with_resource_for_incomplete_handler(self):
         # Resources for handlers that don't specify resource_uri are ignored.
         module = self.make_module()
-        module.urlpatterns = patterns("", url("^foo", BaseHandler))
+        module.urlpatterns = [url("^foo", BaseHandler)]
         self.assertSetEqual(set(), find_api_resources(module))
 
     def test_urlpatterns_with_resource(self):
@@ -105,7 +104,7 @@
         handler = type("\m/", (BaseHandler,), {"resource_uri": True})
         resource = Resource(handler)
         module = self.make_module()
-        module.urlpatterns = patterns("", url("^metal", resource))
+        module.urlpatterns = [url("^metal", resource)]
         self.assertSetEqual({resource}, find_api_resources(module))
 
     def test_urlpatterns_with_resource_hidden(self):
@@ -117,7 +116,7 @@
             })
         resource = Resource(handler)
         module = self.make_module()
-        module.urlpatterns = patterns("", url("^metal", resource))
+        module.urlpatterns = [url("^metal", resource)]
         self.assertSetEqual(set(), find_api_resources(module))
 
     def test_nested_urlpatterns_with_handler(self):
@@ -126,8 +125,8 @@
         resource = Resource(handler)
         module = self.make_module()
         submodule = self.make_module()
-        submodule.urlpatterns = patterns("", url("^metal", resource))
-        module.urlpatterns = patterns("", ("^genre/", include(submodule)))
+        submodule.urlpatterns = [url("^metal", resource)]
+        module.urlpatterns = [url("^genre/", include(submodule))]
         self.assertSetEqual({resource}, find_api_resources(module))
 
     def test_smoke(self):

=== modified file 'src/maasserver/djangosettings/settings.py'
--- src/maasserver/djangosettings/settings.py	2017-05-25 13:19:34 +0000
+++ src/maasserver/djangosettings/settings.py	2017-06-16 07:52:25 +0000
@@ -187,25 +187,31 @@
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = 'zk@qw+fdhu_b4ljx+pmb*8sju4lpx!5zkez%&4hep_(o6y1nf0'
 
-# List of callables that know how to import templates from various sources.
-TEMPLATE_LOADERS = (
-    'django.template.loaders.filesystem.Loader',
-    'django.template.loaders.app_directories.Loader',
-    # 'django.template.loaders.eggs.Loader',
-)
-
-TEMPLATE_CONTEXT_PROCESSORS = (
-    "django.contrib.auth.context_processors.auth",
-    "django.core.context_processors.debug",
-    "django.core.context_processors.i18n",
-    "django.core.context_processors.request",
-    "django.core.context_processors.media",
-    "django.core.context_processors.static",
-    # "django.core.context_processors.tz",
-    "django.contrib.messages.context_processors.messages",
-    "maasserver.context_processors.yui",
-    "maasserver.context_processors.global_options",
-)
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [
+            os.path.join(os.path.dirname(__file__), "templates"),
+        ],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                # Insert your TEMPLATE_CONTEXT_PROCESSORS here or use this
+                # list if you haven't customized them:
+                "django.contrib.auth.context_processors.auth",
+                "django.template.context_processors.debug",
+                "django.template.context_processors.i18n",
+                "django.template.context_processors.media",
+                "django.template.context_processors.request",
+                "django.template.context_processors.static",
+                # "django.template.context_processors.tz",
+                "django.contrib.messages.context_processors.messages",
+                "maasserver.context_processors.yui",
+                "maasserver.context_processors.global_options",
+            ],
+        },
+    },
+]
 
 MIDDLEWARE_CLASSES = (
 
@@ -265,14 +271,6 @@
 
 ROOT_URLCONF = 'maasserver.djangosettings.urls'
 
-TEMPLATE_DIRS = (
-    # Put strings here, like "/home/html/django_templates"
-    # or "C:/www/django/templates".
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
-    os.path.join(os.path.dirname(__file__), "templates"),
-)
-
 SOUTH_MIGRATION_MODULES = {
     # Migrations before MAAS 2.0 are located in south sub-directory.
     'maasserver': 'maasserver.migrations.south.migrations',
@@ -350,6 +348,10 @@
     'maasjson': 'maasserver.json',
 }
 
+# MAAS has no upload limit to allow for big image files.
+# (Django 1.10 introduced this limit with a default of 2.5MB.)
+DATA_UPLOAD_MAX_MEMORY_SIZE = None
+
 # Patch the get_script_prefix method to allow twisted to work with django.
 patch_get_script_prefix()
 

=== modified file 'src/maasserver/forms/__init__.py'
--- src/maasserver/forms/__init__.py	2017-06-13 11:27:29 +0000
+++ src/maasserver/forms/__init__.py	2017-06-16 07:52:25 +0000
@@ -1512,7 +1512,6 @@
             name='main_archive',
             defaults={
                 'url': self.cleaned_data['main_archive'],
-                'description': 'main_archive',
                 'arches': PackageRepository.MAIN_ARCHES,
                 'default': True,
                 'enabled': True})
@@ -1520,7 +1519,6 @@
             name='ports_archive',
             defaults={
                 'url': self.cleaned_data['ports_archive'],
-                'description': 'ports_archive',
                 'arches': PackageRepository.PORTS_ARCHES,
                 'default': True,
                 'enabled': True})

=== modified file 'src/maasserver/models/largefile.py'
--- src/maasserver/models/largefile.py	2016-10-28 15:58:32 +0000
+++ src/maasserver/models/largefile.py	2017-06-16 07:52:25 +0000
@@ -149,8 +149,9 @@
         referencing this object.
         """
         links = [
-            rel.get_accessor_name()
-            for rel in self._meta.get_all_related_objects()
+            f.get_accessor_name() for f in self._meta.get_fields()
+            if ((f.one_to_many or f.one_to_one) and
+                f.auto_created and not f.concrete)
         ]
         for link in links:
             if getattr(self, link).exists():

=== modified file 'src/maasserver/models/tests/test_blockdevice.py'
--- src/maasserver/models/tests/test_blockdevice.py	2017-04-04 21:07:13 +0000
+++ src/maasserver/models/tests/test_blockdevice.py	2017-06-16 07:52:25 +0000
@@ -391,25 +391,25 @@
 
     def test_negative_size(self):
         node = factory.make_Node()
-        blockdevice = BlockDevice(node=node, name='sda', path='/dev/sda',
+        blockdevice = BlockDevice(node=node, name='sda',
                                   block_size=512, size=-1)
         self.assertRaises(ValidationError, blockdevice.save)
 
     def test_minimum_size(self):
         node = factory.make_Node()
-        blockdevice = BlockDevice(node=node, name='sda', path='/dev/sda',
+        blockdevice = BlockDevice(node=node, name='sda',
                                   block_size=512, size=143359)
         self.assertRaises(ValidationError, blockdevice.save)
 
     def test_negative_block_device_size(self):
         node = factory.make_Node()
-        blockdevice = BlockDevice(node=node, name='sda', path='/dev/sda',
+        blockdevice = BlockDevice(node=node, name='sda',
                                   block_size=-1, size=143360)
         self.assertRaises(ValidationError, blockdevice.save)
 
     def test_minimum_block_device_size(self):
         node = factory.make_Node()
-        blockdevice = BlockDevice(node=node, name='sda', path='/dev/sda',
+        blockdevice = BlockDevice(node=node, name='sda',
                                   block_size=511, size=143360)
         self.assertRaises(ValidationError, blockdevice.save)
 

=== modified file 'src/maasserver/models/tests/test_physicalblockdevice.py'
--- src/maasserver/models/tests/test_physicalblockdevice.py	2015-12-01 18:12:59 +0000
+++ src/maasserver/models/tests/test_physicalblockdevice.py	2017-06-16 07:52:25 +0000
@@ -20,33 +20,31 @@
     def test_model_serial_and_no_id_path_requirements_should_save(self):
         node = factory.make_Node()
         blockdevice = PhysicalBlockDevice(
-            node=node, name='sda',
-            path='/dev/sda', block_size=512,
-            size=MIN_BLOCK_DEVICE_SIZE, model='A2M0003',
-            serial='001')
+            node=node, name='sda', block_size=512, size=MIN_BLOCK_DEVICE_SIZE,
+            model='A2M0003', serial='001')
         # Should work without issue
         blockdevice.save()
 
     def test_id_path_and_no_model_serial_requirements_should_save(self):
         node = factory.make_Node()
         blockdevice = PhysicalBlockDevice(
-            node=node, name='sda', path='/dev/sda', block_size=512,
-            size=MIN_BLOCK_DEVICE_SIZE, id_path='/dev/disk/by-id/A2M0003-001')
+            node=node, name='sda', block_size=512, size=MIN_BLOCK_DEVICE_SIZE,
+            id_path='/dev/disk/by-id/A2M0003-001')
         # Should work without issue
         blockdevice.save()
 
     def test_no_id_path_and_no_serial(self):
         node = factory.make_Node()
         blockdevice = PhysicalBlockDevice(
-            node=node, name='sda', path='/dev/sda', block_size=512,
-            size=MIN_BLOCK_DEVICE_SIZE, model='A2M0003')
+            node=node, name='sda', block_size=512, size=MIN_BLOCK_DEVICE_SIZE,
+            model='A2M0003')
         self.assertRaises(ValidationError, blockdevice.save)
 
     def test_no_id_path_and_no_model(self):
         node = factory.make_Node()
         blockdevice = PhysicalBlockDevice(
-            node=node, name='sda', path='/dev/sda', block_size=512,
-            size=MIN_BLOCK_DEVICE_SIZE, serial='001')
+            node=node, name='sda', block_size=512, size=MIN_BLOCK_DEVICE_SIZE,
+            serial='001')
         self.assertRaises(ValidationError, blockdevice.save)
 
     def test_number_of_physical_devices_for_returns_correct_count(self):

=== modified file 'src/maasserver/urls.py'
--- src/maasserver/urls.py	2017-06-01 06:47:23 +0000
+++ src/maasserver/urls.py	2017-06-16 07:52:25 +0000
@@ -12,6 +12,10 @@
 )
 from django.contrib.auth.decorators import user_passes_test
 from django.http import HttpResponse
+from maasserver import (
+    urls_api,
+    urls_combo,
+)
 from maasserver.bootresources import (
     simplestreams_file_handler,
     simplestreams_stream_handler,
@@ -62,7 +66,7 @@
 # # URLs accessible to anonymous users.
 # Combo URLs.
 urlpatterns = [
-    url(r'combo/', include('maasserver.urls_combo'))
+    url(r'combo/', include(urls_combo))
 ]
 
 # Anonymous views.
@@ -160,7 +164,7 @@
 
 # API URLs. If old API requested, provide error message directing to new API.
 urlpatterns += [
-    url(r'^api/2\.0/', include('maasserver.urls_api')),
+    url(r'^api/2\.0/', include(urls_api)),
     url(r'^api/version/', lambda request: HttpResponse(
         content='2.0', content_type="text/plain"), name='api_version'),
     url(r'^api/1.0/', lambda request: HttpResponse(

=== modified file 'src/maasserver/utils/__init__.py'
--- src/maasserver/utils/__init__.py	2016-08-31 08:25:07 +0000
+++ src/maasserver/utils/__init__.py	2017-06-16 07:52:25 +0000
@@ -86,10 +86,7 @@
     if not abs_path.endswith('/'):
         # Add trailing '/' to get urljoin to behave.
         abs_path = abs_path + '/'
-    # Force prefix to be '' so that Django doesn't use the 'script prefix' (
-    # which might be there or not depending on whether or not the thread local
-    # variable has been initialized).
-    reverse_link = reverse(view_name, prefix='', *args, **kwargs)
+    reverse_link = reverse(view_name, *args, **kwargs)
     if reverse_link.startswith('/'):
         # Drop the leading '/'.
         reverse_link = reverse_link[1:]

=== modified file 'src/maasserver/utils/tests/test_views.py'
--- src/maasserver/utils/tests/test_views.py	2017-03-16 16:40:36 +0000
+++ src/maasserver/utils/tests/test_views.py	2017-06-16 07:52:25 +0000
@@ -30,7 +30,6 @@
 from django.core.urlresolvers import get_resolver
 from django.db import connection
 from django.http import HttpResponse
-from django.http.response import REASON_PHRASES
 from fixtures import FakeLogger
 from maasserver.testing.factory import factory
 from maasserver.testing.testcase import (
@@ -387,7 +386,7 @@
         self.expectThat(response.status_code, Equals(http.client.CONFLICT))
         self.expectThat(
             response.reason_phrase,
-            Equals(REASON_PHRASES[http.client.CONFLICT]))
+            Equals(http.client.responses[http.client.CONFLICT]))
 
     def test__get_response_prepare_retry_context_before_each_try(self):
 

=== modified file 'src/maasserver/utils/views.py'
--- src/maasserver/utils/views.py	2017-03-16 16:40:36 +0000
+++ src/maasserver/utils/views.py	2017-06-16 07:52:25 +0000
@@ -117,6 +117,7 @@
 
 class HttpResponseConflict(MAASDjangoTemplateResponse):
     status_code = int(http.client.CONFLICT)
+    reason_phrase = http.client.responses[http.client.CONFLICT]
 
 
 class WebApplicationHandler(WSGIHandler):

=== modified file 'src/maasserver/views/account.py'
--- src/maasserver/views/account.py	2017-03-20 09:45:43 +0000
+++ src/maasserver/views/account.py	2017-06-16 07:52:25 +0000
@@ -26,8 +26,7 @@
     HttpResponseRedirect,
     JsonResponse,
 )
-from django.shortcuts import render_to_response
-from django.template import RequestContext
+from django.shortcuts import render
 from maasserver.models import UserProfile
 from maasserver.models.user import (
     create_auth_token,
@@ -70,11 +69,8 @@
     else:
         form = LogoutForm()
 
-    return render_to_response(
-        'maasserver/logout_confirm.html',
-        {'form': form},
-        context_instance=RequestContext(request),
-    )
+    return render(
+        request, 'maasserver/logout_confirm.html', {'form': form})
 
 
 def authenticate(request):

=== modified file 'src/maasserver/views/prefs.py'
--- src/maasserver/views/prefs.py	2016-09-26 06:36:07 +0000
+++ src/maasserver/views/prefs.py	2017-06-16 07:52:25 +0000
@@ -13,9 +13,8 @@
 from django.core.urlresolvers import reverse
 from django.shortcuts import (
     get_object_or_404,
-    render_to_response,
+    render,
 )
-from django.template import RequestContext
 from django.views.generic import CreateView
 from maasserver.forms import (
     ProfileForm,
@@ -79,10 +78,10 @@
     if response is not None:
         return response
 
-    return render_to_response(
+    return render(
+        request,
         'maasserver/prefs.html',
         {
             'profile_form': profile_form,
-            'password_form': password_form,
-        },
-        context_instance=RequestContext(request))
+            'password_form': password_form
+        })

=== modified file 'src/maasserver/views/settings.py'
--- src/maasserver/views/settings.py	2017-02-23 08:40:22 +0000
+++ src/maasserver/views/settings.py	2017-06-16 07:52:25 +0000
@@ -18,9 +18,8 @@
 from django.http import HttpResponseRedirect
 from django.shortcuts import (
     get_object_or_404,
-    render_to_response,
+    render,
 )
-from django.template import RequestContext
 from django.views.generic import (
     CreateView,
     DeleteView,
@@ -271,7 +270,8 @@
     for license_key in license_keys:
         set_license_key_titles(license_key, osystems)
 
-    return render_to_response(
+    return render(
+        request,
         'maasserver/settings.html',
         {
             'user_list': user_list,
@@ -288,5 +288,4 @@
             'ubuntu_form': ubuntu_form,
             'windows_form': windows_form,
             'kernelopts_form': kernelopts_form,
-        },
-        context_instance=RequestContext(request))
+        })

=== modified file 'src/maasserver/views/tests/test_general.py'
--- src/maasserver/views/tests/test_general.py	2017-02-17 14:23:04 +0000
+++ src/maasserver/views/tests/test_general.py	2017-06-16 07:52:25 +0000
@@ -12,10 +12,11 @@
 )
 
 from django.conf import settings
-from django.conf.urls import patterns
+from django.conf.urls import url
 from django.core.exceptions import PermissionDenied
 from django.http import Http404
 from django.test.client import RequestFactory
+from django.views.defaults import server_error
 from lxml.html import fromstring
 from maasserver.testing import extract_redirect
 from maasserver.testing.factory import factory
@@ -49,10 +50,7 @@
     def test_500(self):
         self.client_log_in()
         from maasserver.urls import urlpatterns
-        urlpatterns += patterns(
-            '',
-            (r'^500/$', 'django.views.defaults.server_error'),
-        )
+        urlpatterns += [url(r'^500/$', server_error)]
         response = self.client.get('/500/')
         doc = fromstring(response.content)
         self.assertIn(