Merge lp:~michael.nelson/ubuntu-webcatalog/rt-61147-nagios-sca-import into lp:ubuntu-webcatalog

[Michael Nelson]

Switch to logging.config.fileConfig (as used in deployments) and add logger for import_sca_apps.

Switch to logging.config.fileConfig (as used in deployments) and add logger for import_sca_apps.

This is to fix RT 61147. Similar update of the deployed configs:

https://code.launchpad.net/~michael.nelson/ca-configs/uwc-add-sca-import-logging-config/+merge/182371

(it would be good to review the bracnhes together)

So I wasn't blindly updating the deployment logging config, I've updated the codebase to use the same logging config method (ie. from file). I hit one stumbling block (see the XXX), but otherwise works as expected.

Unmerged revisions

189. By Michael Nelson on 2013-08-27

Removed other left-over logging config details.

188. By Michael Nelson on 2013-08-27

Switch to file-based logging config to match deployments.

=== modified file '.bzrignore'
--- .bzrignore	2013-02-28 16:23:58 +0000
+++ .bzrignore	2013-08-27 10:50:07 +0000
@@ -16,3 +16,4 @@
 django_project/local.cfg
 django_project/sreclient.py
 oopses/
+tmp/*

=== modified file 'django_project/config/main.cfg'
--- django_project/config/main.cfg	2013-04-26 14:24:42 +0000
+++ django_project/config/main.cfg	2013-08-27 10:50:07 +0000
@@ -6,7 +6,6 @@
                           django.contrib.auth.backends.ModelBackend
 
 databases = django_databases
-logging = django_logging
 
 debug = true
 media_root = django_project/media_root_dev/
@@ -73,29 +72,8 @@
 engine = django.db.backends.sqlite3
 name = webcatalog.db
 
-[django_logging]
-version = 1
-disable_existing_loggers = True
-handlers = django_logging_handlers
-loggers = django_logging_loggers
-root = django_logging_root
-
-[django_logging_root]
-level = WARNING
-
-[django_logging_loggers]
-django = django_logger
-
-[django_logger]
-handlers = null
-level = WARNING
-
-[django_logging_handlers]
-null = django_null_logging_handler
-
-[django_null_logging_handler]
-level = WARNING
-class = django.utils.log.NullHandler
+[logging]
+webapp_logging_config = ./django_project/dev_logging.conf
 
 [openid]
 openid_sso_server_url = https://login.staging.ubuntu.com

=== added file 'django_project/dev_logging.conf'
--- django_project/dev_logging.conf	1970-01-01 00:00:00 +0000
+++ django_project/dev_logging.conf	2013-08-27 10:50:07 +0000
@@ -0,0 +1,32 @@
+[loggers]
+keys=root,import_sca_apps
+
+[handlers]
+keys=webapp,commands
+
+[formatters]
+keys=simple_formatter
+
+[logger_root]
+level=INFO
+handlers=webapp
+
+[logger_import_sca_apps]
+level=INFO
+handlers=commands
+propagate=0
+qualname=webcatalog.management.commands.import_sca_apps
+
+[handler_webapp]
+class=FileHandler
+args=('./tmp/webapp.log',)
+formatter=simple_formatter
+
+[handler_commands]
+class=FileHandler
+args=('./tmp/import_sca_apps.log',)
+formatter=simple_formatter
+
+[formatter_simple_formatter]
+format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+datefmt=

=== modified file 'django_project/settings.py'
--- django_project/settings.py	2012-06-26 23:43:35 +0000
+++ django_project/settings.py	2013-08-27 10:50:07 +0000
@@ -13,7 +13,14 @@
                     'local.cfg'])
                 if os.path.exists(path)]
 
-configglue(WebCatalogSchema, config_files, __name__)
+parser = configglue(WebCatalogSchema, config_files, __name__)
 
 import djcelery
 djcelery.setup_loader()
+
+# XXX 2013-08-27 bug=1217290 django_configglue doesn't support file config.
+# Configure the logging once the settings have been parsed.
+import logging.config
+logging.config.fileConfig(
+    parser.get('logging', 'webapp_logging_config'),
+    disable_existing_loggers=True)

=== modified file 'src/webcatalog/management/commands/import_exhibits.py'
--- src/webcatalog/management/commands/import_exhibits.py	2012-08-20 15:34:18 +0000
+++ src/webcatalog/management/commands/import_exhibits.py	2013-08-27 10:50:07 +0000
@@ -81,4 +81,4 @@
         # Unpublish other exhibits
         Exhibit.objects.filter(published=True).exclude(
             sca_id__in=[xibit['id'] for xibit in exhibits]).update(
-                published=False)
+            published=False)

=== modified file 'src/webcatalog/management/commands/import_sca_apps.py'
--- src/webcatalog/management/commands/import_sca_apps.py	2012-08-21 13:57:11 +0000
+++ src/webcatalog/management/commands/import_sca_apps.py	2013-08-27 10:50:07 +0000
@@ -22,6 +22,7 @@
 from __future__ import absolute_import
 
 import json
+import logging
 import os
 import urllib
 from collections import defaultdict
@@ -42,12 +43,15 @@
 __all__ = []
 
 
+logger = logging.getLogger(__name__)
+
+
 class Command(BaseCommand):
     help = "Import for-purchase applications from Software Center."
-    verbosity = 0
 
     def handle(self, *args, **options):
         url = '%sapplications/any/ubuntu/any/any/' % settings.SCA_API_URL
+        logger.info("Starting import of sca apps at {0}".format(url))
         # call api and get apps for purchase
         response = urllib.urlopen(url)
         if response.code != 200:
@@ -55,8 +59,8 @@
         app_list = json.loads(response.read())
         imported_apps = Application.objects.filter(
             imported_from_sca=True).values_list(
-                'package_name',
-                'distroseries__code_name')
+            'package_name',
+            'distroseries__code_name')
         existing_apps = defaultdict(list)
         for package_name, code_name in imported_apps:
             existing_apps[package_name].append(code_name)
@@ -71,7 +75,7 @@
                 distroseries, created = DistroSeries.objects.get_or_create(
                     code_name=series_name)
                 if created:
-                    self.output(
+                    logger.info(
                         "Created a DistroSeries record called '{0}'.\n".format(
                             series_name), 1)
                 package_name = self.import_app_from_data(
@@ -91,6 +95,7 @@
             Application.objects.filter(
                 package_name=package_name,
                 distroseries__code_name__in=series).delete()
+        logger.info("Finished import of {0} sca apps.".format(len(app_list)))
 
     def import_app_from_data(self, app_data, icon_data, distroseries):
         form = SCAApplicationForm.from_api_data(
@@ -100,7 +105,6 @@
             department_names = app_data.get('department', [])
             app.update_departments(department_names)
             self.add_icon_to_app(app, data=icon_data)
-            self.output(u"{0} created.\n".format(app.name).encode('utf-8'), 1)
         return app.package_name
 
     def get_icon_data(self, app_data):
@@ -125,10 +129,3 @@
         finally:
             if os.path.exists(filename):
                 os.unlink(filename)
-
-    def output(self, message, level=None, flush=False):
-        if hasattr(self, 'stdout'):
-            if level is None or self.verbosity >= level:
-                self.stdout.write(message)
-            if flush:
-                self.stdout.flush()

=== modified file 'src/webcatalog/templatetags/webcatalog.py'
--- src/webcatalog/templatetags/webcatalog.py	2013-02-22 21:02:05 +0000
+++ src/webcatalog/templatetags/webcatalog.py	2013-08-27 10:50:07 +0000
@@ -79,8 +79,8 @@
                 '{application_name} is also available for '
                 '<a href="{app_in_series_url}">your version of '
                 'Ubuntu</a>.').format(
-                    application_name=escape(application.name),
-                    app_in_series_url=app_in_series_url)
+                application_name=escape(application.name),
+                app_in_series_url=app_in_series_url)
             context['message_text'] = mark_safe(
                 context['message_text'])
             return context

=== modified file 'src/webcatalog/wsgi.py'
--- src/webcatalog/wsgi.py	2013-02-28 16:23:58 +0000
+++ src/webcatalog/wsgi.py	2013-08-27 10:50:07 +0000
@@ -17,12 +17,10 @@
 
 """wsgi stack for the Apps Directory."""
 
-import logging.config
 import oops_dictconfig
 import oops_wsgi
 import oops_wsgi.django
 import os
-import platform
 import uuid
 
 from django.conf import settings
@@ -64,12 +62,6 @@
     if settings.SHOULD_SERVE_HTTPS:
         os.environ['HTTPS'] = 'on'
 
-    default_id = ''.join(x for x in platform.node() if x.isalpha())
-    appserver_id = getattr(settings, 'APPSERVER_ID', default_id)
-    logging_config = settings.WEBAPP_LOGGING_CONFIG
-    if logging_config:
-        logging.config.fileConfig(logging_config)
-
     non_oops_app = EagerOOPSWSGIHandler()
 
     config = oops_dictconfig.config_from_dict(settings.OOPSES)

=== added directory 'tmp'