Merge lp:~elachuni/ubuntu-webcatalog/departments into lp:ubuntu-webcatalog

[Anthony Lenton]

Overview
========
Add a Department model to loosely group applications.

Details
=======
This is needed to allow simple browsing of the site, to have apps grouped into departments like the software-center does.
A few tiny bugs were fixed while I was there, to fix failures I was getting when importing app data locally:
 - Encoded unicode strings when writing to stdout in import_app_install_data.py to avoid UnicodeEncodeErrors
 - Changed Application.package_name from a SlugField to a CharField, as there are package names with periods in them
 - Extended max_length for Application.mimetype
 - Allowed blank Application.app_type and Application.categories

[Michael Nelson (michael.nelson)]

Excellent Anthony! I'll assume that you're also working on the ui aspect of this and grab something else this morning.

Thanks for the small import fixes too.

review: Approve

=== modified file 'src/webcatalog/admin.py'
--- src/webcatalog/admin.py	2011-04-08 12:25:25 +0000
+++ src/webcatalog/admin.py	2011-04-13 02:40:59 +0000
@@ -22,7 +22,10 @@
     with_statement,
     )
 from django.contrib import admin
-from webcatalog.models import Application
+from webcatalog.models import (
+    Application,
+    Department,
+    )
 
 __metaclass__ = type
 __all__ = [
@@ -33,6 +36,7 @@
 class ApplicationAdmin(admin.ModelAdmin):
     list_display = ('package_name', 'name', 'comment')
     search_fields = ('package_name', 'name', 'comment')
-
+    list_filter = ('departments',)
 
 admin.site.register(Application, ApplicationAdmin)
+admin.site.register(Department)

=== added file 'src/webcatalog/department_filters.py'
--- src/webcatalog/department_filters.py	1970-01-01 00:00:00 +0000
+++ src/webcatalog/department_filters.py	2011-04-13 02:40:59 +0000
@@ -0,0 +1,116 @@
+#   -*- coding: utf-8 -*-
+#   This file is part of the Ubuntu Web Catalog
+#   Copyright (C) 2011 Canonical Ltd.
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Department filters."""
+
+from __future__ import (
+    absolute_import,
+    with_statement,
+    )
+
+import re
+
+from django.db import models
+
+__metaclass__ = type
+__all__ = [
+    'department_filters',
+    ]
+
+
+def category_filter(categories_set):
+    """Returns a filter func that checks an app for certain categories."""
+    def filter_func(app):
+        return bool(app.categories_set.intersection(categories_set))
+    return filter_func
+
+def package_name_filter(name_regex):
+    """Returns a filter func that checks if an app's name matches a regex"""
+    def filter_func(app):
+        return re.match(name_regex, app.package_name) is not None
+    return filter_func
+
+def section_filter(sections):
+    """Returns a filter that checks if an app is in certain sections."""
+    def filter_func(app):
+        return app.section in sections
+    return filter_func
+
+# Taken from https://wiki.ubuntu.com/SoftwareCenter#line-793
+
+department_filters = {
+    'Accessories': [category_filter(set(['Utility', 'System']))],
+    'Education': [category_filter(set(['Education']))],
+    'Fonts': [package_name_filter(r'[to]tf-.*')],
+    'Games': [category_filter(set(['Game', 'Sports']))],
+    'Board Games': [category_filter(set(['BoardGame']))],
+    'Card Games': [category_filter(set(['CardGame']))],
+    'Puzzles': [category_filter(set(['LogicGame']))],
+    'Role-Playing': [category_filter(set(['RolePlaying']))],
+    'Sports': [category_filter(set(['SportsGame']))],
+    'Graphics': [category_filter(set(['Graphics']))],
+    '3D': [category_filter(set(['3DGraphics']))],
+    'Drawing': [category_filter(set(['VectorGraphics']))],
+    'Painting': [category_filter(set(['RasterGraphics']))],
+    'Photography': [category_filter(set(['Photography']))],
+    'Publishing': [category_filter(set(['Publishing']))],
+    'Scanning & OCR': [category_filter(set(['Scanning', 'OCR']))],
+    'Viewers': [category_filter(set(['Viewer']))],
+    'Internet': [category_filter(set(['Network']))],
+    'Chat': [category_filter(set(['InstantMessaging', 'IRCClient']))],
+    'File Sharing': [category_filter(set(['FileTransfer']))],
+    'Mail': [category_filter(set(['Email']))],
+    'Web Browsers': [category_filter(set(['WebBrowser']))],
+    'Office': [category_filter(set(['Office']))],
+    'Science & Engineering': [category_filter(set(['Science'])),
+        section_filter(['science'])],
+    'Astronomy': [category_filter(set(['Astronomy']))],
+    'Biology': [category_filter(set(['Biology']))],
+    'Chemistry': [category_filter(set(['Chemistry']))],
+    'Computing & Robotics': [category_filter(set(['ArtificialIntelligence',
+        'ComputerScience', 'Robotics']))],
+    'Electronics': [category_filter(set(['Electronics']))],
+    'Engineering': [category_filter(set(['Engineering']))],
+    'Geography': [category_filter(set(['Geography']))],
+    'Geology': [category_filter(set(['Geology', 'Geoscience']))],
+    'Mathematics': [category_filter(set(['DataVisualization', 'Math',
+        'NumericalAnalysis'])), section_filter(['math', 'gnu-r'])],
+    'Medicine': [category_filter(set(['MedicalSoftware']))],
+    'Physics': [category_filter(set(['Electricity', 'Physics']))],
+    'Sound & Video': [category_filter(set(['AudioVideo', 'Audio', 'Video']))],
+    'Themes & Tweaks': [category_filter(set(['Settings']))],
+    'Universal Access': [category_filter(set(['Accessibility']))],
+    'Developer Tools': [category_filter(set(['Development'])),
+        section_filter(['devel'])],
+    'Debugging': [category_filter(set(['Debugger']))],
+    'Graphic Interface Design': [category_filter(set(['GUIDesigner']))],
+    'Haskell': [section_filter(['haskell'])],
+    'IDEs': [category_filter(set(['IDE']))],
+    'Java': [section_filter(['java'])],
+    'Libraries': [section_filter(['libdevel'])],
+    'Lisp': [section_filter(['lisp'])],
+    'Localization': [category_filter(set(['Translation']))],
+    'Mono/CLI': [section_filter(['cli-mono'])],
+    'OCaml': [section_filter(['ocaml'])],
+    'Perl': [section_filter(['perl'])],
+    'Profiling': [category_filter(set(['Profiling']))],
+    'Python': [section_filter(['python'])],
+    'Ruby': [section_filter(['ruby'])],
+    'Version Control': [category_filter(set(['RevisionControl'])),
+        section_filter(['vcs'])],
+    'Web Development': [category_filter(set(['WebDevelopment']))],
+}

=== added directory 'src/webcatalog/fixtures'
=== added file 'src/webcatalog/fixtures/initial_data.json'
--- src/webcatalog/fixtures/initial_data.json	1970-01-01 00:00:00 +0000
+++ src/webcatalog/fixtures/initial_data.json	2011-04-13 02:40:59 +0000
@@ -0,0 +1,370 @@
+[
+    {
+        "pk": 1, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Games", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 2, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Office", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 3, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Sound & Video", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 4, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Developer Tools", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 5, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Science & Engineering", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 6, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Education", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 7, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Biology", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 8, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Accessories", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 9, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Role-Playing", 
+            "parent": 1
+        }
+    }, 
+    {
+        "pk": 10, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Geography", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 11, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Medicine", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 12, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Viewers", 
+            "parent": 13
+        }
+    }, 
+    {
+        "pk": 13, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Graphics", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 14, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Themes & Tweaks", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 15, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Internet", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 16, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Debugging", 
+            "parent": 4
+        }
+    }, 
+    {
+        "pk": 17, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Profiling", 
+            "parent": 4
+        }
+    }, 
+    {
+        "pk": 18, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Chat", 
+            "parent": 15
+        }
+    }, 
+    {
+        "pk": 19, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "IDEs", 
+            "parent": 4
+        }
+    }, 
+    {
+        "pk": 20, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "3D", 
+            "parent": 13
+        }
+    }, 
+    {
+        "pk": 21, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Engineering", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 22, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Electronics", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 23, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Web Browsers", 
+            "parent": 15
+        }
+    }, 
+    {
+        "pk": 24, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Mathematics", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 25, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Chemistry", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 26, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Physics", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 27, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "FileSharing", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 28, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Mail", 
+            "parent": 15
+        }
+    }, 
+    {
+        "pk": 29, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Computing & Robotics", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 30, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Web Development", 
+            "parent": 4
+        }
+    }, 
+    {
+        "pk": 31, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Graphic Interface Design", 
+            "parent": 4
+        }
+    }, 
+    {
+        "pk": 32, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Version Control", 
+            "parent": 4
+        }
+    }, 
+    {
+        "pk": 33, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Photography", 
+            "parent": 13
+        }
+    }, 
+    {
+        "pk": 34, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Astronomy", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 35, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Universal Access", 
+            "parent": null
+        }
+    }, 
+    {
+        "pk": 36, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Drawing", 
+            "parent": 13
+        }
+    }, 
+    {
+        "pk": 37, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Painting", 
+            "parent": 13
+        }
+    }, 
+    {
+        "pk": 38, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Publishing", 
+            "parent": 13
+        }
+    }, 
+    {
+        "pk": 39, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Localization", 
+            "parent": 4
+        }
+    }, 
+    {
+        "pk": 40, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Scanning & OCR", 
+            "parent": 13
+        }
+    }, 
+    {
+        "pk": 41, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Geology", 
+            "parent": 5
+        }
+    }, 
+    {
+        "pk": 42, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Board Games", 
+            "parent": 1
+        }
+    }, 
+    {
+        "pk": 43, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Puzzles", 
+            "parent": 1
+        }
+    }, 
+    {
+        "pk": 44, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "File Sharing", 
+            "parent": 15
+        }
+    }, 
+    {
+        "pk": 45, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Sports", 
+            "parent": 1
+        }
+    }, 
+    {
+        "pk": 46, 
+        "model": "webcatalog.department", 
+        "fields": {
+            "name": "Card Games", 
+            "parent": 1
+        }
+    }
+]
\ No newline at end of file

=== modified file 'src/webcatalog/management/commands/import_app_install_data.py'
--- src/webcatalog/management/commands/import_app_install_data.py	2011-04-12 15:55:30 +0000
+++ src/webcatalog/management/commands/import_app_install_data.py	2011-04-13 02:40:59 +0000
@@ -86,11 +86,12 @@
 
             if form.is_valid():
                 app = form.save()
+                app.update_departments()
                 if self.verbosity > 0:
                     self.stdout.write(
-                        u"{0} created.\n".format(app.name))
+                        u"{0} created.\n".format(app.name).encode('utf-8'))
             else:
                 if self.verbosity > 0:
                     self.stdout.write(
                         u"Skipping {0} as input failed validation: {1}.\n".format(
-                            member.name, form.errors))
+                            member.name, form.errors).encode('utf-8'))

=== modified file 'src/webcatalog/models.py'
--- src/webcatalog/models.py	2011-04-12 15:22:54 +0000
+++ src/webcatalog/models.py	2011-04-13 02:40:59 +0000
@@ -21,8 +21,13 @@
     absolute_import,
     with_statement,
     )
+
+import logging
+
 from django.db import models
 
+from webcatalog.department_filters import department_filters
+
 __metaclass__ = type
 __all__ = [
     'Application',
@@ -37,19 +42,20 @@
     # at runtime instead.
 
     # The following fields are extracted from app-install-data.
-    package_name = models.SlugField(max_length=100)
+    package_name = models.CharField(max_length=100)
     name = models.CharField(max_length=255)
     comment = models.CharField(max_length=255, blank=True)
     popcon = models.IntegerField()
     channel = models.CharField(max_length=255, blank=True)
     screenshot_url = models.URLField(blank=True,
         help_text="Only use this if it is other than the normal screenshot url.")
-    mimetype = models.CharField(max_length=255, blank=True)
+    mimetype = models.CharField(max_length=2048, blank=True)
     architectures = models.CharField(max_length=255, blank=True)
     keywords = models.CharField(max_length=255, blank=True)
-    app_type = models.CharField(max_length=32)
+    app_type = models.CharField(max_length=32, blank=True)
     section = models.CharField(max_length=32)
-    categories = models.CharField(max_length=255)
+    categories = models.CharField(max_length=255, blank=True)
+    departments = models.ManyToManyField('Department', blank=True)
 
     # Other desktop fields used by s-c
     gnome_full_name = models.CharField(max_length=255, blank=True)
@@ -64,3 +70,30 @@
 
     def __unicode__(self):
         return u"{0} ({1})".format(self.name, self.package_name)
+
+    @property
+    def categories_set(self):
+        """Return the set of categories for this app"""
+        stripped = [x.strip() for x in self.categories.split(';')]
+        return set(x for x in stripped if x)
+
+    def update_departments(self):
+        """Update the list of departments for this app"""
+        self.departments.clear()
+        for dept_name, dept_filters in department_filters.items():
+            for dept_filter in dept_filters:
+                if dept_filter(self):
+                    dept, created = Department.objects.get_or_create(
+                        name=dept_name)
+                    if created:
+                        logging.warn("Department %s automatically created!" %
+                            dept_name)
+                    self.departments.add(dept)
+                    break
+
+class Department(models.Model):
+    parent = models.ForeignKey('self', blank=True, null=True)
+    name = models.CharField(max_length=64)
+
+    def __unicode__(self):
+        return self.name

=== modified file 'src/webcatalog/tests/__init__.py'
--- src/webcatalog/tests/__init__.py	2011-04-12 12:54:38 +0000
+++ src/webcatalog/tests/__init__.py	2011-04-13 02:40:59 +0000
@@ -19,3 +19,5 @@
 from .test_forms import *
 from .test_commands import *
 from .test_views import *
+from .test_department_filters import *
+from .test_models import *

=== added file 'src/webcatalog/tests/test_department_filters.py'
--- src/webcatalog/tests/test_department_filters.py	1970-01-01 00:00:00 +0000
+++ src/webcatalog/tests/test_department_filters.py	2011-04-13 02:40:59 +0000
@@ -0,0 +1,69 @@
+#   -*- coding: utf-8 -*-
+#   This file is part of the Ubuntu Web Catalog
+#   Copyright (C) 2011 Canonical Ltd.
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test cases for department filters."""
+
+from __future__ import (
+    absolute_import,
+    with_statement,
+    )
+
+
+from webcatalog.tests.factory import TestCaseWithFactory
+from webcatalog.department_filters import (
+    category_filter,
+    package_name_filter,
+    section_filter
+    )
+
+__metaclass__ = type
+__all__ = [
+    'DepartmentFilterTestCase',
+    ]
+
+
+class DepartmentFilterTestCase(TestCaseWithFactory):
+    def test_package_name_filter(self):
+        dept_filter = package_name_filter('a*$')
+        app1 = self.factory.make_application(package_name='aaaaa')
+        self.assertTrue(dept_filter(app1))
+
+        app2 = self.factory.make_application(package_name='aaabaa')
+        self.assertFalse(dept_filter(app2))
+
+    def test_category_filter(self):
+        dept_filter = category_filter(['foo', 'bar'])
+        app = self.factory.make_application()
+        self.assertFalse(dept_filter(app))
+        app.categories = 'foo;bin'
+        self.assertTrue(dept_filter(app))
+        app.categories = 'foobin'
+        self.assertFalse(dept_filter(app))
+        app.categories = ''
+        self.assertFalse(dept_filter(app))
+
+    def test_section_filter(self):
+        dept_filter = section_filter(['foo', 'bar'])
+        app = self.factory.make_application()
+
+        self.assertFalse(dept_filter(app))
+        app.section = 'foo'
+        self.assertTrue(dept_filter(app))
+        app.section = 'foobin'
+        self.assertFalse(dept_filter(app))
+        app.section = ''
+        self.assertFalse(dept_filter(app))

=== added file 'src/webcatalog/tests/test_models.py'
--- src/webcatalog/tests/test_models.py	1970-01-01 00:00:00 +0000
+++ src/webcatalog/tests/test_models.py	2011-04-13 02:40:59 +0000
@@ -0,0 +1,60 @@
+#   -*- coding: utf-8 -*-
+#   This file is part of the Ubuntu Web Catalog
+#   Copyright (C) 2011 Canonical Ltd.
+#
+#   This program is free software: you can redistribute it and/or modify
+#   it under the terms of the GNU Affero General Public License as
+#   published by the Free Software Foundation, either version 3 of the
+#   License, or (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU Affero General Public License for more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test cases for models."""
+
+from __future__ import (
+    absolute_import,
+    with_statement,
+    )
+
+
+from webcatalog.tests.factory import TestCaseWithFactory
+from webcatalog.models import Application
+
+__metaclass__ = type
+__all__ = [
+    'ApplicationTestCase',
+    ]
+
+
+class ApplicationTestCase(TestCaseWithFactory):
+    def test_categories_set(self):
+        app = self.factory.make_application()
+        app.categories = 'foo;;;; bar  '
+        self.assertEqual(set(['foo', 'bar']), app.categories_set)
+
+    def test_empty_category_set(self):
+        app = self.factory.make_application()
+        app.categories = ''
+        self.assertEqual(set(), app.categories_set)
+
+    def test_update_empty_departments(self):
+        app = self.factory.make_application()
+
+        app.update_departments()
+
+        self.assertEqual(0, app.departments.count())
+
+    def test_update_departments(self):
+        app = self.factory.make_application()
+        app.categories = 'Game;'
+
+        app.update_departments()
+
+        self.assertEqual(1, app.departments.count())
+        self.assertEqual('Games', app.departments.get().name)