Merge lp:~wgrant/launchpad/xref-model into lp:launchpad

[William Grant]

Add lp.services.xref for generic cross-references between artifacts.

Add lp.services.xref for generic cross-references between artifacts.

Schema is at <https://code.launchpad.net/~wgrant/launchpad/xref-db/+merge/272587>.

It's likely that we'll end up with wrappers that automatically resolve known objects to/from their tuple representations, but for now the by-ID API is surprisingly unonerous.

[Colin Watson (cjwatson)]

review: Approve

=== modified file 'lib/lp/services/configure.zcml'
--- lib/lp/services/configure.zcml	2015-05-22 07:12:35 +0000
+++ lib/lp/services/configure.zcml	2015-09-28 23:38:07 +0000
@@ -34,4 +34,5 @@
   <include package=".webhooks" />
   <include package=".webservice" />
   <include package=".worlddata" />
+  <include package=".xref" />
 </configure>

=== added directory 'lib/lp/services/xref'
=== added file 'lib/lp/services/xref/__init__.py'
--- lib/lp/services/xref/__init__.py	1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/__init__.py	2015-09-28 23:38:07 +0000
@@ -0,0 +1,13 @@
+# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Generic cross references between artifacts.
+
+Provides infrastructure for generic information references between
+artifacts, easing weak coupling of apps.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = []

=== added file 'lib/lp/services/xref/configure.zcml'
--- lib/lp/services/xref/configure.zcml	1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/configure.zcml	2015-09-28 23:38:07 +0000
@@ -0,0 +1,13 @@
+<!-- Copyright 2015 Canonical Ltd.  This software is licensed under the
+     GNU Affero General Public License version 3 (see the file LICENSE).
+-->
+
+<configure xmlns="http://namespaces.zope.org/zope">
+
+    <securedutility
+        class="lp.services.xref.model.XRefSet"
+        provides="lp.services.xref.interfaces.IXRefSet">
+        <allow interface="lp.services.xref.interfaces.IXRefSet"/>
+    </securedutility>
+
+</configure>

=== added file 'lib/lp/services/xref/interfaces.py'
--- lib/lp/services/xref/interfaces.py	1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/interfaces.py	2015-09-28 23:38:07 +0000
@@ -0,0 +1,62 @@
+# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'IXRefSet',
+    ]
+
+from zope.interface import Interface
+
+
+class IXRefSet(Interface):
+    """Manager of cross-references between objects.
+
+    Each participant in an xref has an "object ID": a tuple of
+    (str type, str id).
+
+    All xrefs are currently between local objects, so links always exist
+    in both directions, but this can't be assumed to hold in future.
+    """
+
+    def create(xrefs):
+        """Create cross-references.
+
+        Back-links are automatically created.
+
+        :param xrefs: A dict of
+            {from_object_id: {to_object_id:
+                {'creator': `IPerson`, 'metadata': value}}}.
+            The creator and metadata keys are optional.
+        """
+
+    def findFromMany(object_ids, types=None):
+        """Find all cross-references from multiple objects.
+
+        :param object_ids: A collection of object IDs.
+        :param types: An optional collection of the types to include.
+        :return: A dict of
+            {from_object_id: {to_object_id:
+                {'creator': `IPerson`, 'metadata': value}}}.
+            The creator and metadata keys are optional.
+        """
+
+    def delete(xrefs):
+        """Delete cross-references.
+
+        Back-links are automatically deleted.
+
+        :param xrefs: A dict of {from_object_id: [to_object_id]}.
+        """
+
+    def findFrom(object_id, types=None):
+        """Find all cross-references from an object.
+
+        :param object_id: An object ID.
+        :param types: An optional collection of the types to include.
+        :return: A dict of
+            {to_object_id: {'creator': `IPerson`, 'metadata': value}}.
+            The creator and metadata keys are optional.
+        """

=== added file 'lib/lp/services/xref/model.py'
--- lib/lp/services/xref/model.py	1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/model.py	2015-09-28 23:38:07 +0000
@@ -0,0 +1,131 @@
+# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    "XRefSet",
+    ]
+
+import pytz
+from storm.expr import (
+    And,
+    Or,
+    )
+from storm.properties import (
+    DateTime,
+    Int,
+    JSON,
+    Unicode,
+    )
+from storm.references import Reference
+from zope.interface import implementer
+
+from lp.services.database import bulk
+from lp.services.database.constants import UTC_NOW
+from lp.services.database.interfaces import IStore
+from lp.services.database.stormbase import StormBase
+from lp.services.xref.interfaces import IXRefSet
+
+
+class XRef(StormBase):
+    """Cross-reference between two objects.
+
+    For references to local objects (there is currently no other kind),
+    another reference in the opposite direction exists.
+
+    The to_id_int and from_id_int columns exist for efficient SQL joins.
+    They are set automatically when the ID looks like an integer.
+    """
+
+    __storm_table__ = 'XRef'
+    __storm_primary__ = "from_type", "from_id", "to_type", "to_id"
+
+    to_type = Unicode(allow_none=False)
+    to_id = Unicode(allow_none=False)
+    to_id_int = Int()  # For efficient joins.
+    from_type = Unicode(allow_none=False)
+    from_id = Unicode(allow_none=False)
+    from_id_int = Int()  # For efficient joins.
+    creator_id = Int(name="creator")
+    creator = Reference(creator_id, "Person.id")
+    date_created = DateTime(name='date_created', tzinfo=pytz.UTC)
+    metadata = JSON()
+
+
+def _int_or_none(s):
+    if s.isdigit():
+        return int(s)
+    else:
+        return None
+
+
+@implementer(IXRefSet)
+class XRefSet:
+
+    def create(self, xrefs):
+        # All references are currently to local objects, so add
+        # backlinks as well to keep queries in both directions quick.
+        # The *_id_int columns are also set if the ID looks like an int.
+        rows = []
+        for from_, tos in xrefs.items():
+            for to, props in tos.items():
+                rows.append((
+                    from_[0], from_[1], _int_or_none(from_[1]),
+                    to[0], to[1], _int_or_none(to[1]),
+                    props.get('creator'), props.get('date_created', UTC_NOW),
+                    props.get('metadata')))
+                rows.append((
+                    to[0], to[1], _int_or_none(to[1]),
+                    from_[0], from_[1], _int_or_none(from_[1]),
+                    props.get('creator'), props.get('date_created', UTC_NOW),
+                    props.get('metadata')))
+        bulk.create(
+            (XRef.from_type, XRef.from_id, XRef.from_id_int,
+             XRef.to_type, XRef.to_id, XRef.to_id_int,
+             XRef.creator, XRef.date_created, XRef.metadata), rows)
+
+    def delete(self, xrefs):
+        # Delete both directions.
+        pairs = []
+        for from_, tos in xrefs.items():
+            for to in tos:
+                pairs.extend([(from_, to), (to, from_)])
+
+        IStore(XRef).find(
+            XRef,
+            Or(*[
+                And(XRef.from_type == pair[0][0],
+                    XRef.from_id == pair[0][1],
+                    XRef.to_type == pair[1][0],
+                    XRef.to_id == pair[1][1])
+                for pair in pairs])
+            ).remove()
+
+    def findFromMany(self, object_ids, types=None):
+        from lp.registry.model.person import Person
+
+        object_ids = list(object_ids)
+        if not object_ids:
+            return {}
+
+        store = IStore(XRef)
+        rows = list(store.using(XRef).find(
+            (XRef.from_type, XRef.from_id, XRef.to_type, XRef.to_id,
+             XRef.creator_id, XRef.date_created, XRef.metadata),
+            Or(*[
+                And(XRef.from_type == id[0], XRef.from_id == id[1])
+                for id in object_ids]),
+            XRef.to_type.is_in(types) if types is not None else True))
+        bulk.load(Person, [row[4] for row in rows])
+        result = {}
+        for row in rows:
+            result.setdefault((row[0], row[1]), {})[(row[2], row[3])] = {
+                "creator": store.get(Person, row[4]) if row[4] else None,
+                "date_created": row[5],
+                "metadata": row[6]}
+        return result
+
+    def findFrom(self, object_id, types=None):
+        return self.findFromMany([object_id], types=types).get(object_id, {})

=== added directory 'lib/lp/services/xref/tests'
=== added file 'lib/lp/services/xref/tests/__init__.py'
--- lib/lp/services/xref/tests/__init__.py	1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/tests/__init__.py	2015-09-28 23:38:07 +0000
@@ -0,0 +1,7 @@
+# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = []

=== added file 'lib/lp/services/xref/tests/test_model.py'
--- lib/lp/services/xref/tests/test_model.py	1970-01-01 00:00:00 +0000
+++ lib/lp/services/xref/tests/test_model.py	2015-09-28 23:38:07 +0000
@@ -0,0 +1,186 @@
+# Copyright 2015 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+import datetime
+
+import pytz
+from testtools.matchers import (
+    ContainsDict,
+    Equals,
+    MatchesDict,
+    )
+from zope.component import getUtility
+
+from lp.services.database.interfaces import IStore
+from lp.services.database.sqlbase import flush_database_caches
+from lp.services.xref.interfaces import IXRefSet
+from lp.services.xref.model import XRef
+from lp.testing import (
+    StormStatementRecorder,
+    TestCaseWithFactory,
+    )
+from lp.testing.layers import DatabaseFunctionalLayer
+from lp.testing.matchers import HasQueryCount
+
+
+class TestXRefSet(TestCaseWithFactory):
+
+    layer = DatabaseFunctionalLayer
+
+    def test_create_sets_date_created(self):
+        # date_created defaults to now, but can be overridden.
+        old = datetime.datetime.strptime('2005-01-01', '%Y-%m-%d').replace(
+            tzinfo=pytz.UTC)
+        now = IStore(XRef).execute(
+            "SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'"
+            ).get_one()[0].replace(tzinfo=pytz.UTC)
+        getUtility(IXRefSet).create({
+            ('a', '1'): {('b', 'foo'): {}},
+            ('a', '2'): {('b', 'bar'): {'date_created': old}}})
+        rows = IStore(XRef).find(
+            (XRef.from_id, XRef.to_id, XRef.date_created),
+            XRef.from_type == 'a')
+        self.assertContentEqual(
+            [('1', 'foo', now), ('2', 'bar', old)], rows)
+
+    def test_create_sets_int_columns(self):
+        # The string ID columns have integers equivalents for quick and
+        # easy joins to integer PKs. They're set automatically when the
+        # string ID looks like an integer.
+        getUtility(IXRefSet).create({
+            ('a', '1234'): {('b', 'foo'): {}, ('b', '2468'): {}},
+            ('a', '12ab'): {('b', '1234'): {}, ('b', 'foo'): {}}})
+        rows = IStore(XRef).find(
+            (XRef.from_type, XRef.from_id, XRef.from_id_int, XRef.to_type,
+             XRef.to_id, XRef.to_id_int),
+            XRef.from_type == 'a')
+        self.assertContentEqual(
+            [('a', '1234', 1234, 'b', 'foo', None),
+             ('a', '1234', 1234, 'b', '2468', 2468),
+             ('a', '12ab', None, 'b', '1234', 1234),
+             ('a', '12ab', None, 'b', 'foo', None)
+             ],
+            rows)
+
+    def test_findFrom(self):
+        creator = self.factory.makePerson()
+        now = IStore(XRef).execute(
+            "SELECT CURRENT_TIMESTAMP AT TIME ZONE 'UTC'"
+            ).get_one()[0].replace(tzinfo=pytz.UTC)
+        getUtility(IXRefSet).create({
+            ('a', 'bar'): {
+                ('b', 'foo'): {'creator': creator, 'metadata': {'test': 1}}},
+            ('b', 'foo'): {
+                ('a', 'baz'): {'creator': creator, 'metadata': {'test': 2}}},
+            })
+
+        with StormStatementRecorder() as recorder:
+            bar_refs = getUtility(IXRefSet).findFrom(('a', 'bar'))
+        self.assertThat(recorder, HasQueryCount(Equals(2)))
+        self.assertEqual(
+            {('b', 'foo'): {
+                'creator': creator, 'date_created': now,
+                'metadata': {'test': 1}}},
+            bar_refs)
+
+        with StormStatementRecorder() as recorder:
+            foo_refs = getUtility(IXRefSet).findFrom(('b', 'foo'))
+        self.assertThat(recorder, HasQueryCount(Equals(2)))
+        self.assertEqual(
+            {('a', 'bar'): {
+                'creator': creator, 'date_created': now,
+                'metadata': {'test': 1}},
+             ('a', 'baz'): {
+                 'creator': creator, 'date_created': now,
+                 'metadata': {'test': 2}}},
+            foo_refs)
+
+        with StormStatementRecorder() as recorder:
+            bar_refs = getUtility(IXRefSet).findFrom(('a', 'baz'))
+        self.assertThat(recorder, HasQueryCount(Equals(2)))
+        self.assertEqual(
+            {('b', 'foo'): {
+                'creator': creator, 'date_created': now,
+                'metadata': {'test': 2}}},
+            bar_refs)
+
+        with StormStatementRecorder() as recorder:
+            bar_baz_refs = getUtility(IXRefSet).findFromMany(
+                [('a', 'bar'), ('a', 'baz')])
+        self.assertThat(recorder, HasQueryCount(Equals(2)))
+        self.assertEqual(
+            {('a', 'bar'): {
+                ('b', 'foo'): {
+                    'creator': creator, 'date_created': now,
+                    'metadata': {'test': 1}}},
+             ('a', 'baz'): {
+                ('b', 'foo'): {
+                    'creator': creator, 'date_created': now,
+                    'metadata': {'test': 2}}}},
+             bar_baz_refs)
+
+    def test_findFrom_creator(self):
+        # findFrom issues a single query to get all of the people.
+        people = [self.factory.makePerson() for i in range(3)]
+        getUtility(IXRefSet).create({
+            ('a', '0'): {
+                ('b', '0'): {'creator': people[2]},
+                ('b', '1'): {'creator': people[0]},
+                ('b', '2'): {'creator': people[1]},
+                },
+            })
+        flush_database_caches()
+        with StormStatementRecorder() as recorder:
+            xrefs = getUtility(IXRefSet).findFrom(('a', '0'))
+        self.assertThat(
+            xrefs,
+            MatchesDict({
+                ('b', '0'): ContainsDict({'creator': Equals(people[2])}),
+                ('b', '1'): ContainsDict({'creator': Equals(people[0])}),
+                ('b', '2'): ContainsDict({'creator': Equals(people[1])}),
+                }))
+        self.assertThat(recorder, HasQueryCount(Equals(2)))
+
+    def test_findFrom_types(self):
+        # findFrom can look for only particular types of related
+        # objects.
+        getUtility(IXRefSet).create({
+            ('a', '1'): {('a', '2'): {}, ('b', '3'): {}},
+            ('b', '4'): {('a', '5'): {}, ('c', '6'): {}},
+            })
+        self.assertContentEqual(
+            [('a', '2')],
+            getUtility(IXRefSet).findFrom(('a', '1'), types=['a', 'c']).keys())
+        self.assertContentEqual(
+            [('a', '5'), ('c', '6')],
+            getUtility(IXRefSet).findFrom(('b', '4'), types=['a', 'c']).keys())
+
+        # Asking for no types or types that don't exist finds nothing.
+        self.assertContentEqual(
+            [],
+            getUtility(IXRefSet).findFrom(('b', '4'), types=[]).keys())
+        self.assertContentEqual(
+            [],
+            getUtility(IXRefSet).findFrom(('b', '4'), types=['d']).keys())
+
+    def test_findFromMany_none(self):
+        self.assertEqual({}, getUtility(IXRefSet).findFromMany([]))
+
+    def test_delete(self):
+        getUtility(IXRefSet).create({
+            ('a', 'bar'): {('b', 'foo'): {}},
+            ('b', 'foo'): {('a', 'baz'): {}},
+            })
+        self.assertContentEqual(
+            [('a', 'bar'), ('a', 'baz')],
+            getUtility(IXRefSet).findFrom(('b', 'foo')).keys())
+        with StormStatementRecorder() as recorder:
+            getUtility(IXRefSet).delete({('b', 'foo'): [('a', 'bar')]})
+        self.assertThat(recorder, HasQueryCount(Equals(1)))
+        self.assertEqual(
+            [('a', 'baz')],
+            getUtility(IXRefSet).findFrom(('b', 'foo')).keys())

=== modified file 'lib/lp/testing/tests/test_standard_test_template.py'
--- lib/lp/testing/tests/test_standard_test_template.py	2015-01-30 10:13:51 +0000
+++ lib/lp/testing/tests/test_standard_test_template.py	2015-09-28 23:38:07 +0000
@@ -3,6 +3,8 @@
 
 """XXX: Module docstring goes here."""
 
+from __future__ import absolute_import, print_function, unicode_literals
+
 __metaclass__ = type
 
 # or TestCaseWithFactory

=== modified file 'lib/lp/testing/tests/test_standard_yuixhr_test_template.py'
--- lib/lp/testing/tests/test_standard_yuixhr_test_template.py	2015-01-30 10:13:51 +0000
+++ lib/lp/testing/tests/test_standard_yuixhr_test_template.py	2015-09-28 23:38:07 +0000
@@ -4,6 +4,8 @@
 """{Describe your test suite here}.
 """
 
+from __future__ import absolute_import, print_function, unicode_literals
+
 __metaclass__ = type
 __all__ = []
 

=== modified file 'standard_template.py'
--- standard_template.py	2015-01-30 10:13:51 +0000
+++ standard_template.py	2015-09-28 23:38:07 +0000
@@ -3,5 +3,7 @@
 
 """XXX: Module docstring goes here."""
 
+from __future__ import absolute_import, print_function, unicode_literals
+
 __metaclass__ = type
 __all__ = []