Merge lp:~sinzui/launchpad/project-notify-3 into lp:launchpad

[Curtis Hovey]

Add base productjob model classes and tests.

    Launchpad bug: https://bugs.launchpad.net/bugs/956246
    Pre-implementation: abentley, jcsackett

This branch adds a productjob models to track automated and manual jobs
that are scheduled for a product. The immediate use is for an automated
system that send out commercial subscription expiration emails at 4 weeks
and 1 week before expiration. After expiration an automated process will
update or deactivate the project and send an email.

This branch adds the base classes for the job, but the specifc jobs
we need are deferred to my next branch.

--------------------------------------------------------------------

RULES

    * Create base and derived classes for ProductJob that can be
      used to create specific classes to for tasks like sending
      a commericial subscription expiration email.

QA

    None because this branch only contains the classes needed by other
    branches.

LINT

    lib/lp/registry/enums.py
    lib/lp/registry/interfaces/productjob.py
    lib/lp/registry/model/productjob.py
    lib/lp/registry/tests/test_productjob.py

TEST

    ./bin/test -vv lp.registry.tests.test_productjob

IMPLEMENTATION

Update enums with the actual types of job Lp needs to support.
    lib/lp/registry/enums.py

Created the base and derived job classes based on the person transfer job.
I the branch was already large after adding and testing the support classes
so I decided to submit this for merging separately from the real work.
    lib/lp/registry/interfaces/productjob.py
    lib/lp/registry/model/productjob.py
    lib/lp/registry/tests/test_productjob.py

[Benji York (benji)]

This branch looks good. It also makes me wish we had an easier way of creating jobs.

review: Approve (code)

=== modified file 'lib/lp/registry/enums.py'
--- lib/lp/registry/enums.py	2012-03-07 00:23:37 +0000
+++ lib/lp/registry/enums.py	2012-03-20 13:26:37 +0000
@@ -9,6 +9,7 @@
     'DistroSeriesDifferenceType',
     'InformationType',
     'PersonTransferJobType',
+    'ProductJobType',
     'SharingPermission',
     ]
 
@@ -155,3 +156,34 @@
 
         Merge one person or team into another person or team.
         """)
+
+
+class ProductJobType(DBEnumeratedType):
+    """Values that IProductJob.job_type can take."""
+
+    REVIEWER_NOTIFICATION = DBItem(0, """
+        Reviewer notification
+
+        A notification sent by a project reviewer to the project maintainers.
+        """)
+
+    COMMERCIAL_EXPIRATION_30_DAYS = DBItem(1, """
+        Commercial subscription expires in 30 days.
+
+        A notification stating that the project's commercial subscription
+        expires in 30 days.
+        """)
+
+    COMMERCIAL_EXPIRATION_7_DAYS = DBItem(2, """
+        Commercial subscription expires in 7 days.
+
+        A notification stating that the project's commercial subscription
+        expires in 7 days.
+        """)
+
+    COMMERCIAL_EXPIRED = DBItem(3, """
+        Commercial subscription expired.
+
+        A notification stating that the project's commercial subscription
+        expired.
+        """)

=== added file 'lib/lp/registry/interfaces/productjob.py'
--- lib/lp/registry/interfaces/productjob.py	1970-01-01 00:00:00 +0000
+++ lib/lp/registry/interfaces/productjob.py	2012-03-20 13:26:37 +0000
@@ -0,0 +1,67 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Interfaces for the Jobs system to update products and send notifications."""
+
+__metaclass__ = type
+__all__ = [
+    'IProductJob',
+    'IProductJobSource',
+    ]
+
+from zope.interface import Attribute
+from zope.schema import (
+    Int,
+    Object,
+    )
+
+from lp import _
+from lp.registry.interfaces.product import IProduct
+
+from lp.services.job.interfaces.job import (
+    IJob,
+    IJobSource,
+    IRunnableJob,
+    )
+
+
+class IProductJob(IRunnableJob):
+    """A Job related to an `IProduct`."""
+
+    id = Int(
+        title=_('DB ID'), required=True, readonly=True,
+        description=_("The tracking number of this job."))
+
+    job = Object(
+        title=_('The common Job attributes'),
+        schema=IJob,
+        required=True)
+
+    product = Object(
+        title=_('The product the job is for'),
+        schema=IProduct,
+        required=True)
+
+    metadata = Attribute('A dict of data for the job')
+
+
+class IProductJobSource(IJobSource):
+    """An interface for creating and finding `IProductJob`s."""
+
+    def create(product, metadata):
+        """Create a new `IProductJob`.
+
+        :param product: An IProduct.
+        :param metadata: a dict of configuration data for the job.
+            The data must be JSON compatible keys and values.
+        """
+
+    def find(product=None, date_since=None, job_type=None):
+        """Find `IProductJob`s that match the specified criteria.
+
+        :param product: Match jobs for specific product.
+        :param date_since: Match jobs since the specified date.
+        :param job_type: Match jobs of a specific type. Type is expected
+            to be a class name.
+        :return: A `ResultSet` yielding `IProductJob`.
+        """

=== added file 'lib/lp/registry/model/productjob.py'
--- lib/lp/registry/model/productjob.py	1970-01-01 00:00:00 +0000
+++ lib/lp/registry/model/productjob.py	2012-03-20 13:26:37 +0000
@@ -0,0 +1,151 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Jobs classes to update products and send notifications."""
+
+__metaclass__ = type
+__all__ = [
+    'ProductJob',
+    ]
+
+from lazr.delegates import delegates
+import simplejson
+from storm.expr import (
+    And,
+    )
+from storm.locals import (
+    Int,
+    Reference,
+    Unicode,
+    )
+from zope.interface import (
+    classProvides,
+    implements,
+    )
+
+from lp.registry.enums import ProductJobType
+from lp.registry.interfaces.product import (
+    IProduct,
+    )
+from lp.registry.interfaces.productjob import (
+    IProductJob,
+    IProductJobSource,
+    )
+from lp.registry.model.product import Product
+from lp.services.database.decoratedresultset import DecoratedResultSet
+from lp.services.database.enumcol import EnumCol
+from lp.services.database.lpstorm import (
+    IMasterStore,
+    IStore,
+    )
+from lp.services.database.stormbase import StormBase
+from lp.services.job.model.job import Job
+from lp.services.job.runner import BaseRunnableJob
+
+
+class ProductJob(StormBase):
+    """Base class for product jobs."""
+
+    implements(IProductJob)
+
+    __storm_table__ = 'ProductJob'
+
+    id = Int(primary=True)
+
+    job_id = Int(name='job')
+    job = Reference(job_id, Job.id)
+
+    product_id = Int(name='product')
+    product = Reference(product_id, Product.id)
+
+    job_type = EnumCol(enum=ProductJobType, notNull=True)
+
+    _json_data = Unicode('json_data')
+
+    @property
+    def metadata(self):
+        return simplejson.loads(self._json_data)
+
+    def __init__(self, product, job_type, metadata):
+        """Constructor.
+
+        :param product: The product the job is for.
+        :param job_type: The type job the product needs run.
+        :param metadata: A dict of JSON-compatible data to pass to the job.
+        """
+        super(ProductJob, self).__init__()
+        self.job = Job()
+        self.product = product
+        self.job_type = job_type
+        json_data = simplejson.dumps(metadata)
+        self._json_data = json_data.decode('utf-8')
+
+
+class ProductJobDerived(BaseRunnableJob):
+    """Intermediate class for deriving from ProductJob.
+
+    Storm classes can't simply be subclassed or you can end up with
+    multiple objects referencing the same row in the db. This class uses
+    lazr.delegates, which is a little bit simpler than storm's
+    inheritance solution to the problem. Subclasses need to override
+    the run() method.
+    """
+
+    delegates(IProductJob)
+    classProvides(IProductJobSource)
+
+    def __init__(self, job):
+        self.context = job
+
+    def __repr__(self):
+        return (
+            "<{self.__class__.__name__} for {self.product.name} "
+            "status={self.job.status}>").format(self=self)
+
+    @classmethod
+    def create(cls, product, metadata):
+        """See `IProductJob`."""
+        if not IProduct.providedBy(product):
+            raise TypeError("Product must be an IProduct: %s" % repr(product))
+        job = ProductJob(
+            product=product, job_type=cls.class_job_type, metadata=metadata)
+        return cls(job)
+
+    @classmethod
+    def find(cls, product, date_since=None, job_type=None):
+        """See `IPersonMergeJobSource`."""
+        conditions = [
+            ProductJob.job_id == Job.id,
+            ProductJob.product == product.id,
+            ]
+        if date_since is not None:
+            conditions.append(
+                Job.date_created >= date_since)
+        if job_type is not None:
+            conditions.append(
+                ProductJob.job_type == job_type)
+        return DecoratedResultSet(
+            IStore(ProductJob).find(
+                ProductJob, *conditions), cls)
+
+    @classmethod
+    def iterReady(cls):
+        """Iterate through all ready ProductJobs."""
+        store = IMasterStore(ProductJob)
+        jobs = store.find(
+            ProductJob,
+            And(ProductJob.job_type == cls.class_job_type,
+                ProductJob.job_id.is_in(Job.ready_jobs)))
+        return (cls(job) for job in jobs)
+
+    @property
+    def log_name(self):
+        return self.__class__.__name__
+
+    def getOopsVars(self):
+        """See `IRunnableJob`."""
+        vars = BaseRunnableJob.getOopsVars(self)
+        vars.extend([
+            ('product', self.context.product.name),
+            ])
+        return vars

=== added file 'lib/lp/registry/tests/test_productjob.py'
--- lib/lp/registry/tests/test_productjob.py	1970-01-01 00:00:00 +0000
+++ lib/lp/registry/tests/test_productjob.py	2012-03-20 13:26:37 +0000
@@ -0,0 +1,186 @@
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Tests for ProductJobs."""
+
+__metaclass__ = type
+
+from datetime import (
+    datetime,
+    timedelta,
+    )
+
+import pytz
+
+from zope.interface import (
+    classProvides,
+    implements,
+    )
+from zope.security.proxy import removeSecurityProxy
+
+from lp.registry.enums import ProductJobType
+from lp.registry.interfaces.productjob import (
+    IProductJob,
+    IProductJobSource,
+    )
+from lp.registry.model.productjob import (
+    ProductJob,
+    ProductJobDerived,
+    )
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import (
+    DatabaseFunctionalLayer,
+    LaunchpadZopelessLayer,
+    )
+
+
+class ProductJobTestCase(TestCaseWithFactory):
+    """Test case for basic ProductJob class."""
+
+    layer = LaunchpadZopelessLayer
+
+    def test_init(self):
+        product = self.factory.makeProduct()
+        metadata = ('some', 'arbitrary', 'metadata')
+        product_job = ProductJob(
+            product, ProductJobType.REVIEWER_NOTIFICATION, metadata)
+        self.assertEqual(product, product_job.product)
+        self.assertEqual(
+            ProductJobType.REVIEWER_NOTIFICATION, product_job.job_type)
+        expected_json_data = '["some", "arbitrary", "metadata"]'
+        self.assertEqual(expected_json_data, product_job._json_data)
+
+    def test_metadata(self):
+        # The python structure stored as json is returned as python.
+        product = self.factory.makeProduct()
+        metadata = {
+            'a_list': ('some', 'arbitrary', 'metadata'),
+            'a_number': 1,
+            'a_string': 'string',
+            }
+        product_job = ProductJob(
+            product, ProductJobType.REVIEWER_NOTIFICATION, metadata)
+        metadata['a_list'] = list(metadata['a_list'])
+        self.assertEqual(metadata, product_job.metadata)
+
+
+class IProductThingJob(IProductJob):
+    """An interface for testing derived job classes."""
+
+
+class IProductThingJobSource(IProductJobSource):
+    """An interface for testing derived job source classes."""
+
+
+class FakeProductJob(ProductJobDerived):
+    """A class that reuses other interfaces and types for testing."""
+    class_job_type = ProductJobType.REVIEWER_NOTIFICATION
+    implements(IProductThingJob)
+    classProvides(IProductThingJobSource)
+
+
+class OtherFakeProductJob(ProductJobDerived):
+    """A class that reuses other interfaces and types for testing."""
+    class_job_type = ProductJobType.COMMERCIAL_EXPIRED
+    implements(IProductThingJob)
+    classProvides(IProductThingJobSource)
+
+
+class ProductJobDerivedTestCase(TestCaseWithFactory):
+    """Test case for the ProductJobDerived class."""
+
+    layer = DatabaseFunctionalLayer
+
+    def test_repr(self):
+        product = self.factory.makeProduct('fnord')
+        metadata = {'foo': 'bar'}
+        job = FakeProductJob.create(product, metadata)
+        self.assertEqual(
+            '<FakeProductJob for fnord status=Waiting>', repr(job))
+
+    def test_create_success(self):
+        # Create an instance of ProductJobDerived that delegates to
+        # ProductJob.
+        product = self.factory.makeProduct()
+        metadata = {'foo': 'bar'}
+        self.assertIs(True, IProductJobSource.providedBy(ProductJobDerived))
+        job = FakeProductJob.create(product, metadata)
+        self.assertIsInstance(job, ProductJobDerived)
+        self.assertIs(True, IProductJob.providedBy(job))
+        self.assertIs(True, IProductJob.providedBy(job.context))
+
+    def test_create_raises_error(self):
+        # ProductJobDerived.create() raises an error because it
+        # needs to be subclassed to work properly.
+        product = self.factory.makeProduct()
+        metadata = {'foo': 'bar'}
+        self.assertRaises(
+            AttributeError, ProductJobDerived.create, product, metadata)
+
+    def test_iterReady(self):
+        # iterReady finds job in the READY status that are of the same type.
+        product = self.factory.makeProduct()
+        metadata = {'foo': 'bar'}
+        job_1 = FakeProductJob.create(product, metadata)
+        job_2 = FakeProductJob.create(product, metadata)
+        job_2.start()
+        OtherFakeProductJob.create(product, metadata)
+        jobs = list(FakeProductJob.iterReady())
+        self.assertEqual(1, len(jobs))
+        self.assertEqual(job_1, jobs[0])
+
+    def test_find_product(self):
+        # Find all the jobs for a product regardless of date or job type.
+        product = self.factory.makeProduct()
+        metadata = {'foo': 'bar'}
+        job_1 = FakeProductJob.create(product, metadata)
+        job_2 = OtherFakeProductJob.create(product, metadata)
+        FakeProductJob.create(self.factory.makeProduct(), metadata)
+        jobs = list(ProductJobDerived.find(product=product))
+        self.assertEqual(2, len(jobs))
+        self.assertContentEqual([job_1.id, job_2.id], [job.id for job in jobs])
+
+    def test_find_job_type(self):
+        # Find all the jobs for a product and job_type regardless of date.
+        product = self.factory.makeProduct()
+        metadata = {'foo': 'bar'}
+        job_1 = FakeProductJob.create(product, metadata)
+        job_2 = FakeProductJob.create(product, metadata)
+        OtherFakeProductJob.create(product, metadata)
+        jobs = list(ProductJobDerived.find(
+            product, job_type=ProductJobType.REVIEWER_NOTIFICATION))
+        self.assertEqual(2, len(jobs))
+        self.assertContentEqual([job_1.id, job_2.id], [job.id for job in jobs])
+
+    def test_find_date_since(self):
+        # Find all the jobs for a product since a date regardless of job_type.
+        now = datetime.now(pytz.utc)
+        seven_days_ago = now - timedelta(7)
+        thirty_days_ago = now - timedelta(30)
+        product = self.factory.makeProduct()
+        metadata = {'foo': 'bar'}
+        job_1 = FakeProductJob.create(product, metadata)
+        removeSecurityProxy(job_1.job).date_created = thirty_days_ago
+        job_2 = FakeProductJob.create(product, metadata)
+        removeSecurityProxy(job_2.job).date_created = seven_days_ago
+        job_3 = OtherFakeProductJob.create(product, metadata)
+        removeSecurityProxy(job_3.job).date_created = now
+        jobs = list(ProductJobDerived.find(product, date_since=seven_days_ago))
+        self.assertEqual(2, len(jobs))
+        self.assertContentEqual([job_2.id, job_3.id], [job.id for job in jobs])
+
+    def test_log_name(self):
+        # The log_name is the name of the implementing class.
+        product = self.factory.makeProduct('fnord')
+        metadata = {'foo': 'bar'}
+        job = FakeProductJob.create(product, metadata)
+        self.assertEqual('FakeProductJob', job.log_name)
+
+    def test_getOopsVars(self):
+        # The project name is added to the oops vars.
+        product = self.factory.makeProduct('fnord')
+        metadata = {'foo': 'bar'}
+        job = FakeProductJob.create(product, metadata)
+        oops_vars = job.getOopsVars()
+        self.assertIs(True, len(oops_vars) > 1)
+        self.assertIn(('product', product.name), oops_vars)