Merge lp:~cjwatson/launchpad/archive-file-model into lp:launchpad

[Colin Watson]

Add model for ArchiveFile.

Add model for ArchiveFile. This will be used to implement by-hash, and the current set of methods is tailored to that. It should be possible to use it for diskless archives as well later on.

[William Grant (wgrant)]

review: Approve (code)

=== modified file 'lib/lp/soyuz/configure.zcml'
--- lib/lp/soyuz/configure.zcml	2016-03-04 14:18:23 +0000
+++ lib/lp/soyuz/configure.zcml	2016-03-18 15:10:41 +0000
@@ -422,6 +422,23 @@
             interface="lp.soyuz.interfaces.archive.IArchiveSet"/>
     </securedutility>
 
+    <!-- ArchiveFile -->
+
+    <class class="lp.soyuz.model.archivefile.ArchiveFile">
+        <allow interface="lp.soyuz.interfaces.archivefile.IArchiveFile"/>
+    </class>
+
+    <!-- ArchiveFileSet -->
+
+    <class class="lp.soyuz.model.archivefile.ArchiveFileSet">
+        <allow interface="lp.soyuz.interfaces.archivefile.IArchiveFileSet"/>
+    </class>
+    <securedutility
+        class="lp.soyuz.model.archivefile.ArchiveFileSet"
+        provides="lp.soyuz.interfaces.archivefile.IArchiveFileSet">
+        <allow interface="lp.soyuz.interfaces.archivefile.IArchiveFileSet"/>
+    </securedutility>
+
     <!-- ArchiveJob -->
 
     <class class="lp.soyuz.model.archivejob.ArchiveJob">

=== added file 'lib/lp/soyuz/interfaces/archivefile.py'
--- lib/lp/soyuz/interfaces/archivefile.py	1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/interfaces/archivefile.py	2016-03-18 15:10:41 +0000
@@ -0,0 +1,112 @@
+# Copyright 2016 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Interface for a file in an archive."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'IArchiveFile',
+    'IArchiveFileSet',
+    ]
+
+from lazr.restful.fields import Reference
+from zope.interface import Interface
+from zope.schema import (
+    Datetime,
+    Int,
+    TextLine,
+    )
+
+from lp import _
+from lp.services.librarian.interfaces import ILibraryFileAlias
+from lp.soyuz.interfaces.archive import IArchive
+
+
+class IArchiveFile(Interface):
+    """A file in an archive.
+
+    This covers files that are not published in the archive's package pool,
+    such as the Packages and Sources index files.
+    """
+
+    id = Int(title=_("ID"), required=True, readonly=True)
+
+    archive = Reference(
+        title=_("The archive containing the index file."),
+        schema=IArchive, required=True, readonly=True)
+
+    container = TextLine(
+        title=_("An identifier for the component that manages this file."),
+        required=True, readonly=True)
+
+    path = TextLine(
+        title=_("The path to the index file within the published archive."),
+        required=True, readonly=True)
+
+    library_file = Reference(
+        title=_("The index file in the librarian."),
+        schema=ILibraryFileAlias, required=True, readonly=True)
+
+    scheduled_deletion_date = Datetime(
+        title=_("The date when this file should stop being published."),
+        required=False, readonly=False)
+
+
+class IArchiveFileSet(Interface):
+    """Bulk operations on files in an archive."""
+
+    def new(archive, container, path, library_file):
+        """Create a new `IArchiveFile`.
+
+        :param archive: The `IArchive` containing the new file.
+        :param container: An identifier for the component that manages this
+            file.
+        :param path: The path to the new file within its archive.
+        :param library_file: The `ILibraryFileAlias` embodying the new file.
+        """
+
+    def newFromFile(archive, container, path, fileobj, size, content_type):
+        """Create a new `IArchiveFile` from a file on the file system.
+
+        :param archive: The `IArchive` containing the new file.
+        :param container: An identifier for the component that manages this
+            file.
+        :param path: The path to the new file within its archive.
+        :param fileobj: A file-like object to read the data from.
+        :param size: The size of the file in bytes.
+        :param content_type: The MIME type of the file.
+        """
+
+    def getByArchive(archive, container=None, eager_load=False):
+        """Get files in an archive.
+
+        :param archive: Return files in this `IArchive`.
+        :param container: Return only files with this container.
+        :param eager_load: If True, preload related `LibraryFileAlias` and
+            `LibraryFileContent` rows.
+        """
+
+    def scheduleDeletion(archive_files, stay_of_execution):
+        """Schedule these archive files for future deletion.
+
+        :param archive_files: The `IArchiveFile`s to schedule for deletion.
+        :param stay_of_execution: A `timedelta`; schedule files for deletion
+            this amount of time in the future.
+        """
+
+    def getContainersToReap(archive, container_prefix=None):
+        """Return containers in this archive with files that should be reaped.
+
+        :param archive: Return containers in this `IArchive`.
+        :param container_prefix: Return only containers that start with this
+            prefix.
+        """
+
+    def reap(archive, container=None):
+        """Delete archive files that are past their scheduled deletion date.
+
+        :param archive: Delete files from this `IArchive`.
+        :param container: Delete only files with this container.
+        """

=== added file 'lib/lp/soyuz/model/archivefile.py'
--- lib/lp/soyuz/model/archivefile.py	1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/model/archivefile.py	2016-03-18 15:10:41 +0000
@@ -0,0 +1,143 @@
+# Copyright 2016 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""A file in an archive."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'ArchiveFile',
+    'ArchiveFileSet',
+    ]
+
+import os.path
+
+import pytz
+from storm.locals import (
+    DateTime,
+    Int,
+    Reference,
+    Storm,
+    Unicode,
+    )
+from zope.component import getUtility
+from zope.interface import implementer
+
+from lp.services.database.bulk import load_related
+from lp.services.database.constants import UTC_NOW
+from lp.services.database.decoratedresultset import DecoratedResultSet
+from lp.services.database.interfaces import (
+    IMasterStore,
+    IStore,
+    )
+from lp.services.librarian.interfaces import ILibraryFileAliasSet
+from lp.services.librarian.model import (
+    LibraryFileAlias,
+    LibraryFileContent,
+    )
+from lp.soyuz.interfaces.archivefile import (
+    IArchiveFile,
+    IArchiveFileSet,
+    )
+
+
+@implementer(IArchiveFile)
+class ArchiveFile(Storm):
+    """See `IArchiveFile`."""
+
+    __storm_table__ = 'ArchiveFile'
+
+    id = Int(primary=True)
+
+    archive_id = Int(name='archive', allow_none=False)
+    archive = Reference(archive_id, 'Archive.id')
+
+    container = Unicode(name='container', allow_none=False)
+
+    path = Unicode(name='path', allow_none=False)
+
+    library_file_id = Int(name='library_file', allow_none=False)
+    library_file = Reference(library_file_id, 'LibraryFileAlias.id')
+
+    scheduled_deletion_date = DateTime(
+        name='scheduled_deletion_date', tzinfo=pytz.UTC, allow_none=True)
+
+    def __init__(self, archive, container, path, library_file):
+        """Construct an `ArchiveFile`."""
+        super(ArchiveFile, self).__init__()
+        self.archive = archive
+        self.container = container
+        self.path = path
+        self.library_file = library_file
+        self.scheduled_deletion_date = None
+
+
+@implementer(IArchiveFileSet)
+class ArchiveFileSet:
+    """See `IArchiveFileSet`."""
+
+    @staticmethod
+    def new(archive, container, path, library_file):
+        """See `IArchiveFileSet`."""
+        archive_file = ArchiveFile(archive, container, path, library_file)
+        IMasterStore(ArchiveFile).add(archive_file)
+        return archive_file
+
+    @classmethod
+    def newFromFile(cls, archive, container, path, fileobj, size,
+                    content_type):
+        library_file = getUtility(ILibraryFileAliasSet).create(
+            os.path.basename(path), size, fileobj, content_type,
+            restricted=archive.private)
+        return cls.new(archive, container, path, library_file)
+
+    @staticmethod
+    def getByArchive(archive, container=None, eager_load=False):
+        """See `IArchiveFileSet`."""
+        clauses = [ArchiveFile.archive == archive]
+        # XXX cjwatson 2016-03-15: We'll need some more sophisticated way to
+        # match containers once we're using them for custom uploads.
+        if container is not None:
+            clauses.append(ArchiveFile.container == container)
+        archive_files = IStore(ArchiveFile).find(ArchiveFile, *clauses)
+
+        def eager_load(rows):
+            lfas = load_related(LibraryFileAlias, rows, ["library_file_id"])
+            load_related(LibraryFileContent, lfas, ["contentID"])
+
+        if eager_load:
+            return DecoratedResultSet(archive_files, pre_iter_hook=eager_load)
+        else:
+            return archive_files
+
+    @staticmethod
+    def scheduleDeletion(archive_files, stay_of_execution):
+        """See `IArchiveFileSet`."""
+        archive_file_ids = set(
+            archive_file.id for archive_file in archive_files)
+        rows = IMasterStore(ArchiveFile).find(
+            ArchiveFile, ArchiveFile.id.is_in(archive_file_ids))
+        rows.set(scheduled_deletion_date=UTC_NOW + stay_of_execution)
+
+    @staticmethod
+    def getContainersToReap(archive, container_prefix=None):
+        clauses = [
+            ArchiveFile.archive == archive,
+            ArchiveFile.scheduled_deletion_date < UTC_NOW,
+            ]
+        if container_prefix is not None:
+            clauses.append(ArchiveFile.container.startswith(container_prefix))
+        return IStore(ArchiveFile).find(
+            ArchiveFile.container, *clauses).group_by(ArchiveFile.container)
+
+    @staticmethod
+    def reap(archive, container=None):
+        """See `IArchiveFileSet`."""
+        clauses = [
+            ArchiveFile.archive == archive,
+            ArchiveFile.scheduled_deletion_date < UTC_NOW,
+            ]
+        if container is not None:
+            clauses.append(ArchiveFile.container == container)
+        IMasterStore(ArchiveFile).find(ArchiveFile, *clauses).remove()

=== added file 'lib/lp/soyuz/tests/test_archivefile.py'
--- lib/lp/soyuz/tests/test_archivefile.py	1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/tests/test_archivefile.py	2016-03-18 15:10:41 +0000
@@ -0,0 +1,154 @@
+# Copyright 2016 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""ArchiveFile tests."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+
+from datetime import (
+    datetime,
+    timedelta,
+    )
+import os
+
+import pytz
+from testtools.matchers import LessThan
+import transaction
+from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
+
+from lp.services.osutils import open_for_writing
+from lp.soyuz.interfaces.archivefile import IArchiveFileSet
+from lp.testing import TestCaseWithFactory
+from lp.testing.layers import LaunchpadZopelessLayer
+
+
+class TestArchiveFile(TestCaseWithFactory):
+
+    layer = LaunchpadZopelessLayer
+
+    def test_new(self):
+        archive = self.factory.makeArchive()
+        library_file = self.factory.makeLibraryFileAlias()
+        archive_file = getUtility(IArchiveFileSet).new(
+            archive, "foo", "dists/foo", library_file)
+        self.assertEqual(archive, archive_file.archive)
+        self.assertEqual("foo", archive_file.container)
+        self.assertEqual("dists/foo", archive_file.path)
+        self.assertEqual(library_file, archive_file.library_file)
+        self.assertIsNone(archive_file.scheduled_deletion_date)
+
+    def test_newFromFile(self):
+        root = self.makeTemporaryDirectory()
+        with open_for_writing(os.path.join(root, "dists/foo"), "w") as f:
+            f.write("abc\n")
+        archive = self.factory.makeArchive()
+        with open(os.path.join(root, "dists/foo"), "rb") as f:
+            archive_file = getUtility(IArchiveFileSet).newFromFile(
+                archive, "foo", "dists/foo", f, 4, "text/plain")
+        transaction.commit()
+        self.assertEqual(archive, archive_file.archive)
+        self.assertEqual("foo", archive_file.container)
+        self.assertEqual("dists/foo", archive_file.path)
+        archive_file.library_file.open()
+        try:
+            self.assertEqual("abc\n", archive_file.library_file.read())
+        finally:
+            archive_file.library_file.close()
+        self.assertIsNone(archive_file.scheduled_deletion_date)
+
+    def test_getByArchive(self):
+        archives = [self.factory.makeArchive(), self.factory.makeArchive()]
+        archive_files = []
+        for archive in archives:
+            archive_files.append(self.factory.makeArchiveFile(archive=archive))
+            archive_files.append(self.factory.makeArchiveFile(
+                archive=archive, container="foo"))
+        archive_file_set = getUtility(IArchiveFileSet)
+        self.assertContentEqual(
+            archive_files[:2], archive_file_set.getByArchive(archives[0]))
+        self.assertContentEqual(
+            [archive_files[1]],
+            archive_file_set.getByArchive(archives[0], container="foo"))
+        self.assertContentEqual(
+            [], archive_file_set.getByArchive(archives[0], container="bar"))
+        self.assertContentEqual(
+            archive_files[2:], archive_file_set.getByArchive(archives[1]))
+        self.assertContentEqual(
+            [archive_files[3]],
+            archive_file_set.getByArchive(archives[1], container="foo"))
+        self.assertContentEqual(
+            [], archive_file_set.getByArchive(archives[1], container="bar"))
+
+    def test_scheduleDeletion(self):
+        archive_files = [self.factory.makeArchiveFile() for _ in range(3)]
+        getUtility(IArchiveFileSet).scheduleDeletion(
+            archive_files[:2], timedelta(days=1))
+        tomorrow = datetime.now(pytz.UTC) + timedelta(days=1)
+        # Allow a bit of timing slack for slow tests.
+        self.assertThat(
+            tomorrow - archive_files[0].scheduled_deletion_date,
+            LessThan(timedelta(minutes=5)))
+        self.assertThat(
+            tomorrow - archive_files[1].scheduled_deletion_date,
+            LessThan(timedelta(minutes=5)))
+        self.assertIsNone(archive_files[2].scheduled_deletion_date)
+
+    def test_getContainersToReap(self):
+        archive = self.factory.makeArchive()
+        archive_files = []
+        for container in ("release:foo", "other:bar", "baz"):
+            for _ in range(2):
+                archive_files.append(self.factory.makeArchiveFile(
+                    archive=archive, container=container))
+        other_archive = self.factory.makeArchive()
+        archive_files.append(self.factory.makeArchiveFile(
+            archive=other_archive, container="baz"))
+        now = datetime.now(pytz.UTC)
+        removeSecurityProxy(archive_files[0]).scheduled_deletion_date = (
+            now - timedelta(days=1))
+        removeSecurityProxy(archive_files[1]).scheduled_deletion_date = (
+            now - timedelta(days=1))
+        removeSecurityProxy(archive_files[2]).scheduled_deletion_date = (
+            now + timedelta(days=1))
+        removeSecurityProxy(archive_files[6]).scheduled_deletion_date = (
+            now - timedelta(days=1))
+        archive_file_set = getUtility(IArchiveFileSet)
+        self.assertContentEqual(
+            ["release:foo"], archive_file_set.getContainersToReap(archive))
+        self.assertContentEqual(
+            ["baz"], archive_file_set.getContainersToReap(other_archive))
+        removeSecurityProxy(archive_files[3]).scheduled_deletion_date = (
+            now - timedelta(days=1))
+        self.assertContentEqual(
+            ["release:foo", "other:bar"],
+            archive_file_set.getContainersToReap(archive))
+        self.assertContentEqual(
+            ["release:foo"],
+            archive_file_set.getContainersToReap(
+                archive, container_prefix="release:"))
+
+    def test_reap(self):
+        archive = self.factory.makeArchive()
+        archive_files = [
+            self.factory.makeArchiveFile(archive=archive, container="foo")
+            for _ in range(3)]
+        archive_files.append(self.factory.makeArchiveFile(archive=archive))
+        other_archive = self.factory.makeArchive()
+        archive_files.append(
+            self.factory.makeArchiveFile(archive=other_archive))
+        now = datetime.now(pytz.UTC)
+        removeSecurityProxy(archive_files[0]).scheduled_deletion_date = (
+            now - timedelta(days=1))
+        removeSecurityProxy(archive_files[1]).scheduled_deletion_date = (
+            now + timedelta(days=1))
+        removeSecurityProxy(archive_files[3]).scheduled_deletion_date = (
+            now - timedelta(days=1))
+        removeSecurityProxy(archive_files[4]).scheduled_deletion_date = (
+            now - timedelta(days=1))
+        archive_file_set = getUtility(IArchiveFileSet)
+        archive_file_set.reap(archive, container="foo")
+        self.assertContentEqual(
+            archive_files[1:4], archive_file_set.getByArchive(archive))

=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py	2016-03-09 01:37:56 +0000
+++ lib/lp/testing/factory.py	2016-03-18 15:10:41 +0000
@@ -289,6 +289,7 @@
     default_name_by_purpose,
     IArchiveSet,
     )
+from lp.soyuz.interfaces.archivefile import IArchiveFileSet
 from lp.soyuz.interfaces.archivepermission import IArchivePermissionSet
 from lp.soyuz.interfaces.binarypackagebuild import IBinaryPackageBuildSet
 from lp.soyuz.interfaces.binarypackagename import IBinaryPackageNameSet
@@ -2878,6 +2879,20 @@
         permission_set.newQueueAdmin(archive, person, 'main')
         return person
 
+    def makeArchiveFile(self, archive=None, container=None, path=None,
+                        library_file=None):
+        if archive is None:
+            archive = self.makeArchive()
+        if container is None:
+            container = self.getUniqueUnicode()
+        if path is None:
+            path = self.getUniqueUnicode()
+        if library_file is None:
+            library_file = self.makeLibraryFileAlias()
+        return getUtility(IArchiveFileSet).new(
+            archive=archive, container=container, path=path,
+            library_file=library_file)
+
     def makeBuilder(self, processors=None, url=None, name=None, title=None,
                     owner=None, active=True, virtualized=True, vm_host=None,
                     vm_reset_protocol=None, manual=False):