Merge lp:~cjwatson/launchpad/git-ref-scanner into lp:launchpad

Proposed by Colin Watson on 2015-03-12
Status: Merged
Approved by: Colin Watson on 2015-03-17
Approved revision: no longer in the source branch.
Merged at revision: 17402
Proposed branch: lp:~cjwatson/launchpad/git-ref-scanner
Merge into: lp:launchpad
Diff against target: 1526 lines (+1080/-11)
25 files modified
lib/lp/code/configure.zcml (+22/-0)
lib/lp/code/enums.py (+33/-1)
lib/lp/code/errors.py (+5/-0)
lib/lp/code/githosting.py (+23/-3)
lib/lp/code/interfaces/gitapi.py (+11/-0)
lib/lp/code/interfaces/gitjob.py (+53/-0)
lib/lp/code/interfaces/gitlookup.py (+6/-0)
lib/lp/code/interfaces/gitref.py (+43/-0)
lib/lp/code/interfaces/gitrepository.py (+35/-0)
lib/lp/code/model/gitjob.py (+197/-0)
lib/lp/code/model/gitlookup.py (+10/-0)
lib/lp/code/model/gitref.py (+37/-0)
lib/lp/code/model/gitrepository.py (+154/-1)
lib/lp/code/model/tests/test_gitjob.py (+126/-0)
lib/lp/code/model/tests/test_gitlookup.py (+20/-0)
lib/lp/code/model/tests/test_gitrepository.py (+172/-0)
lib/lp/code/xmlrpc/git.py (+10/-0)
lib/lp/code/xmlrpc/tests/test_git.py (+17/-0)
lib/lp/scripts/garbo.py (+19/-1)
lib/lp/scripts/tests/test_garbo.py (+48/-1)
lib/lp/security.py (+10/-0)
lib/lp/services/config/schema-lazr.conf (+4/-0)
lib/lp/services/database/locking.py (+6/-1)
lib/lp/services/database/stormexpr.py (+4/-3)
lib/lp/testing/factory.py (+15/-0)
To merge this branch: bzr merge lp:~cjwatson/launchpad/git-ref-scanner
Reviewer Review Type Date Requested Status
William Grant code 2015-03-12 Approve on 2015-03-17
Review via email: mp+252763@code.launchpad.net

Commit message

Add a ref scanning job for Git repositories.

Description of the change

Add a ref scanning job for Git repositories.

This needs https://code.launchpad.net/~cjwatson/launchpad/db-git-more/+merge/252672 for the GitJob table and the GitRef.object_type column. I haven't filled in the other (nullable) GitRef columns here yet; that can wait for a later branch.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)
Colin Watson (cjwatson) :
William Grant (wgrant) :
review: Approve (code)
Colin Watson (cjwatson) :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/code/configure.zcml'
2--- lib/lp/code/configure.zcml 2015-03-06 16:31:30 +0000
3+++ lib/lp/code/configure.zcml 2015-03-17 10:51:52 +0000
4@@ -859,6 +859,14 @@
5 <allow interface="lp.code.interfaces.gitnamespace.IGitNamespaceSet" />
6 </securedutility>
7
8+ <!-- GitRef -->
9+
10+ <class class="lp.code.model.gitref.GitRef">
11+ <require
12+ permission="launchpad.View"
13+ interface="lp.code.interfaces.gitref.IGitRef" />
14+ </class>
15+
16 <!-- GitCollection -->
17
18 <class class="lp.code.model.gitcollection.GenericGitCollection">
19@@ -933,6 +941,20 @@
20 <adapter factory="lp.code.model.gitlookup.DistributionGitTraversable" />
21 <adapter factory="lp.code.model.gitlookup.DistributionSourcePackageGitTraversable" />
22
23+ <!-- Git-related jobs -->
24+ <class class="lp.code.model.gitjob.GitJob">
25+ <allow interface="lp.code.interfaces.gitjob.IGitJob" />
26+ </class>
27+ <securedutility
28+ component="lp.code.model.gitjob.GitRefScanJob"
29+ provides="lp.code.interfaces.gitjob.IGitRefScanJobSource">
30+ <allow interface="lp.code.interfaces.gitjob.IGitRefScanJobSource" />
31+ </securedutility>
32+ <class class="lp.code.model.gitjob.GitRefScanJob">
33+ <allow interface="lp.code.interfaces.gitjob.IGitJob" />
34+ <allow interface="lp.code.interfaces.gitjob.IGitRefScanJob" />
35+ </class>
36+
37 <lp:help-folder folder="help" name="+help-code" />
38
39 <!-- Diffs -->
40
41=== modified file 'lib/lp/code/enums.py'
42--- lib/lp/code/enums.py 2014-02-24 07:19:52 +0000
43+++ lib/lp/code/enums.py 2015-03-17 10:51:52 +0000
44@@ -1,4 +1,4 @@
45-# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
46+# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
47 # GNU Affero General Public License version 3 (see the file LICENSE).
48
49 """Enumerations used in the lp/code modules."""
50@@ -20,6 +20,7 @@
51 'CodeImportReviewStatus',
52 'CodeReviewNotificationLevel',
53 'CodeReviewVote',
54+ 'GitObjectType',
55 'NON_CVS_RCS_TYPES',
56 'RevisionControlSystems',
57 'UICreatableBranchType',
58@@ -115,6 +116,37 @@
59 use_template(BranchType, exclude='IMPORTED')
60
61
62+class GitObjectType(DBEnumeratedType):
63+ """Git Object Type
64+
65+ Keep these in sync with the concrete GIT_OBJ_* enum values in libgit2.
66+ """
67+
68+ COMMIT = DBItem(1, """
69+ Commit
70+
71+ A commit object.
72+ """)
73+
74+ TREE = DBItem(2, """
75+ Tree
76+
77+ A tree (directory listing) object.
78+ """)
79+
80+ BLOB = DBItem(3, """
81+ Blob
82+
83+ A file revision object.
84+ """)
85+
86+ TAG = DBItem(4, """
87+ Tag
88+
89+ An annotated tag object.
90+ """)
91+
92+
93 class BranchLifecycleStatusFilter(EnumeratedType):
94 """Branch Lifecycle Status Filter
95
96
97=== modified file 'lib/lp/code/errors.py'
98--- lib/lp/code/errors.py 2015-03-04 18:27:40 +0000
99+++ lib/lp/code/errors.py 2015-03-17 10:51:52 +0000
100@@ -36,6 +36,7 @@
101 'GitRepositoryCreatorNotMemberOfOwnerTeam',
102 'GitRepositoryCreatorNotOwner',
103 'GitRepositoryExists',
104+ 'GitRepositoryRefScanFault',
105 'GitTargetError',
106 'InvalidBranchMergeProposal',
107 'InvalidMergeQueueConfig',
108@@ -381,6 +382,10 @@
109 """Raised when there is a hosting fault creating a Git repository."""
110
111
112+class GitRepositoryRefScanFault(Exception):
113+ """Raised when there is a fault getting the refs for a repository."""
114+
115+
116 class GitTargetError(Exception):
117 """Raised when there is an error determining a Git repository target."""
118
119
120=== modified file 'lib/lp/code/githosting.py'
121--- lib/lp/code/githosting.py 2015-03-03 14:51:10 +0000
122+++ lib/lp/code/githosting.py 2015-03-17 10:51:52 +0000
123@@ -9,11 +9,14 @@
124 ]
125
126 import json
127-from urlparse import urljoin
128
129+from bzrlib import urlutils
130 import requests
131
132-from lp.code.errors import GitRepositoryCreationFault
133+from lp.code.errors import (
134+ GitRepositoryCreationFault,
135+ GitRepositoryRefScanFault,
136+ )
137
138
139 class GitHostingClient:
140@@ -40,7 +43,7 @@
141 # should just use post(json=) and drop the explicit Content-Type
142 # header.
143 response = self._makeSession().post(
144- urljoin(self.endpoint, "repo"),
145+ urlutils.join(self.endpoint, "repo"),
146 headers={"Content-Type": "application/json"},
147 data=json.dumps({"repo_path": path, "bare_repo": True}),
148 timeout=self.timeout)
149@@ -50,3 +53,20 @@
150 if response.status_code != 200:
151 raise GitRepositoryCreationFault(
152 "Failed to create Git repository: %s" % response.text)
153+
154+ def get_refs(self, path):
155+ try:
156+ response = self._makeSession().get(
157+ urlutils.join(self.endpoint, "repo", path, "refs"),
158+ timeout=self.timeout)
159+ except Exception as e:
160+ raise GitRepositoryRefScanFault(
161+ "Failed to get refs from Git repository: %s" % unicode(e))
162+ if response.status_code != 200:
163+ raise GitRepositoryRefScanFault(
164+ "Failed to get refs from Git repository: %s" % response.text)
165+ try:
166+ return response.json()
167+ except ValueError as e:
168+ raise GitRepositoryRefScanFault(
169+ "Failed to decode ref-scan response: %s" % unicode(e))
170
171=== modified file 'lib/lp/code/interfaces/gitapi.py'
172--- lib/lp/code/interfaces/gitapi.py 2015-03-04 11:12:06 +0000
173+++ lib/lp/code/interfaces/gitapi.py 2015-03-17 10:51:52 +0000
174@@ -49,3 +49,14 @@
175 "writable", whose value is True if the requester can push to
176 this repository, otherwise False.
177 """
178+
179+ def notify(translated_path):
180+ """Notify of a change to the repository at 'translated_path'.
181+
182+ :param translated_path: The translated path to the repository. (We
183+ use translated paths here in order to avoid problems with
184+ repository names etc. being changed during a push.)
185+
186+ :returns: A `NotFound` fault if no repository can be found for
187+ 'translated_path'; otherwise None.
188+ """
189
190=== added file 'lib/lp/code/interfaces/gitjob.py'
191--- lib/lp/code/interfaces/gitjob.py 1970-01-01 00:00:00 +0000
192+++ lib/lp/code/interfaces/gitjob.py 2015-03-17 10:51:52 +0000
193@@ -0,0 +1,53 @@
194+# Copyright 2015 Canonical Ltd. This software is licensed under the
195+# GNU Affero General Public License version 3 (see the file LICENSE).
196+
197+"""GitJob interfaces."""
198+
199+__metaclass__ = type
200+
201+__all__ = [
202+ 'IGitJob',
203+ 'IGitRefScanJob',
204+ 'IGitRefScanJobSource',
205+ ]
206+
207+from lazr.restful.fields import Reference
208+from zope.interface import (
209+ Attribute,
210+ Interface,
211+ )
212+
213+from lp import _
214+from lp.code.interfaces.gitrepository import IGitRepository
215+from lp.services.job.interfaces.job import (
216+ IJob,
217+ IJobSource,
218+ IRunnableJob,
219+ )
220+
221+
222+class IGitJob(Interface):
223+ """A job related to a Git repository."""
224+
225+ job = Reference(
226+ title=_("The common Job attributes."), schema=IJob,
227+ required=True, readonly=True)
228+
229+ repository = Reference(
230+ title=_("The Git repository to use for this job."),
231+ schema=IGitRepository, required=True, readonly=True)
232+
233+ metadata = Attribute(_("A dict of data about the job."))
234+
235+
236+class IGitRefScanJob(IRunnableJob):
237+ """A Job that scans a Git repository for its current list of references."""
238+
239+
240+class IGitRefScanJobSource(IJobSource):
241+
242+ def create(repository):
243+ """Scan a repository for refs.
244+
245+ :param repository: The database repository to scan.
246+ """
247
248=== modified file 'lib/lp/code/interfaces/gitlookup.py'
249--- lib/lp/code/interfaces/gitlookup.py 2015-03-05 11:39:06 +0000
250+++ lib/lp/code/interfaces/gitlookup.py 2015-03-17 10:51:52 +0000
251@@ -90,6 +90,12 @@
252 Return the default value if there is no such repository.
253 """
254
255+ def getByHostingPath(path):
256+ """Get information about a given path on the hosting backend.
257+
258+ :return: An `IGitRepository`, or None.
259+ """
260+
261 def getByUniqueName(unique_name):
262 """Find a repository by its unique name.
263
264
265=== added file 'lib/lp/code/interfaces/gitref.py'
266--- lib/lp/code/interfaces/gitref.py 1970-01-01 00:00:00 +0000
267+++ lib/lp/code/interfaces/gitref.py 2015-03-17 10:51:52 +0000
268@@ -0,0 +1,43 @@
269+# Copyright 2015 Canonical Ltd. This software is licensed under the
270+# GNU Affero General Public License version 3 (see the file LICENSE).
271+
272+"""Git reference ("ref") interfaces."""
273+
274+__metaclass__ = type
275+
276+__all__ = [
277+ 'IGitRef',
278+ ]
279+
280+from zope.interface import (
281+ Attribute,
282+ Interface,
283+ )
284+from zope.schema import (
285+ Choice,
286+ TextLine,
287+ )
288+
289+from lp import _
290+from lp.code.enums import GitObjectType
291+
292+
293+class IGitRef(Interface):
294+ """A reference in a Git repository."""
295+
296+ repository = Attribute("The Git repository containing this reference.")
297+
298+ path = TextLine(
299+ title=_("Path"), required=True, readonly=True,
300+ description=_(
301+ "The full path of this reference, e.g. refs/heads/master."))
302+
303+ commit_sha1 = TextLine(
304+ title=_("Commit SHA-1"), required=True, readonly=True,
305+ description=_(
306+ "The full SHA-1 object name of the commit object referenced by "
307+ "this reference."))
308+
309+ object_type = Choice(
310+ title=_("Object type"), required=True, readonly=True,
311+ vocabulary=GitObjectType)
312
313=== modified file 'lib/lp/code/interfaces/gitrepository.py'
314--- lib/lp/code/interfaces/gitrepository.py 2015-03-06 16:31:30 +0000
315+++ lib/lp/code/interfaces/gitrepository.py 2015-03-17 10:51:52 +0000
316@@ -186,6 +186,41 @@
317 "'lp:' plus a shortcut version of the path via that target. "
318 "Otherwise it is simply 'lp:' plus the unique name.")))
319
320+ refs = Attribute("The references present in this repository.")
321+
322+ def getRefByPath(path):
323+ """Look up a single reference in this repository by path.
324+
325+ :param path: A string to look up as a path.
326+
327+ :return: An `IGitRef`, or None.
328+ """
329+
330+ def createOrUpdateRefs(refs_info, get_objects=False):
331+ """Create or update a set of references in this repository.
332+
333+ :param refs_info: A dict mapping ref paths to
334+ {"sha1": sha1, "type": `GitObjectType`}.
335+ :param get_objects: Return the created/updated references.
336+
337+ :return: A list of the created/updated references if get_objects,
338+ otherwise None.
339+ """
340+
341+ def removeRefs(paths):
342+ """Remove a set of references in this repository.
343+
344+ :params paths: An iterable of paths.
345+ """
346+
347+ def synchroniseRefs(hosting_refs, logger=None):
348+ """Synchronise references with those from the hosting service.
349+
350+ :param hosting_refs: A dictionary of reference information returned
351+ from the hosting service's `/repo/PATH/refs` collection.
352+ :param logger: An optional logger.
353+ """
354+
355 def setOwnerDefault(value):
356 """Set whether this repository is the default for its owner-target.
357
358
359=== added file 'lib/lp/code/model/gitjob.py'
360--- lib/lp/code/model/gitjob.py 1970-01-01 00:00:00 +0000
361+++ lib/lp/code/model/gitjob.py 2015-03-17 10:51:52 +0000
362@@ -0,0 +1,197 @@
363+# Copyright 2015 Canonical Ltd. This software is licensed under the
364+# GNU Affero General Public License version 3 (see the file LICENSE).
365+
366+__metaclass__ = type
367+
368+__all__ = [
369+ 'GitJob',
370+ 'GitRefScanJob',
371+ ]
372+
373+from lazr.delegates import delegates
374+from lazr.enum import (
375+ DBEnumeratedType,
376+ DBItem,
377+ )
378+from storm.exceptions import LostObjectError
379+from storm.locals import (
380+ Int,
381+ JSON,
382+ Reference,
383+ Store,
384+ )
385+from zope.interface import (
386+ classProvides,
387+ implements,
388+ )
389+
390+from lp.app.errors import NotFoundError
391+from lp.code.githosting import GitHostingClient
392+from lp.code.interfaces.gitjob import (
393+ IGitJob,
394+ IGitRefScanJob,
395+ IGitRefScanJobSource,
396+ )
397+from lp.services.config import config
398+from lp.services.database.enumcol import EnumCol
399+from lp.services.database.interfaces import (
400+ IMasterStore,
401+ IStore,
402+ )
403+from lp.services.database.locking import (
404+ AdvisoryLockHeld,
405+ LockType,
406+ try_advisory_lock,
407+ )
408+from lp.services.database.stormbase import StormBase
409+from lp.services.job.model.job import (
410+ EnumeratedSubclass,
411+ Job,
412+ )
413+from lp.services.job.runner import BaseRunnableJob
414+from lp.services.mail.sendmail import format_address_for_person
415+from lp.services.scripts import log
416+
417+
418+class GitJobType(DBEnumeratedType):
419+ """Values that `IGitJob.job_type` can take."""
420+
421+ REF_SCAN = DBItem(0, """
422+ Ref scan
423+
424+ This job scans a repository for its current list of references.
425+ """)
426+
427+
428+class GitJob(StormBase):
429+ """See `IGitJob`."""
430+
431+ __storm_table__ = 'GitJob'
432+
433+ implements(IGitJob)
434+
435+ job_id = Int(name='job', primary=True, allow_none=False)
436+ job = Reference(job_id, 'Job.id')
437+
438+ repository_id = Int(name='repository', allow_none=False)
439+ repository = Reference(repository_id, 'GitRepository.id')
440+
441+ job_type = EnumCol(enum=GitJobType, notNull=True)
442+
443+ metadata = JSON('json_data')
444+
445+ def __init__(self, repository, job_type, metadata, **job_args):
446+ """Constructor.
447+
448+ Extra keyword arguments are used to construct the underlying Job
449+ object.
450+
451+ :param repository: The database repository this job relates to.
452+ :param job_type: The `GitJobType` of this job.
453+ :param metadata: The type-specific variables, as a JSON-compatible
454+ dict.
455+ """
456+ super(GitJob, self).__init__()
457+ self.job = Job(**job_args)
458+ self.repository = repository
459+ self.job_type = job_type
460+ self.metadata = metadata
461+
462+ def makeDerived(self):
463+ return GitJobDerived.makeSubclass(self)
464+
465+
466+class GitJobDerived(BaseRunnableJob):
467+
468+ __metaclass__ = EnumeratedSubclass
469+
470+ delegates(IGitJob)
471+
472+ def __init__(self, git_job):
473+ self.context = git_job
474+
475+ @classmethod
476+ def get(cls, job_id):
477+ """Get a job by id.
478+
479+ :return: The `GitJob` with the specified id, as the current
480+ `GitJobDerived` subclass.
481+ :raises: `NotFoundError` if there is no job with the specified id,
482+ or its `job_type` does not match the desired subclass.
483+ """
484+ git_job = IStore(GitJob).get(GitJob, job_id)
485+ if git_job.job_type != cls.class_job_type:
486+ raise NotFoundError(
487+ "No object found with id %d and type %s" %
488+ (job_id, cls.class_job_type.title))
489+ return cls(git_job)
490+
491+ @classmethod
492+ def iterReady(cls):
493+ """See `IJobSource`."""
494+ jobs = IMasterStore(GitJob).find(
495+ GitJob,
496+ GitJob.job_type == cls.class_job_type,
497+ GitJob.job == Job.id,
498+ Job.id.is_in(Job.ready_jobs))
499+ return (cls(job) for job in jobs)
500+
501+ def getOopsVars(self):
502+ """See `IRunnableJob`."""
503+ oops_vars = super(GitJobDerived, self).getOopsVars()
504+ oops_vars.extend([
505+ ('git_job_id', self.context.job.id),
506+ ('git_job_type', self.context.job_type.title),
507+ ('git_repository_id', self.context.repository.id),
508+ ('git_repository_name', self.context.repository.unique_name)])
509+ return oops_vars
510+
511+ def getErrorRecipients(self):
512+ if self.requester is None:
513+ return []
514+ return [format_address_for_person(self.requester)]
515+
516+
517+class GitRefScanJob(GitJobDerived):
518+ """A Job that scans a Git repository for its current list of references."""
519+
520+ implements(IGitRefScanJob)
521+
522+ classProvides(IGitRefScanJobSource)
523+ class_job_type = GitJobType.REF_SCAN
524+
525+ max_retries = 5
526+
527+ retry_error_types = (AdvisoryLockHeld,)
528+
529+ config = config.IGitRefScanJobSource
530+
531+ @classmethod
532+ def create(cls, repository):
533+ """See `IGitRefScanJobSource`."""
534+ git_job = GitJob(
535+ repository, cls.class_job_type,
536+ {"repository_name": repository.unique_name})
537+ job = cls(git_job)
538+ job.celeryRunOnCommit()
539+ return job
540+
541+ def __init__(self, git_job):
542+ super(GitRefScanJob, self).__init__(git_job)
543+ self._cached_repository_name = self.metadata["repository_name"]
544+ self._hosting_client = GitHostingClient(
545+ config.codehosting.internal_git_api_endpoint)
546+
547+ def run(self):
548+ """See `IGitRefScanJob`."""
549+ try:
550+ with try_advisory_lock(
551+ LockType.GIT_REF_SCAN, self.repository.id,
552+ Store.of(self.repository)):
553+ hosting_path = self.repository.getInternalPath()
554+ self.repository.synchroniseRefs(
555+ self._hosting_client.get_refs(hosting_path), logger=log)
556+ except LostObjectError:
557+ log.info(
558+ "Skipping repository %s because it has been deleted." %
559+ self._cached_repository_name)
560
561=== modified file 'lib/lp/code/model/gitlookup.py'
562--- lib/lp/code/model/gitlookup.py 2015-03-05 11:39:06 +0000
563+++ lib/lp/code/model/gitlookup.py 2015-03-17 10:51:52 +0000
564@@ -297,6 +297,16 @@
565 return default
566 return repository
567
568+ def getByHostingPath(self, path):
569+ """See `IGitLookup`."""
570+ # This may need to change later to improve support for sharding.
571+ # See also `IGitRepository.getInternalPath`.
572+ try:
573+ repository_id = int(path)
574+ except ValueError:
575+ return None
576+ return self.get(repository_id)
577+
578 @staticmethod
579 def uriToPath(uri):
580 """See `IGitLookup`."""
581
582=== added file 'lib/lp/code/model/gitref.py'
583--- lib/lp/code/model/gitref.py 1970-01-01 00:00:00 +0000
584+++ lib/lp/code/model/gitref.py 2015-03-17 10:51:52 +0000
585@@ -0,0 +1,37 @@
586+# Copyright 2015 Canonical Ltd. This software is licensed under the
587+# GNU Affero General Public License version 3 (see the file LICENSE).
588+
589+__metaclass__ = type
590+__all__ = [
591+ 'GitRef',
592+ ]
593+
594+from storm.locals import (
595+ Int,
596+ Reference,
597+ Unicode,
598+ )
599+from zope.interface import implements
600+
601+from lp.code.enums import GitObjectType
602+from lp.code.interfaces.gitref import IGitRef
603+from lp.services.database.enumcol import EnumCol
604+from lp.services.database.stormbase import StormBase
605+
606+
607+class GitRef(StormBase):
608+ """See `IGitRef`."""
609+
610+ __storm_table__ = 'GitRef'
611+ __storm_primary__ = ('repository_id', 'path')
612+
613+ implements(IGitRef)
614+
615+ repository_id = Int(name='repository', allow_none=False)
616+ repository = Reference(repository_id, 'GitRepository.id')
617+
618+ path = Unicode(name='path', allow_none=False)
619+
620+ commit_sha1 = Unicode(name='commit_sha1', allow_none=False)
621+
622+ object_type = EnumCol(enum=GitObjectType, notNull=True)
623
624=== modified file 'lib/lp/code/model/gitrepository.py'
625--- lib/lp/code/model/gitrepository.py 2015-03-05 14:13:16 +0000
626+++ lib/lp/code/model/gitrepository.py 2015-03-17 10:51:52 +0000
627@@ -8,15 +8,24 @@
628 'GitRepositorySet',
629 ]
630
631+from itertools import chain
632+
633 from bzrlib import urlutils
634 import pytz
635+from storm.databases.postgres import Returning
636 from storm.expr import (
637+ And,
638 Coalesce,
639+ Insert,
640 Join,
641 Or,
642 Select,
643 SQL,
644 )
645+from storm.info import (
646+ ClassAlias,
647+ get_cls_info,
648+ )
649 from storm.locals import (
650 Bool,
651 DateTime,
652@@ -24,6 +33,7 @@
653 Reference,
654 Unicode,
655 )
656+from storm.store import Store
657 from zope.component import getUtility
658 from zope.interface import implements
659 from zope.security.proxy import removeSecurityProxy
660@@ -36,6 +46,7 @@
661 from lp.app.interfaces.informationtype import IInformationType
662 from lp.app.interfaces.launchpad import IPrivacy
663 from lp.app.interfaces.services import IService
664+from lp.code.enums import GitObjectType
665 from lp.code.errors import (
666 GitDefaultConflict,
667 GitFeatureDisabled,
668@@ -57,6 +68,7 @@
669 IGitRepositorySet,
670 user_has_special_git_repository_access,
671 )
672+from lp.code.model.gitref import GitRef
673 from lp.registry.enums import PersonVisibility
674 from lp.registry.errors import CannotChangeInformationType
675 from lp.registry.interfaces.accesspolicy import (
676@@ -78,6 +90,7 @@
677 )
678 from lp.registry.model.teammembership import TeamParticipation
679 from lp.services.config import config
680+from lp.services.database import bulk
681 from lp.services.database.constants import (
682 DEFAULT,
683 UTC_NOW,
684@@ -89,12 +102,25 @@
685 Array,
686 ArrayAgg,
687 ArrayIntersects,
688+ BulkUpdate,
689+ Values,
690 )
691 from lp.services.features import getFeatureFlag
692-from lp.services.propertycache import cachedproperty
693+from lp.services.propertycache import (
694+ cachedproperty,
695+ get_property_cache,
696+ )
697 from lp.services.webapp.authorization import available_with_permission
698
699
700+object_type_map = {
701+ "commit": GitObjectType.COMMIT,
702+ "tree": GitObjectType.TREE,
703+ "blob": GitObjectType.BLOB,
704+ "tag": GitObjectType.TAG,
705+ }
706+
707+
708 def git_repository_modified(repository, event):
709 """Update the date_last_modified property when a GitRepository is modified.
710
711@@ -243,6 +269,7 @@
712 def getInternalPath(self):
713 """See `IGitRepository`."""
714 # This may need to change later to improve support for sharding.
715+ # See also `IGitLookup.getByHostingPath`.
716 return str(self.id)
717
718 def getCodebrowseUrl(self):
719@@ -279,6 +306,132 @@
720 self, self.information_type, pillars, wanted_links)
721
722 @cachedproperty
723+ def refs(self):
724+ """See `IGitRepository`."""
725+ return list(Store.of(self).find(
726+ GitRef, GitRef.repository_id == self.id).order_by(GitRef.path))
727+
728+ def getRefByPath(self, path):
729+ return Store.of(self).find(
730+ GitRef,
731+ GitRef.repository_id == self.id,
732+ GitRef.path == path).one()
733+
734+ @staticmethod
735+ def _convertRefInfo(info):
736+ """Validate and canonicalise ref info from the hosting service.
737+
738+ :param info: A dict of {"object":
739+ {"sha1": sha1, "type": "commit"/"tree"/"blob"/"tag"}}.
740+
741+ :raises ValueError: if the dict is malformed.
742+ :return: A dict of {"sha1": sha1, "type": `GitObjectType`}.
743+ """
744+ if "object" not in info:
745+ raise ValueError('ref info does not contain "object" key')
746+ obj = info["object"]
747+ if "sha1" not in obj:
748+ raise ValueError('ref info object does not contain "sha1" key')
749+ if "type" not in obj:
750+ raise ValueError('ref info object does not contain "type" key')
751+ if not isinstance(obj["sha1"], basestring) or len(obj["sha1"]) != 40:
752+ raise ValueError('ref info sha1 is not a 40-character string')
753+ if obj["type"] not in object_type_map:
754+ raise ValueError('ref info type is not a recognised object type')
755+ sha1 = obj["sha1"]
756+ if isinstance(sha1, bytes):
757+ sha1 = sha1.decode("US-ASCII")
758+ return {"sha1": sha1, "type": object_type_map[obj["type"]]}
759+
760+ def createOrUpdateRefs(self, refs_info, get_objects=False):
761+ """See `IGitRepository`."""
762+ def dbify_values(values):
763+ return [
764+ list(chain.from_iterable(
765+ bulk.dbify_value(col, val)
766+ for col, val in zip(columns, value)))
767+ for value in values]
768+
769+ # Flush everything up to here, as we may need to invalidate the
770+ # cache after updating.
771+ store = Store.of(self)
772+ store.flush()
773+
774+ # Try a bulk update first.
775+ column_names = ["repository_id", "path", "commit_sha1", "object_type"]
776+ column_types = [
777+ ("repository", "integer"),
778+ ("path", "text"),
779+ ("commit_sha1", "character(40)"),
780+ ("object_type", "integer"),
781+ ]
782+ columns = [getattr(GitRef, name) for name in column_names]
783+ values = [
784+ (self.id, path, info["sha1"], info["type"])
785+ for path, info in refs_info.items()]
786+ db_values = dbify_values(values)
787+ new_refs_expr = Values("new_refs", column_types, db_values)
788+ new_refs = ClassAlias(GitRef, "new_refs")
789+ updated_columns = {
790+ getattr(GitRef, name): getattr(new_refs, name)
791+ for name in column_names if name not in ("repository_id", "path")}
792+ update_filter = And(
793+ GitRef.repository_id == new_refs.repository_id,
794+ GitRef.path == new_refs.path)
795+ primary_key = get_cls_info(GitRef).primary_key
796+ updated = list(store.execute(Returning(BulkUpdate(
797+ updated_columns, table=GitRef, values=new_refs_expr,
798+ where=update_filter, primary_columns=primary_key))))
799+ if updated:
800+ # Some existing GitRef objects may no longer be valid. Without
801+ # knowing which ones we already have, it's safest to just
802+ # invalidate everything.
803+ store.invalidate()
804+
805+ # If there are any remaining items, create them.
806+ create_db_values = dbify_values([
807+ value for value in values if (value[0], value[1]) not in updated])
808+ if create_db_values:
809+ created = list(store.execute(Returning(Insert(
810+ columns, values=create_db_values,
811+ primary_columns=primary_key))))
812+ else:
813+ created = []
814+
815+ del get_property_cache(self).refs
816+ if get_objects:
817+ return bulk.load(GitRef, updated + created)
818+
819+ def removeRefs(self, paths):
820+ """See `IGitRepository`."""
821+ Store.of(self).find(
822+ GitRef,
823+ GitRef.repository == self, GitRef.path.is_in(paths)).remove()
824+ del get_property_cache(self).refs
825+
826+ def synchroniseRefs(self, hosting_refs, logger=None):
827+ """See `IGitRepository`."""
828+ new_refs = {}
829+ for path, info in hosting_refs.items():
830+ try:
831+ new_refs[path] = self._convertRefInfo(info)
832+ except ValueError as e:
833+ logger.warning("Unconvertible ref %s %s: %s" % (path, info, e))
834+ current_refs = {ref.path: ref for ref in self.refs}
835+ refs_to_upsert = {}
836+ for path, info in new_refs.items():
837+ current_ref = current_refs.get(path)
838+ if (current_ref is None or
839+ info["sha1"] != current_ref.commit_sha1 or
840+ info["type"] != current_ref.object_type):
841+ refs_to_upsert[path] = info
842+ refs_to_remove = set(current_refs) - set(new_refs)
843+ if refs_to_upsert:
844+ self.createOrUpdateRefs(refs_to_upsert)
845+ if refs_to_remove:
846+ self.removeRefs(refs_to_remove)
847+
848+ @cachedproperty
849 def _known_viewers(self):
850 """A set of known persons able to view this repository.
851
852
853=== added file 'lib/lp/code/model/tests/test_gitjob.py'
854--- lib/lp/code/model/tests/test_gitjob.py 1970-01-01 00:00:00 +0000
855+++ lib/lp/code/model/tests/test_gitjob.py 2015-03-17 10:51:52 +0000
856@@ -0,0 +1,126 @@
857+# Copyright 2015 Canonical Ltd. This software is licensed under the
858+# GNU Affero General Public License version 3 (see the file LICENSE).
859+
860+"""Tests for `GitJob`s."""
861+
862+__metaclass__ = type
863+
864+import hashlib
865+
866+from testtools.matchers import (
867+ MatchesSetwise,
868+ MatchesStructure,
869+ )
870+
871+from lp.code.enums import GitObjectType
872+from lp.code.interfaces.gitjob import (
873+ IGitJob,
874+ IGitRefScanJob,
875+ )
876+from lp.code.interfaces.gitrepository import GIT_FEATURE_FLAG
877+from lp.code.model.gitjob import (
878+ GitJob,
879+ GitJobDerived,
880+ GitJobType,
881+ GitRefScanJob,
882+ )
883+from lp.services.features.testing import FeatureFixture
884+from lp.testing import TestCaseWithFactory
885+from lp.testing.dbuser import dbuser
886+from lp.testing.fakemethod import FakeMethod
887+from lp.testing.layers import (
888+ DatabaseFunctionalLayer,
889+ LaunchpadZopelessLayer,
890+ )
891+
892+
893+class TestGitJob(TestCaseWithFactory):
894+ """Tests for `GitJob`."""
895+
896+ layer = DatabaseFunctionalLayer
897+
898+ def test_provides_interface(self):
899+ # `GitJob` objects provide `IGitJob`.
900+ self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
901+ repository = self.factory.makeGitRepository()
902+ self.assertProvides(
903+ GitJob(repository, GitJobType.REF_SCAN, {}), IGitJob)
904+
905+
906+class TestGitJobDerived(TestCaseWithFactory):
907+ """Tests for `GitJobDerived`."""
908+
909+ layer = LaunchpadZopelessLayer
910+
911+ def test_getOopsMailController(self):
912+ """By default, no mail is sent about failed BranchJobs."""
913+ self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
914+ repository = self.factory.makeGitRepository()
915+ job = GitJob(repository, GitJobType.REF_SCAN, {})
916+ derived = GitJobDerived(job)
917+ self.assertIsNone(derived.getOopsMailController("x"))
918+
919+
920+class TestGitRefScanJobMixin:
921+
922+ @staticmethod
923+ def makeFakeRefs(paths):
924+ return {
925+ path: {"object": {
926+ "sha1": hashlib.sha1(path).hexdigest(),
927+ "type": "commit",
928+ }}
929+ for path in paths}
930+
931+ def assertRefsMatch(self, refs, repository, paths):
932+ matchers = [
933+ MatchesStructure.byEquality(
934+ repository=repository,
935+ path=path,
936+ commit_sha1=unicode(hashlib.sha1(path).hexdigest()),
937+ object_type=GitObjectType.COMMIT)
938+ for path in paths]
939+ self.assertThat(refs, MatchesSetwise(*matchers))
940+
941+
942+class TestGitRefScanJob(TestGitRefScanJobMixin, TestCaseWithFactory):
943+ """Tests for `GitRefScanJob`."""
944+
945+ layer = LaunchpadZopelessLayer
946+
947+ def setUp(self):
948+ super(TestGitRefScanJob, self).setUp()
949+ self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
950+
951+ def test_provides_interface(self):
952+ # `GitRefScanJob` objects provide `IGitRefScanJob`.
953+ repository = self.factory.makeGitRepository()
954+ self.assertProvides(GitRefScanJob.create(repository), IGitRefScanJob)
955+
956+ def test_run(self):
957+ # Ensure the job scans the repository.
958+ repository = self.factory.makeGitRepository()
959+ job = GitRefScanJob.create(repository)
960+ paths = (u"refs/heads/master", u"refs/tags/1.0")
961+ job._hosting_client.get_refs = FakeMethod(
962+ result=self.makeFakeRefs(paths))
963+ with dbuser("branchscanner"):
964+ job.run()
965+ self.assertRefsMatch(repository.refs, repository, paths)
966+
967+ def test_logs_bad_ref_info(self):
968+ repository = self.factory.makeGitRepository()
969+ job = GitRefScanJob.create(repository)
970+ job._hosting_client.get_refs = FakeMethod(
971+ result={u"refs/heads/master": {}})
972+ expected_message = (
973+ 'Unconvertible ref refs/heads/master {}: '
974+ 'ref info does not contain "object" key')
975+ with self.expectedLog(expected_message):
976+ with dbuser("branchscanner"):
977+ job.run()
978+ self.assertEqual([], repository.refs)
979+
980+
981+# XXX cjwatson 2015-03-12: We should test that the job works via Celery too,
982+# but that isn't feasible until we have a proper turnip fixture.
983
984=== modified file 'lib/lp/code/model/tests/test_gitlookup.py'
985--- lib/lp/code/model/tests/test_gitlookup.py 2015-03-05 14:13:16 +0000
986+++ lib/lp/code/model/tests/test_gitlookup.py 2015-03-17 10:51:52 +0000
987@@ -35,6 +35,26 @@
988 from lp.testing.layers import DatabaseFunctionalLayer
989
990
991+class TestGetByHostingPath(TestCaseWithFactory):
992+ """Test `IGitLookup.getByHostingPath`."""
993+
994+ layer = DatabaseFunctionalLayer
995+
996+ def setUp(self):
997+ super(TestGetByHostingPath, self).setUp()
998+ self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
999+ self.lookup = getUtility(IGitLookup)
1000+
1001+ def test_exists(self):
1002+ repository = self.factory.makeGitRepository()
1003+ self.assertEqual(
1004+ repository,
1005+ self.lookup.getByHostingPath(repository.getInternalPath()))
1006+
1007+ def test_missing(self):
1008+ self.assertIsNone(self.lookup.getByHostingPath("nonexistent"))
1009+
1010+
1011 class TestGetByUniqueName(TestCaseWithFactory):
1012 """Tests for `IGitLookup.getByUniqueName`."""
1013
1014
1015=== modified file 'lib/lp/code/model/tests/test_gitrepository.py'
1016--- lib/lp/code/model/tests/test_gitrepository.py 2015-03-06 16:31:30 +0000
1017+++ lib/lp/code/model/tests/test_gitrepository.py 2015-03-17 10:51:52 +0000
1018@@ -7,10 +7,15 @@
1019
1020 from datetime import datetime
1021 from functools import partial
1022+import hashlib
1023 import json
1024
1025 from lazr.lifecycle.event import ObjectModifiedEvent
1026 import pytz
1027+from testtools.matchers import (
1028+ MatchesSetwise,
1029+ MatchesStructure,
1030+ )
1031 from zope.component import getUtility
1032 from zope.event import notify
1033 from zope.security.proxy import removeSecurityProxy
1034@@ -21,6 +26,7 @@
1035 PUBLIC_INFORMATION_TYPES,
1036 )
1037 from lp.app.interfaces.launchpad import ILaunchpadCelebrities
1038+from lp.code.enums import GitObjectType
1039 from lp.code.errors import (
1040 GitFeatureDisabled,
1041 GitRepositoryCreatorNotMemberOfOwnerTeam,
1042@@ -37,6 +43,7 @@
1043 IGitRepository,
1044 IGitRepositorySet,
1045 )
1046+from lp.code.model.gitrepository import GitRepository
1047 from lp.registry.enums import (
1048 BranchSharingPolicy,
1049 PersonVisibility,
1050@@ -410,6 +417,171 @@
1051 get_policies_for_artifact(repository))
1052
1053
1054+class TestGitRepositoryRefs(TestCaseWithFactory):
1055+ """Tests for ref handling."""
1056+
1057+ layer = DatabaseFunctionalLayer
1058+
1059+ def setUp(self):
1060+ super(TestGitRepositoryRefs, self).setUp()
1061+ self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u"on"}))
1062+
1063+ def test__convertRefInfo(self):
1064+ # _convertRefInfo converts a valid info dictionary.
1065+ sha1 = unicode(hashlib.sha1("").hexdigest())
1066+ info = {"object": {"sha1": sha1, "type": u"commit"}}
1067+ expected_info = {"sha1": sha1, "type": GitObjectType.COMMIT}
1068+ self.assertEqual(expected_info, GitRepository._convertRefInfo(info))
1069+
1070+ def test__convertRefInfo_requires_object(self):
1071+ self.assertRaisesWithContent(
1072+ ValueError, 'ref info does not contain "object" key',
1073+ GitRepository._convertRefInfo, {})
1074+
1075+ def test__convertRefInfo_requires_object_sha1(self):
1076+ self.assertRaisesWithContent(
1077+ ValueError, 'ref info object does not contain "sha1" key',
1078+ GitRepository._convertRefInfo, {"object": {}})
1079+
1080+ def test__convertRefInfo_requires_object_type(self):
1081+ info = {
1082+ "object": {"sha1": u"0000000000000000000000000000000000000000"},
1083+ }
1084+ self.assertRaisesWithContent(
1085+ ValueError, 'ref info object does not contain "type" key',
1086+ GitRepository._convertRefInfo, info)
1087+
1088+ def test__convertRefInfo_bad_sha1(self):
1089+ info = {"object": {"sha1": "x", "type": "commit"}}
1090+ self.assertRaisesWithContent(
1091+ ValueError, 'ref info sha1 is not a 40-character string',
1092+ GitRepository._convertRefInfo, info)
1093+
1094+ def test__convertRefInfo_bad_type(self):
1095+ info = {
1096+ "object": {
1097+ "sha1": u"0000000000000000000000000000000000000000",
1098+ "type": u"nonsense",
1099+ },
1100+ }
1101+ self.assertRaisesWithContent(
1102+ ValueError, 'ref info type is not a recognised object type',
1103+ GitRepository._convertRefInfo, info)
1104+
1105+ def assertRefsMatch(self, refs, repository, paths):
1106+ matchers = [
1107+ MatchesStructure.byEquality(
1108+ repository=repository,
1109+ path=path,
1110+ commit_sha1=unicode(hashlib.sha1(path).hexdigest()),
1111+ object_type=GitObjectType.COMMIT)
1112+ for path in paths]
1113+ self.assertThat(refs, MatchesSetwise(*matchers))
1114+
1115+ def test_create(self):
1116+ repository = self.factory.makeGitRepository()
1117+ self.assertEqual([], repository.refs)
1118+ paths = (u"refs/heads/master", u"refs/tags/1.0")
1119+ self.factory.makeGitRefs(repository=repository, paths=paths)
1120+ self.assertRefsMatch(repository.refs, repository, paths)
1121+ master_ref = repository.getRefByPath(u"refs/heads/master")
1122+ new_refs_info = {
1123+ u"refs/tags/1.1": {
1124+ u"sha1": master_ref.commit_sha1,
1125+ u"type": master_ref.object_type,
1126+ },
1127+ }
1128+ repository.createOrUpdateRefs(new_refs_info)
1129+ self.assertRefsMatch(
1130+ [ref for ref in repository.refs if ref.path != u"refs/tags/1.1"],
1131+ repository, paths)
1132+ self.assertThat(
1133+ repository.getRefByPath(u"refs/tags/1.1"),
1134+ MatchesStructure.byEquality(
1135+ repository=repository,
1136+ path=u"refs/tags/1.1",
1137+ commit_sha1=master_ref.commit_sha1,
1138+ object_type=master_ref.object_type,
1139+ ))
1140+
1141+ def test_remove(self):
1142+ repository = self.factory.makeGitRepository()
1143+ paths = (u"refs/heads/master", u"refs/heads/branch", u"refs/tags/1.0")
1144+ self.factory.makeGitRefs(repository=repository, paths=paths)
1145+ self.assertRefsMatch(repository.refs, repository, paths)
1146+ repository.removeRefs([u"refs/heads/branch", u"refs/tags/1.0"])
1147+ self.assertRefsMatch(
1148+ repository.refs, repository, [u"refs/heads/master"])
1149+
1150+ def test_update(self):
1151+ repository = self.factory.makeGitRepository()
1152+ paths = (u"refs/heads/master", u"refs/tags/1.0")
1153+ self.factory.makeGitRefs(repository=repository, paths=paths)
1154+ self.assertRefsMatch(repository.refs, repository, paths)
1155+ new_info = {
1156+ u"sha1": u"0000000000000000000000000000000000000000",
1157+ u"type": GitObjectType.BLOB,
1158+ }
1159+ repository.createOrUpdateRefs({u"refs/tags/1.0": new_info})
1160+ self.assertRefsMatch(
1161+ [ref for ref in repository.refs if ref.path != u"refs/tags/1.0"],
1162+ repository, [u"refs/heads/master"])
1163+ self.assertThat(
1164+ repository.getRefByPath(u"refs/tags/1.0"),
1165+ MatchesStructure.byEquality(
1166+ repository=repository,
1167+ path=u"refs/tags/1.0",
1168+ commit_sha1=u"0000000000000000000000000000000000000000",
1169+ object_type=GitObjectType.BLOB,
1170+ ))
1171+
1172+ def test_synchroniseRefs(self):
1173+ # synchroniseRefs copes with synchronising a repository where some
1174+ # refs have been created, some deleted, and some changed.
1175+ repository = self.factory.makeGitRepository()
1176+ paths = (u"refs/heads/master", u"refs/heads/foo", u"refs/heads/bar")
1177+ self.factory.makeGitRefs(repository=repository, paths=paths)
1178+ self.assertRefsMatch(repository.refs, repository, paths)
1179+ repository.synchroniseRefs({
1180+ u"refs/heads/master": {
1181+ u"object": {
1182+ u"sha1": u"1111111111111111111111111111111111111111",
1183+ u"type": u"commit",
1184+ },
1185+ },
1186+ u"refs/heads/foo": {
1187+ u"object": {
1188+ u"sha1": repository.getRefByPath(
1189+ u"refs/heads/foo").commit_sha1,
1190+ u"type": u"commit",
1191+ },
1192+ },
1193+ u"refs/tags/1.0": {
1194+ u"object": {
1195+ u"sha1": repository.getRefByPath(
1196+ u"refs/heads/master").commit_sha1,
1197+ u"type": u"commit",
1198+ },
1199+ },
1200+ })
1201+ expected_sha1s = [
1202+ (u"refs/heads/master",
1203+ u"1111111111111111111111111111111111111111"),
1204+ (u"refs/heads/foo",
1205+ unicode(hashlib.sha1(u"refs/heads/foo").hexdigest())),
1206+ (u"refs/tags/1.0",
1207+ unicode(hashlib.sha1(u"refs/heads/master").hexdigest())),
1208+ ]
1209+ matchers = [
1210+ MatchesStructure.byEquality(
1211+ repository=repository,
1212+ path=path,
1213+ commit_sha1=sha1,
1214+ object_type=GitObjectType.COMMIT,
1215+ ) for path, sha1 in expected_sha1s]
1216+ self.assertThat(repository.refs, MatchesSetwise(*matchers))
1217+
1218+
1219 class TestGitRepositoryGetAllowedInformationTypes(TestCaseWithFactory):
1220 """Test `IGitRepository.getAllowedInformationTypes`."""
1221
1222
1223=== modified file 'lib/lp/code/xmlrpc/git.py'
1224--- lib/lp/code/xmlrpc/git.py 2015-03-03 17:09:33 +0000
1225+++ lib/lp/code/xmlrpc/git.py 2015-03-17 10:51:52 +0000
1226@@ -38,6 +38,7 @@
1227 split_git_unique_name,
1228 )
1229 from lp.code.interfaces.gitrepository import IGitRepositorySet
1230+from lp.code.interfaces.gitjob import IGitRefScanJobSource
1231 from lp.code.xmlrpc.codehosting import run_with_login
1232 from lp.registry.errors import (
1233 InvalidName,
1234@@ -232,3 +233,12 @@
1235 return run_with_login(
1236 requester_id, self._translatePath,
1237 path.strip("/"), permission, can_authenticate)
1238+
1239+ def notify(self, translated_path):
1240+ """See `IGitAPI`."""
1241+ repository = getUtility(IGitLookup).getByHostingPath(translated_path)
1242+ if repository is None:
1243+ return faults.NotFound(
1244+ "No repository found for '%s'." % translated_path)
1245+ job = getUtility(IGitRefScanJobSource).create(repository)
1246+ job.celeryRunOnCommit()
1247
1248=== modified file 'lib/lp/code/xmlrpc/tests/test_git.py'
1249--- lib/lp/code/xmlrpc/tests/test_git.py 2015-03-04 18:27:40 +0000
1250+++ lib/lp/code/xmlrpc/tests/test_git.py 2015-03-17 10:51:52 +0000
1251@@ -15,6 +15,7 @@
1252 LAUNCHPAD_SERVICES,
1253 )
1254 from lp.code.interfaces.gitcollection import IAllGitRepositories
1255+from lp.code.interfaces.gitjob import IGitRefScanJobSource
1256 from lp.code.interfaces.gitrepository import (
1257 GIT_FEATURE_FLAG,
1258 GIT_REPOSITORY_NAME_VALIDATION_ERROR_MESSAGE,
1259@@ -621,6 +622,22 @@
1260 "GitRepositoryCreationFault: nothing here",
1261 self.oopses[0]["tb_text"])
1262
1263+ def test_notify(self):
1264+ # The notify call creates a GitRefScanJob.
1265+ repository = self.factory.makeGitRepository()
1266+ self.assertIsNone(self.git_api.notify(repository.getInternalPath()))
1267+ job_source = getUtility(IGitRefScanJobSource)
1268+ [job] = list(job_source.iterReady())
1269+ self.assertEqual(repository, job.repository)
1270+
1271+ def test_notify_missing_repository(self):
1272+ # A notify call on a non-existent repository returns a fault and
1273+ # does not create a job.
1274+ fault = self.git_api.notify("10000")
1275+ self.assertIsInstance(fault, faults.NotFound)
1276+ job_source = getUtility(IGitRefScanJobSource)
1277+ self.assertEqual([], list(job_source.iterReady()))
1278+
1279
1280 class TestGitAPISecurity(TestGitAPIMixin, TestCaseWithFactory):
1281 """Slow tests for `IGitAPI`.
1282
1283=== modified file 'lib/lp/scripts/garbo.py'
1284--- lib/lp/scripts/garbo.py 2014-11-06 02:22:57 +0000
1285+++ lib/lp/scripts/garbo.py 2015-03-17 10:51:52 +0000
1286@@ -1,4 +1,4 @@
1287-# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
1288+# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
1289 # GNU Affero General Public License version 3 (see the file LICENSE).
1290
1291 """Database garbage collection."""
1292@@ -1092,6 +1092,23 @@
1293 """
1294
1295
1296+class GitJobPruner(BulkPruner):
1297+ """Prune `GitJob`s that are in a final state and more than a month old.
1298+
1299+ When a GitJob is completed, it gets set to a final state. These jobs
1300+ should be pruned from the database after a month.
1301+ """
1302+ target_table_class = Job
1303+ ids_to_prune_query = """
1304+ SELECT DISTINCT Job.id
1305+ FROM Job, GitJob
1306+ WHERE
1307+ Job.id = GitJob.job
1308+ AND Job.date_finished < CURRENT_TIMESTAMP AT TIME ZONE 'UTC'
1309+ - CAST('30 days' AS interval)
1310+ """
1311+
1312+
1313 class BugHeatUpdater(TunableLoop):
1314 """A `TunableLoop` for bug heat calculations."""
1315
1316@@ -1645,6 +1662,7 @@
1317 BugWatchActivityPruner,
1318 CodeImportEventPruner,
1319 CodeImportResultPruner,
1320+ GitJobPruner,
1321 HWSubmissionEmailLinker,
1322 LiveFSFilePruner,
1323 LoginTokenPruner,
1324
1325=== modified file 'lib/lp/scripts/tests/test_garbo.py'
1326--- lib/lp/scripts/tests/test_garbo.py 2015-01-07 00:35:41 +0000
1327+++ lib/lp/scripts/tests/test_garbo.py 2015-03-17 10:51:52 +0000
1328@@ -1,4 +1,4 @@
1329-# Copyright 2009-2014 Canonical Ltd. This software is licensed under the
1330+# Copyright 2009-2015 Canonical Ltd. This software is licensed under the
1331 # GNU Affero General Public License version 3 (see the file LICENSE).
1332
1333 """Test the database garbage collector."""
1334@@ -49,6 +49,7 @@
1335 )
1336 from lp.code.enums import CodeImportResultStatus
1337 from lp.code.interfaces.codeimportevent import ICodeImportEventSet
1338+from lp.code.interfaces.gitrepository import GIT_FEATURE_FLAG
1339 from lp.code.model.branchjob import (
1340 BranchJob,
1341 BranchUpgradeJob,
1342@@ -56,6 +57,10 @@
1343 from lp.code.model.codeimportevent import CodeImportEvent
1344 from lp.code.model.codeimportresult import CodeImportResult
1345 from lp.code.model.diff import Diff
1346+from lp.code.model.gitjob import (
1347+ GitJob,
1348+ GitRefScanJob,
1349+ )
1350 from lp.registry.enums import (
1351 BranchSharingPolicy,
1352 BugSharingPolicy,
1353@@ -930,6 +935,48 @@
1354 switch_dbuser('testadmin')
1355 self.assertEqual(store.find(BranchJob).count(), 1)
1356
1357+ def test_GitJobPruner(self):
1358+ # Garbo should remove jobs completed over 30 days ago.
1359+ self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u'on'}))
1360+ switch_dbuser('testadmin')
1361+ store = IMasterStore(Job)
1362+
1363+ db_repository = self.factory.makeGitRepository()
1364+ Store.of(db_repository).flush()
1365+ git_job = GitRefScanJob.create(db_repository)
1366+ git_job.job.date_finished = THIRTY_DAYS_AGO
1367+
1368+ self.assertEqual(
1369+ 1,
1370+ store.find(GitJob, GitJob.repository == db_repository.id).count())
1371+
1372+ self.runDaily()
1373+
1374+ switch_dbuser('testadmin')
1375+ self.assertEqual(
1376+ 0,
1377+ store.find(GitJob, GitJob.repository == db_repository.id).count())
1378+
1379+ def test_GitJobPruner_doesnt_prune_recent_jobs(self):
1380+ # Check to make sure the garbo doesn't remove jobs that aren't more
1381+ # than thirty days old.
1382+ self.useFixture(FeatureFixture({GIT_FEATURE_FLAG: u'on'}))
1383+ switch_dbuser('testadmin')
1384+ store = IMasterStore(Job)
1385+
1386+ db_repository = self.factory.makeGitRepository()
1387+
1388+ git_job = GitRefScanJob.create(db_repository)
1389+ git_job.job.date_finished = THIRTY_DAYS_AGO
1390+
1391+ db_repository2 = self.factory.makeGitRepository()
1392+ GitRefScanJob.create(db_repository2)
1393+
1394+ self.runDaily()
1395+
1396+ switch_dbuser('testadmin')
1397+ self.assertEqual(1, store.find(GitJob).count())
1398+
1399 def test_ObsoleteBugAttachmentPruner(self):
1400 # Bug attachments without a LibraryFileContent record are removed.
1401
1402
1403=== modified file 'lib/lp/security.py'
1404--- lib/lp/security.py 2015-03-16 00:04:39 +0000
1405+++ lib/lp/security.py 2015-03-17 10:51:52 +0000
1406@@ -84,6 +84,7 @@
1407 from lp.code.interfaces.codereviewvote import ICodeReviewVoteReference
1408 from lp.code.interfaces.diff import IPreviewDiff
1409 from lp.code.interfaces.gitcollection import IGitCollection
1410+from lp.code.interfaces.gitref import IGitRef
1411 from lp.code.interfaces.gitrepository import (
1412 IGitRepository,
1413 user_has_special_git_repository_access,
1414@@ -2269,6 +2270,15 @@
1415 usedfor = IGitRepository
1416
1417
1418+class ViewGitRef(DelegatedAuthorization):
1419+ """Anyone who can see a Git repository can see references within it."""
1420+ permission = 'launchpad.View'
1421+ usedfor = IGitRef
1422+
1423+ def __init__(self, obj):
1424+ super(ViewGitRef, self).__init__(obj, obj.repository)
1425+
1426+
1427 class AdminDistroSeriesTranslations(AuthorizationBase):
1428 permission = 'launchpad.TranslationsAdmin'
1429 usedfor = IDistroSeries
1430
1431=== modified file 'lib/lp/services/config/schema-lazr.conf'
1432--- lib/lp/services/config/schema-lazr.conf 2015-02-19 17:33:35 +0000
1433+++ lib/lp/services/config/schema-lazr.conf 2015-03-17 10:51:52 +0000
1434@@ -1750,6 +1750,10 @@
1435 module: lp.soyuz.interfaces.distributionjob
1436 dbuser: distroseriesdifferencejob
1437
1438+[IGitRefScanJobSource]
1439+module: lp.code.interfaces.gitjob
1440+dbuser: branchscanner
1441+
1442 [IInitializeDistroSeriesJobSource]
1443 module: lp.soyuz.interfaces.distributionjob
1444 dbuser: initializedistroseries
1445
1446=== modified file 'lib/lp/services/database/locking.py'
1447--- lib/lp/services/database/locking.py 2012-06-14 05:31:23 +0000
1448+++ lib/lp/services/database/locking.py 2015-03-17 10:51:52 +0000
1449@@ -1,4 +1,4 @@
1450-# Copyright 2011-2012 Canonical Ltd. This software is licensed under the
1451+# Copyright 2011-2015 Canonical Ltd. This software is licensed under the
1452 # GNU Affero General Public License version 3 (see the file LICENSE).
1453
1454 __metaclass__ = type
1455@@ -34,6 +34,11 @@
1456 Branch scan.
1457 """)
1458
1459+ GIT_REF_SCAN = DBItem(1, """Git repository reference scan.
1460+
1461+ Git repository reference scan.
1462+ """)
1463+
1464
1465 @contextmanager
1466 def try_advisory_lock(lock_type, lock_id, store):
1467
1468=== modified file 'lib/lp/services/database/stormexpr.py'
1469--- lib/lp/services/database/stormexpr.py 2013-05-02 22:22:16 +0000
1470+++ lib/lp/services/database/stormexpr.py 2015-03-17 10:51:52 +0000
1471@@ -1,4 +1,4 @@
1472-# Copyright 2011 Canonical Ltd. This software is licensed under the
1473+# Copyright 2011-2015 Canonical Ltd. This software is licensed under the
1474 # GNU Affero General Public License version 3 (see the file LICENSE).
1475
1476 __metaclass__ = type
1477@@ -47,13 +47,14 @@
1478
1479 class BulkUpdate(Expr):
1480 # Perform a bulk table update using literal values.
1481- __slots__ = ("map", "where", "table", "values")
1482+ __slots__ = ("map", "where", "table", "values", "primary_columns")
1483
1484- def __init__(self, map, table, values, where=Undef):
1485+ def __init__(self, map, table, values, where=Undef, primary_columns=Undef):
1486 self.map = map
1487 self.where = where
1488 self.table = table
1489 self.values = values
1490+ self.primary_columns = primary_columns
1491
1492
1493 @compile.when(BulkUpdate)
1494
1495=== modified file 'lib/lp/testing/factory.py'
1496--- lib/lp/testing/factory.py 2015-02-20 00:56:57 +0000
1497+++ lib/lp/testing/factory.py 2015-03-17 10:51:52 +0000
1498@@ -108,6 +108,7 @@
1499 CodeImportResultStatus,
1500 CodeImportReviewStatus,
1501 CodeReviewNotificationLevel,
1502+ GitObjectType,
1503 RevisionControlSystems,
1504 )
1505 from lp.code.errors import UnknownBranchTypeError
1506@@ -1700,6 +1701,20 @@
1507 information_type, registrant, verify_policy=False)
1508 return repository
1509
1510+ def makeGitRefs(self, repository=None, paths=None):
1511+ """Create and return a list of new, arbitrary GitRefs."""
1512+ if repository is None:
1513+ repository = self.makeGitRepository()
1514+ if paths is None:
1515+ paths = [self.getUniqueString('refs/heads/path').decode('utf-8')]
1516+ refs_info = {
1517+ path: {
1518+ u"sha1": unicode(hashlib.sha1(path).hexdigest()),
1519+ u"type": GitObjectType.COMMIT,
1520+ }
1521+ for path in paths}
1522+ return repository.createOrUpdateRefs(refs_info, get_objects=True)
1523+
1524 def makeBug(self, target=None, owner=None, bug_watch_url=None,
1525 information_type=None, date_closed=None, title=None,
1526 date_created=None, description=None, comment=None,