Merge lp:~cjwatson/launchpad/git-ref-scanner-commits into lp:launchpad

Proposed by Colin Watson on 2015-03-19
Status: Merged
Merged at revision: 17411
Proposed branch: lp:~cjwatson/launchpad/git-ref-scanner-commits
Merge into: lp:launchpad
Diff against target: 670 lines (+382/-38)
10 files modified
lib/lp/code/errors.py (+3/-3)
lib/lp/code/githosting.py (+32/-5)
lib/lp/code/interfaces/gitref.py (+20/-0)
lib/lp/code/interfaces/gitrepository.py (+28/-4)
lib/lp/code/interfaces/revision.py (+1/-0)
lib/lp/code/model/gitjob.py (+7/-2)
lib/lp/code/model/gitref.py (+14/-0)
lib/lp/code/model/gitrepository.py (+81/-5)
lib/lp/code/model/tests/test_gitjob.py (+38/-3)
lib/lp/code/model/tests/test_gitrepository.py (+158/-16)
To merge this branch: bzr merge lp:~cjwatson/launchpad/git-ref-scanner-commits
Reviewer Review Type Date Requested Status
William Grant code 2015-03-19 Approve on 2015-03-20
Review via email: mp+253501@code.launchpad.net

Commit message

Scan author/committer/commit-message information from the tip commit of Git references.

Description of the change

Scan author/committer/commit-message information from the tip commit of Git references.

I ended up refactoring GitRepository.synchroniseRefs a bit in the process; it now has a separate planning stage that takes a GitHostingClient so that it can go off and fetch commit information. The method split is mainly to make it easier to test.

To post a comment you must log in.
William Grant (wgrant) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/code/errors.py'
2--- lib/lp/code/errors.py 2015-03-12 15:21:27 +0000
3+++ lib/lp/code/errors.py 2015-03-20 13:28:37 +0000
4@@ -36,7 +36,7 @@
5 'GitRepositoryCreatorNotMemberOfOwnerTeam',
6 'GitRepositoryCreatorNotOwner',
7 'GitRepositoryExists',
8- 'GitRepositoryRefScanFault',
9+ 'GitRepositoryScanFault',
10 'GitTargetError',
11 'InvalidBranchMergeProposal',
12 'InvalidMergeQueueConfig',
13@@ -382,8 +382,8 @@
14 """Raised when there is a hosting fault creating a Git repository."""
15
16
17-class GitRepositoryRefScanFault(Exception):
18- """Raised when there is a fault getting the refs for a repository."""
19+class GitRepositoryScanFault(Exception):
20+ """Raised when there is a fault scanning a repository."""
21
22
23 class GitTargetError(Exception):
24
25=== modified file 'lib/lp/code/githosting.py'
26--- lib/lp/code/githosting.py 2015-03-12 15:21:27 +0000
27+++ lib/lp/code/githosting.py 2015-03-20 13:28:37 +0000
28@@ -15,7 +15,7 @@
29
30 from lp.code.errors import (
31 GitRepositoryCreationFault,
32- GitRepositoryRefScanFault,
33+ GitRepositoryScanFault,
34 )
35
36
37@@ -54,19 +54,46 @@
38 raise GitRepositoryCreationFault(
39 "Failed to create Git repository: %s" % response.text)
40
41- def get_refs(self, path):
42+ def getRefs(self, path):
43 try:
44 response = self._makeSession().get(
45 urlutils.join(self.endpoint, "repo", path, "refs"),
46 timeout=self.timeout)
47 except Exception as e:
48- raise GitRepositoryRefScanFault(
49+ raise GitRepositoryScanFault(
50 "Failed to get refs from Git repository: %s" % unicode(e))
51 if response.status_code != 200:
52- raise GitRepositoryRefScanFault(
53+ raise GitRepositoryScanFault(
54 "Failed to get refs from Git repository: %s" % response.text)
55 try:
56 return response.json()
57 except ValueError as e:
58- raise GitRepositoryRefScanFault(
59+ raise GitRepositoryScanFault(
60 "Failed to decode ref-scan response: %s" % unicode(e))
61+
62+ def getCommits(self, path, commit_oids, logger=None):
63+ commit_oids = list(commit_oids)
64+ try:
65+ # XXX cjwatson 2015-03-01: Once we're on requests >= 2.4.2, we
66+ # should just use post(json=) and drop the explicit Content-Type
67+ # header.
68+ if logger is not None:
69+ logger.info("Requesting commit details for %s" % commit_oids)
70+ response = self._makeSession().post(
71+ urlutils.join(self.endpoint, "repo", path, "commits"),
72+ headers={"Content-Type": "application/json"},
73+ data=json.dumps({"commits": commit_oids}),
74+ timeout=self.timeout)
75+ except Exception as e:
76+ raise GitRepositoryScanFault(
77+ "Failed to get commit details from Git repository: %s" %
78+ unicode(e))
79+ if response.status_code != 200:
80+ raise GitRepositoryScanFault(
81+ "Failed to get commit details from Git repository: %s" %
82+ response.text)
83+ try:
84+ return response.json()
85+ except ValueError as e:
86+ raise GitRepositoryScanFault(
87+ "Failed to decode commit-scan response: %s" % unicode(e))
88
89=== modified file 'lib/lp/code/interfaces/gitref.py'
90--- lib/lp/code/interfaces/gitref.py 2015-03-13 14:15:24 +0000
91+++ lib/lp/code/interfaces/gitref.py 2015-03-20 13:28:37 +0000
92@@ -15,6 +15,8 @@
93 )
94 from zope.schema import (
95 Choice,
96+ Datetime,
97+ Text,
98 TextLine,
99 )
100
101@@ -42,6 +44,24 @@
102 title=_("Object type"), required=True, readonly=True,
103 vocabulary=GitObjectType)
104
105+ author = Attribute(
106+ "The author of the commit pointed to by this reference.")
107+ author_date = Datetime(
108+ title=_("The author date of the commit pointed to by this reference."),
109+ required=False, readonly=True)
110+
111+ committer = Attribute(
112+ "The committer of the commit pointed to by this reference.")
113+ committer_date = Datetime(
114+ title=_(
115+ "The committer date of the commit pointed to by this reference."),
116+ required=False, readonly=True)
117+
118+ commit_message = Text(
119+ title=_(
120+ "The commit message of the commit pointed to by this reference."),
121+ required=False, readonly=True)
122+
123 display_name = TextLine(
124 title=_("Display name"), required=True, readonly=True,
125 description=_("Display name of the reference."))
126
127=== modified file 'lib/lp/code/interfaces/gitrepository.py'
128--- lib/lp/code/interfaces/gitrepository.py 2015-03-17 16:05:54 +0000
129+++ lib/lp/code/interfaces/gitrepository.py 2015-03-20 13:28:37 +0000
130@@ -213,12 +213,36 @@
131 :params paths: An iterable of paths.
132 """
133
134- def synchroniseRefs(hosting_refs, logger=None):
135+ def planRefChanges(hosting_client, hosting_path, logger=None):
136+ """Plan ref changes based on information from the hosting service.
137+
138+ :param hosting_client: A `GitHostingClient`.
139+ :param hosting_path: A path on the hosting service.
140+ :param logger: An optional logger.
141+
142+ :return: A dict of refs to create or update as appropriate, mapping
143+ ref paths to dictionaries of their fields; and a set of ref
144+ paths to remove.
145+ """
146+
147+ def fetchRefCommits(hosting_client, hosting_path, refs, logger=None):
148+ """Fetch commit information from the hosting service for a set of refs.
149+
150+ :param hosting_client: A `GitHostingClient`.
151+ :param hosting_path: A path on the hosting service.
152+ :param refs: A dict mapping ref paths to dictionaries of their
153+ fields; the field dictionaries will be updated with any detailed
154+ commit information that is available.
155+ :param logger: An optional logger.
156+ """
157+
158+ def synchroniseRefs(refs_to_upsert, refs_to_remove):
159 """Synchronise references with those from the hosting service.
160
161- :param hosting_refs: A dictionary of reference information returned
162- from the hosting service's `/repo/PATH/refs` collection.
163- :param logger: An optional logger.
164+ :param refs_to_upsert: A dictionary mapping ref paths to
165+ dictionaries of their fields; these refs will be created or
166+ updated as appropriate.
167+ :param refs_to_remove: A set of ref paths to remove.
168 """
169
170 def setOwnerDefault(value):
171
172=== modified file 'lib/lp/code/interfaces/revision.py'
173--- lib/lp/code/interfaces/revision.py 2013-01-07 02:40:55 +0000
174+++ lib/lp/code/interfaces/revision.py 2015-03-20 13:28:37 +0000
175@@ -81,6 +81,7 @@
176 class IRevisionAuthor(Interface):
177 """Committer of a Bazaar revision."""
178
179+ id = Int(title=_('The database revision author ID'))
180 name = TextLine(title=_("Revision Author Name"), required=True)
181 name_without_email = Attribute(
182 "Revision author name without email address.")
183
184=== modified file 'lib/lp/code/model/gitjob.py'
185--- lib/lp/code/model/gitjob.py 2015-03-17 10:42:24 +0000
186+++ lib/lp/code/model/gitjob.py 2015-03-20 13:28:37 +0000
187@@ -189,8 +189,13 @@
188 LockType.GIT_REF_SCAN, self.repository.id,
189 Store.of(self.repository)):
190 hosting_path = self.repository.getInternalPath()
191- self.repository.synchroniseRefs(
192- self._hosting_client.get_refs(hosting_path), logger=log)
193+ refs_to_upsert, refs_to_remove = (
194+ self.repository.planRefChanges(
195+ self._hosting_client, hosting_path, logger=log))
196+ self.repository.fetchRefCommits(
197+ self._hosting_client, hosting_path, refs_to_upsert,
198+ logger=log)
199+ self.repository.synchroniseRefs(refs_to_upsert, refs_to_remove)
200 except LostObjectError:
201 log.info(
202 "Skipping repository %s because it has been deleted." %
203
204=== modified file 'lib/lp/code/model/gitref.py'
205--- lib/lp/code/model/gitref.py 2015-03-13 14:15:24 +0000
206+++ lib/lp/code/model/gitref.py 2015-03-20 13:28:37 +0000
207@@ -6,7 +6,9 @@
208 'GitRef',
209 ]
210
211+import pytz
212 from storm.locals import (
213+ DateTime,
214 Int,
215 Reference,
216 Unicode,
217@@ -36,6 +38,18 @@
218
219 object_type = EnumCol(enum=GitObjectType, notNull=True)
220
221+ author_id = Int(name='author', allow_none=True)
222+ author = Reference(author_id, 'RevisionAuthor.id')
223+ author_date = DateTime(
224+ name='author_date', tzinfo=pytz.UTC, allow_none=True)
225+
226+ committer_id = Int(name='committer', allow_none=True)
227+ committer = Reference(committer_id, 'RevisionAuthor.id')
228+ committer_date = DateTime(
229+ name='committer_date', tzinfo=pytz.UTC, allow_none=True)
230+
231+ commit_message = Unicode(name='commit_message', allow_none=True)
232+
233 @property
234 def display_name(self):
235 return self.path.split("/", 2)[-1]
236
237=== modified file 'lib/lp/code/model/gitrepository.py'
238--- lib/lp/code/model/gitrepository.py 2015-03-17 16:05:54 +0000
239+++ lib/lp/code/model/gitrepository.py 2015-03-20 13:28:37 +0000
240@@ -8,6 +8,8 @@
241 'GitRepositorySet',
242 ]
243
244+from datetime import datetime
245+import email
246 from itertools import chain
247
248 from bzrlib import urlutils
249@@ -68,6 +70,7 @@
250 IGitRepositorySet,
251 user_has_special_git_repository_access,
252 )
253+from lp.code.interfaces.revision import IRevisionSet
254 from lp.code.model.gitref import GitRef
255 from lp.registry.enums import PersonVisibility
256 from lp.registry.errors import CannotChangeInformationType
257@@ -361,16 +364,28 @@
258 store.flush()
259
260 # Try a bulk update first.
261- column_names = ["repository_id", "path", "commit_sha1", "object_type"]
262+ column_names = [
263+ "repository_id", "path", "commit_sha1", "object_type",
264+ "author_id", "author_date", "committer_id", "committer_date",
265+ "commit_message",
266+ ]
267 column_types = [
268 ("repository", "integer"),
269 ("path", "text"),
270 ("commit_sha1", "character(40)"),
271 ("object_type", "integer"),
272+ ("author", "integer"),
273+ ("author_date", "timestamp without time zone"),
274+ ("committer", "integer"),
275+ ("committer_date", "timestamp without time zone"),
276+ ("commit_message", "text"),
277 ]
278 columns = [getattr(GitRef, name) for name in column_names]
279 values = [
280- (self.id, path, info["sha1"], info["type"])
281+ (self.id, path, info["sha1"], info["type"],
282+ info.get("author"), info.get("author_date"),
283+ info.get("committer"), info.get("committer_date"),
284+ info.get("commit_message"))
285 for path, info in refs_info.items()]
286 db_values = dbify_values(values)
287 new_refs_expr = Values("new_refs", column_types, db_values)
288@@ -412,14 +427,16 @@
289 GitRef.repository == self, GitRef.path.is_in(paths)).remove()
290 del get_property_cache(self).refs
291
292- def synchroniseRefs(self, hosting_refs, logger=None):
293+ def planRefChanges(self, hosting_client, hosting_path, logger=None):
294 """See `IGitRepository`."""
295 new_refs = {}
296- for path, info in hosting_refs.items():
297+ for path, info in hosting_client.getRefs(hosting_path).items():
298 try:
299 new_refs[path] = self._convertRefInfo(info)
300 except ValueError as e:
301- logger.warning("Unconvertible ref %s %s: %s" % (path, info, e))
302+ if logger is not None:
303+ logger.warning(
304+ "Unconvertible ref %s %s: %s" % (path, info, e))
305 current_refs = {ref.path: ref for ref in self.refs}
306 refs_to_upsert = {}
307 for path, info in new_refs.items():
308@@ -428,7 +445,66 @@
309 info["sha1"] != current_ref.commit_sha1 or
310 info["type"] != current_ref.object_type):
311 refs_to_upsert[path] = info
312+ elif (info["type"] == GitObjectType.COMMIT and
313+ (current_ref.author_id is None or
314+ current_ref.author_date is None or
315+ current_ref.committer_id is None or
316+ current_ref.committer_date is None or
317+ current_ref.commit_message is None)):
318+ # Only request detailed commit metadata for refs that point
319+ # to commits.
320+ refs_to_upsert[path] = info
321 refs_to_remove = set(current_refs) - set(new_refs)
322+ return refs_to_upsert, refs_to_remove
323+
324+ @staticmethod
325+ def fetchRefCommits(hosting_client, hosting_path, refs, logger=None):
326+ """See `IGitRepository`."""
327+ oids = sorted(set(info["sha1"] for info in refs.values()))
328+ commits = {
329+ commit.get("sha1"): commit
330+ for commit in hosting_client.getCommits(
331+ hosting_path, oids, logger=logger)}
332+ authors_to_acquire = []
333+ committers_to_acquire = []
334+ for info in refs.values():
335+ commit = commits.get(info["sha1"])
336+ if commit is None:
337+ continue
338+ author = commit.get("author")
339+ if author is not None:
340+ if "time" in author:
341+ info["author_date"] = datetime.fromtimestamp(
342+ author["time"], tz=pytz.UTC)
343+ if "name" in author and "email" in author:
344+ author_addr = email.utils.formataddr(
345+ (author["name"], author["email"]))
346+ info["author_addr"] = author_addr
347+ authors_to_acquire.append(author_addr)
348+ committer = commit.get("committer")
349+ if committer is not None:
350+ if "time" in committer:
351+ info["committer_date"] = datetime.fromtimestamp(
352+ committer["time"], tz=pytz.UTC)
353+ if "name" in committer and "email" in committer:
354+ committer_addr = email.utils.formataddr(
355+ (committer["name"], committer["email"]))
356+ info["committer_addr"] = committer_addr
357+ committers_to_acquire.append(committer_addr)
358+ if "message" in commit:
359+ info["commit_message"] = commit["message"]
360+ revision_authors = getUtility(IRevisionSet).acquireRevisionAuthors(
361+ authors_to_acquire + committers_to_acquire)
362+ for info in refs.values():
363+ author = revision_authors.get(info.get("author_addr"))
364+ if author is not None:
365+ info["author"] = author.id
366+ committer = revision_authors.get(info.get("committer_addr"))
367+ if committer is not None:
368+ info["committer"] = committer.id
369+
370+ def synchroniseRefs(self, refs_to_upsert, refs_to_remove):
371+ """See `IGitRepository`."""
372 if refs_to_upsert:
373 self.createOrUpdateRefs(refs_to_upsert)
374 if refs_to_remove:
375
376=== modified file 'lib/lp/code/model/tests/test_gitjob.py'
377--- lib/lp/code/model/tests/test_gitjob.py 2015-03-17 10:51:15 +0000
378+++ lib/lp/code/model/tests/test_gitjob.py 2015-03-20 13:28:37 +0000
379@@ -5,8 +5,13 @@
380
381 __metaclass__ = type
382
383+from datetime import (
384+ datetime,
385+ timedelta,
386+ )
387 import hashlib
388
389+import pytz
390 from testtools.matchers import (
391 MatchesSetwise,
392 MatchesStructure,
393@@ -25,7 +30,10 @@
394 GitRefScanJob,
395 )
396 from lp.services.features.testing import FeatureFixture
397-from lp.testing import TestCaseWithFactory
398+from lp.testing import (
399+ TestCaseWithFactory,
400+ time_counter,
401+ )
402 from lp.testing.dbuser import dbuser
403 from lp.testing.fakemethod import FakeMethod
404 from lp.testing.layers import (
405@@ -72,6 +80,27 @@
406 }}
407 for path in paths}
408
409+ @staticmethod
410+ def makeFakeCommits(author, author_date_gen, paths):
411+ epoch = datetime.fromtimestamp(0, tz=pytz.UTC)
412+ dates = {path: next(author_date_gen) for path in paths}
413+ return [{
414+ "sha1": hashlib.sha1(path).hexdigest(),
415+ "message": "tip of %s" % path,
416+ "author": {
417+ "name": author.displayname,
418+ "email": author.preferredemail.email,
419+ "time": int((dates[path] - epoch).total_seconds()),
420+ },
421+ "committer": {
422+ "name": author.displayname,
423+ "email": author.preferredemail.email,
424+ "time": int((dates[path] - epoch).total_seconds()),
425+ },
426+ "parents": [],
427+ "tree": hashlib.sha1("").hexdigest(),
428+ } for path in paths]
429+
430 def assertRefsMatch(self, refs, repository, paths):
431 matchers = [
432 MatchesStructure.byEquality(
433@@ -102,8 +131,13 @@
434 repository = self.factory.makeGitRepository()
435 job = GitRefScanJob.create(repository)
436 paths = (u"refs/heads/master", u"refs/tags/1.0")
437- job._hosting_client.get_refs = FakeMethod(
438+ job._hosting_client.getRefs = FakeMethod(
439 result=self.makeFakeRefs(paths))
440+ author = repository.owner
441+ author_date_start = datetime(2015, 01, 01, tzinfo=pytz.UTC)
442+ author_date_gen = time_counter(author_date_start, timedelta(days=1))
443+ job._hosting_client.getCommits = FakeMethod(
444+ result=self.makeFakeCommits(author, author_date_gen, paths))
445 with dbuser("branchscanner"):
446 job.run()
447 self.assertRefsMatch(repository.refs, repository, paths)
448@@ -111,8 +145,9 @@
449 def test_logs_bad_ref_info(self):
450 repository = self.factory.makeGitRepository()
451 job = GitRefScanJob.create(repository)
452- job._hosting_client.get_refs = FakeMethod(
453+ job._hosting_client.getRefs = FakeMethod(
454 result={u"refs/heads/master": {}})
455+ job._hosting_client.getCommits = FakeMethod(result=[])
456 expected_message = (
457 'Unconvertible ref refs/heads/master {}: '
458 'ref info does not contain "object" key')
459
460=== modified file 'lib/lp/code/model/tests/test_gitrepository.py'
461--- lib/lp/code/model/tests/test_gitrepository.py 2015-03-17 16:05:54 +0000
462+++ lib/lp/code/model/tests/test_gitrepository.py 2015-03-20 13:28:37 +0000
463@@ -43,6 +43,7 @@
464 IGitRepository,
465 IGitRepositorySet,
466 )
467+from lp.code.interfaces.revision import IRevisionSet
468 from lp.code.model.gitrepository import GitRepository
469 from lp.registry.enums import (
470 BranchSharingPolicy,
471@@ -72,6 +73,7 @@
472 TestCaseWithFactory,
473 verifyObject,
474 )
475+from lp.testing.fakemethod import FakeMethod
476 from lp.testing.layers import (
477 DatabaseFunctionalLayer,
478 ZopelessDatabaseLayer,
479@@ -535,6 +537,150 @@
480 object_type=GitObjectType.BLOB,
481 ))
482
483+ def test_planRefChanges(self):
484+ # planRefChanges copes with planning changes to refs in a repository
485+ # where some refs have been created, some deleted, and some changed.
486+ repository = self.factory.makeGitRepository()
487+ paths = (u"refs/heads/master", u"refs/heads/foo", u"refs/heads/bar")
488+ self.factory.makeGitRefs(repository=repository, paths=paths)
489+ self.assertRefsMatch(repository.refs, repository, paths)
490+ master_sha1 = repository.getRefByPath(u"refs/heads/master").commit_sha1
491+ foo_sha1 = repository.getRefByPath(u"refs/heads/foo").commit_sha1
492+ hosting_client = FakeMethod()
493+ hosting_client.getRefs = FakeMethod(result={
494+ u"refs/heads/master": {
495+ u"object": {
496+ u"sha1": u"1111111111111111111111111111111111111111",
497+ u"type": u"commit",
498+ },
499+ },
500+ u"refs/heads/foo": {
501+ u"object": {
502+ u"sha1": foo_sha1,
503+ u"type": u"commit",
504+ },
505+ },
506+ u"refs/tags/1.0": {
507+ u"object": {
508+ u"sha1": master_sha1,
509+ u"type": u"commit",
510+ },
511+ },
512+ })
513+ refs_to_upsert, refs_to_remove = repository.planRefChanges(
514+ hosting_client, "dummy")
515+
516+ expected_upsert = {
517+ u"refs/heads/master": {
518+ u"sha1": u"1111111111111111111111111111111111111111",
519+ u"type": GitObjectType.COMMIT,
520+ },
521+ u"refs/heads/foo": {
522+ u"sha1": unicode(hashlib.sha1(u"refs/heads/foo").hexdigest()),
523+ u"type": GitObjectType.COMMIT,
524+ },
525+ u"refs/tags/1.0": {
526+ u"sha1": unicode(
527+ hashlib.sha1(u"refs/heads/master").hexdigest()),
528+ u"type": GitObjectType.COMMIT,
529+ },
530+ }
531+ self.assertEqual(expected_upsert, refs_to_upsert)
532+ self.assertEqual(set([u"refs/heads/bar"]), refs_to_remove)
533+
534+ def test_planRefChanges_skips_non_commits(self):
535+ # planRefChanges does not attempt to update refs that point to
536+ # non-commits.
537+ repository = self.factory.makeGitRepository()
538+ blob_sha1 = unicode(hashlib.sha1(u"refs/heads/blob").hexdigest())
539+ refs_info = {
540+ u"refs/heads/blob": {
541+ u"sha1": blob_sha1,
542+ u"type": GitObjectType.BLOB,
543+ },
544+ }
545+ repository.createOrUpdateRefs(refs_info)
546+ hosting_client = FakeMethod()
547+ hosting_client.getRefs = FakeMethod(result={
548+ u"refs/heads/blob": {
549+ u"object": {
550+ u"sha1": blob_sha1,
551+ u"type": u"blob",
552+ },
553+ },
554+ })
555+ self.assertEqual(
556+ ({}, set()), repository.planRefChanges(hosting_client, "dummy"))
557+
558+ def test_fetchRefCommits(self):
559+ # fetchRefCommits fetches detailed tip commit metadata for the
560+ # requested refs.
561+ master_sha1 = unicode(hashlib.sha1(u"refs/heads/master").hexdigest())
562+ foo_sha1 = unicode(hashlib.sha1(u"refs/heads/foo").hexdigest())
563+ author = self.factory.makePerson()
564+ with person_logged_in(author):
565+ author_email = author.preferredemail.email
566+ epoch = datetime.fromtimestamp(0, tz=pytz.UTC)
567+ author_date = datetime(2015, 1, 1, tzinfo=pytz.UTC)
568+ committer_date = datetime(2015, 1, 2, tzinfo=pytz.UTC)
569+ hosting_client = FakeMethod()
570+ hosting_client.getCommits = FakeMethod(result=[
571+ {
572+ u"sha1": master_sha1,
573+ u"message": u"tip of master",
574+ u"author": {
575+ u"name": author.displayname,
576+ u"email": author_email,
577+ u"time": int((author_date - epoch).total_seconds()),
578+ },
579+ u"committer": {
580+ u"name": u"New Person",
581+ u"email": u"new-person@example.org",
582+ u"time": int((committer_date - epoch).total_seconds()),
583+ },
584+ u"parents": [],
585+ u"tree": unicode(hashlib.sha1("").hexdigest()),
586+ }])
587+ refs = {
588+ u"refs/heads/master": {
589+ u"sha1": master_sha1,
590+ u"type": GitObjectType.COMMIT,
591+ },
592+ u"refs/heads/foo": {
593+ u"sha1": foo_sha1,
594+ u"type": GitObjectType.COMMIT,
595+ },
596+ }
597+ GitRepository.fetchRefCommits(hosting_client, "dummy", refs)
598+
599+ expected_oids = [master_sha1, foo_sha1]
600+ [(_, observed_oids)] = hosting_client.getCommits.extract_args()
601+ self.assertContentEqual(expected_oids, observed_oids)
602+ expected_author_addr = u"%s <%s>" % (author.displayname, author_email)
603+ [expected_author] = getUtility(IRevisionSet).acquireRevisionAuthors(
604+ [expected_author_addr]).values()
605+ expected_committer_addr = u"New Person <new-person@example.org>"
606+ [expected_committer] = getUtility(IRevisionSet).acquireRevisionAuthors(
607+ [expected_committer_addr]).values()
608+ expected_refs = {
609+ u"refs/heads/master": {
610+ u"sha1": master_sha1,
611+ u"type": GitObjectType.COMMIT,
612+ u"author": expected_author.id,
613+ u"author_addr": expected_author_addr,
614+ u"author_date": author_date,
615+ u"committer": expected_committer.id,
616+ u"committer_addr": expected_committer_addr,
617+ u"committer_date": committer_date,
618+ u"commit_message": u"tip of master",
619+ },
620+ u"refs/heads/foo": {
621+ u"sha1": foo_sha1,
622+ u"type": GitObjectType.COMMIT,
623+ },
624+ }
625+ self.assertEqual(expected_refs, refs)
626+
627 def test_synchroniseRefs(self):
628 # synchroniseRefs copes with synchronising a repository where some
629 # refs have been created, some deleted, and some changed.
630@@ -542,28 +688,24 @@
631 paths = (u"refs/heads/master", u"refs/heads/foo", u"refs/heads/bar")
632 self.factory.makeGitRefs(repository=repository, paths=paths)
633 self.assertRefsMatch(repository.refs, repository, paths)
634- repository.synchroniseRefs({
635+ refs_to_upsert = {
636 u"refs/heads/master": {
637- u"object": {
638- u"sha1": u"1111111111111111111111111111111111111111",
639- u"type": u"commit",
640- },
641+ u"sha1": u"1111111111111111111111111111111111111111",
642+ u"type": GitObjectType.COMMIT,
643 },
644 u"refs/heads/foo": {
645- u"object": {
646- u"sha1": repository.getRefByPath(
647- u"refs/heads/foo").commit_sha1,
648- u"type": u"commit",
649- },
650+ u"sha1": repository.getRefByPath(
651+ u"refs/heads/foo").commit_sha1,
652+ u"type": GitObjectType.COMMIT,
653 },
654 u"refs/tags/1.0": {
655- u"object": {
656- u"sha1": repository.getRefByPath(
657- u"refs/heads/master").commit_sha1,
658- u"type": u"commit",
659- },
660+ u"sha1": repository.getRefByPath(
661+ u"refs/heads/master").commit_sha1,
662+ u"type": GitObjectType.COMMIT,
663 },
664- })
665+ }
666+ refs_to_remove = set([u"refs/heads/bar"])
667+ repository.synchroniseRefs(refs_to_upsert, refs_to_remove)
668 expected_sha1s = [
669 (u"refs/heads/master",
670 u"1111111111111111111111111111111111111111"),