Merge ~pappacena/launchpad:git-repo-async-privacy-virtualrefs into launchpad:master

Proposed by Thiago F. Pappacena
Status: Needs review
Proposed branch: ~pappacena/launchpad:git-repo-async-privacy-virtualrefs
Merge into: launchpad:master
Prerequisite: ~pappacena/launchpad:git-repo-async-provacy-garbo
Diff against target: 1053 lines (+547/-36)
14 files modified
database/schema/security.cfg (+1/-1)
lib/lp/code/browser/tests/test_branchmergeproposal.py (+1/-1)
lib/lp/code/interfaces/branchmergeproposal.py (+45/-1)
lib/lp/code/interfaces/githosting.py (+8/-0)
lib/lp/code/interfaces/gitrepository.py (+12/-1)
lib/lp/code/model/branchmergeproposal.py (+98/-0)
lib/lp/code/model/githosting.py (+29/-4)
lib/lp/code/model/gitjob.py (+26/-1)
lib/lp/code/model/gitrepository.py (+5/-2)
lib/lp/code/model/tests/test_branchmergeproposal.py (+234/-0)
lib/lp/code/model/tests/test_githosting.py (+2/-3)
lib/lp/code/model/tests/test_gitrepository.py (+75/-20)
lib/lp/code/subscribers/branchmergeproposal.py (+8/-2)
lib/lp/code/tests/helpers.py (+3/-0)
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+393075@code.launchpad.net

This proposal supersedes a proposal from 2020-10-29.

Commit message

Doing virtual refs synchronization when running git repository privacy change background task

Description of the change

This branch carries code from https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/390581, but it does a big chunch of refactoring of the code contained there. Mostly, the bulk operations got encapsulated in the GitHostingClient, so we would have a single point of change once Turnip will be able to run the copy/delete refs operations in bulk (instead of doing a bunch of requests).

To post a comment you must log in.

Unmerged commits

0f268fd... by Thiago F. Pappacena

Small bug fixes on permission change job

e954159... by Thiago F. Pappacena

Merge branch 'git-repo-async-privacy' into git-repo-async-privacy-virtualrefs

974b22a... by Thiago F. Pappacena

Fixing test

6fde200... by Thiago F. Pappacena

Refactoring the way we deal with copy/delete virtual refs operations

7793c43... by Thiago F. Pappacena

Merge branch 'create-mp-refs' into git-repo-async-privacy-virtualrefs

b4af659... by Thiago F. Pappacena

Merge branch 'master' into create-mp-refs

1688ced... by Thiago F. Pappacena

Running the branch sync when changing repository privacy

6dbcc2a... by Thiago F. Pappacena

Merge branch 'create-mp-refs' into git-repo-async-privacy-virtualrefs

6118345... by Thiago F. Pappacena

Improving tests for more scenarios

20ea95f... by Thiago F. Pappacena

Adding garbo to keep GitRepo status in line with info type change job

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/database/schema/security.cfg b/database/schema/security.cfg
index 290e7bc..94cb548 100644
--- a/database/schema/security.cfg
+++ b/database/schema/security.cfg
@@ -2083,7 +2083,7 @@ public.xref = SELECT, INSERT, DELETE
2083type=user2083type=user
20842084
2085[privacy-change-jobs]2085[privacy-change-jobs]
2086groups=script2086groups=script, branchscanner
2087public.accessartifact = SELECT, UPDATE, DELETE, INSERT2087public.accessartifact = SELECT, UPDATE, DELETE, INSERT
2088public.accessartifactgrant = SELECT, UPDATE, DELETE, INSERT2088public.accessartifactgrant = SELECT, UPDATE, DELETE, INSERT
2089public.accesspolicyartifact = SELECT, UPDATE, DELETE, INSERT2089public.accesspolicyartifact = SELECT, UPDATE, DELETE, INSERT
diff --git a/lib/lp/code/browser/tests/test_branchmergeproposal.py b/lib/lp/code/browser/tests/test_branchmergeproposal.py
index f2ec4ec..83c51dc 100644
--- a/lib/lp/code/browser/tests/test_branchmergeproposal.py
+++ b/lib/lp/code/browser/tests/test_branchmergeproposal.py
@@ -2336,7 +2336,7 @@ class TestLatestProposalsForEachBranchGit(
23362336
2337 @staticmethod2337 @staticmethod
2338 def _setBranchInvisible(branch):2338 def _setBranchInvisible(branch):
2339 removeSecurityProxy(branch.repository).transitionToInformationType(2339 removeSecurityProxy(branch.repository)._transitionToInformationType(
2340 InformationType.USERDATA, branch.owner, verify_policy=False)2340 InformationType.USERDATA, branch.owner, verify_policy=False)
23412341
23422342
diff --git a/lib/lp/code/interfaces/branchmergeproposal.py b/lib/lp/code/interfaces/branchmergeproposal.py
index bf481c9..efe1e4a 100644
--- a/lib/lp/code/interfaces/branchmergeproposal.py
+++ b/lib/lp/code/interfaces/branchmergeproposal.py
@@ -1,4 +1,4 @@
1# Copyright 2009-2018 Canonical Ltd. This software is licensed under the1# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4"""The interface for branch merge proposals."""4"""The interface for branch merge proposals."""
@@ -519,6 +519,50 @@ class IBranchMergeProposalView(Interface):
519 def getLatestDiffUpdateJob():519 def getLatestDiffUpdateJob():
520 """Return the latest IUpdatePreviewDiffJob for this MP."""520 """Return the latest IUpdatePreviewDiffJob for this MP."""
521521
522 def getGitRefCopyOperations():
523 """Gets the list of GitHosting copy operations that should be done
524 in order to keep this MP's virtual refs up-to-date.
525
526 Note that, for now, the only possible virtual ref that could
527 possibly be copied is GIT_CREATE_MP_VIRTUAL_REF. So, this method
528 will return at most 1 copy operation. Once new virtual refs will be
529 created, this method should be extended to add more copy operations
530 too.
531
532 :return: A list of RefCopyOperation objects.
533 """
534
535 def getGitRefDeleteOperations(force_delete=False):
536 """Gets the list of git refs that should be deleted in order to keep
537 this MP's virtual refs up-to-date.
538
539 Note that, for now, the only existing virtual ref to be deleted is
540 GIT_CREATE_MP_VIRTUAL_REF. So this method will only ever delete at
541 most 1 virtual ref. Once new virtual refs will be created, this method
542 should be extended to delete them also.
543
544 :param force_delete: True if we should get delete operation
545 regardless of any check. False otherwise.
546 :return: A list of ref names.
547 """
548
549 def syncGitVirtualRefs(force_delete=False):
550 """Requests all copies and deletion of virtual refs to make git code
551 hosting in sync with this MP.
552
553 :param force_delete: Do not try to copy any new virtual ref. Just
554 delete all virtual refs possibly created.
555 """
556
557 def bulkSyncGitVirtualRefs(merge_proposals):
558 """Synchronizes a set of merge proposals' virtual refs.
559
560 :params merge_proposals: The set of merge proposals that should have
561 the virtual refs synced.
562 :return: A tuple with the list of (copy operations, delete operations)
563 executed.
564 """
565
522566
523class IBranchMergeProposalEdit(Interface):567class IBranchMergeProposalEdit(Interface):
524568
diff --git a/lib/lp/code/interfaces/githosting.py b/lib/lp/code/interfaces/githosting.py
index 441e850..7fe0d0e 100644
--- a/lib/lp/code/interfaces/githosting.py
+++ b/lib/lp/code/interfaces/githosting.py
@@ -148,3 +148,11 @@ class IGitHostingClient(Interface):
148 :param refs: A list of tuples like (repo_path, ref_name) to be deleted.148 :param refs: A list of tuples like (repo_path, ref_name) to be deleted.
149 :param logger: An optional logger.149 :param logger: An optional logger.
150 """150 """
151
152 def bulkSyncRefs(copy_operations, delete_operations):
153 """Executes all copy operations and delete operations on Turnip
154 side.
155
156 :param copy_operations: The list of RefCopyOperation to run.
157 :param delete_operations: The list of RefDeleteOperation to run.
158 """
diff --git a/lib/lp/code/interfaces/gitrepository.py b/lib/lp/code/interfaces/gitrepository.py
index 192133a..48e7dd3 100644
--- a/lib/lp/code/interfaces/gitrepository.py
+++ b/lib/lp/code/interfaces/gitrepository.py
@@ -8,6 +8,8 @@ __metaclass__ = type
8__all__ = [8__all__ = [
9 'ContributorGitIdentity',9 'ContributorGitIdentity',
10 'GitIdentityMixin',10 'GitIdentityMixin',
11 'GIT_CREATE_MP_VIRTUAL_REF',
12 'GIT_MP_VIRTUAL_REF_FORMAT',
11 'GIT_REPOSITORY_NAME_VALIDATION_ERROR_MESSAGE',13 'GIT_REPOSITORY_NAME_VALIDATION_ERROR_MESSAGE',
12 'git_repository_name_validator',14 'git_repository_name_validator',
13 'IGitRepository',15 'IGitRepository',
@@ -105,6 +107,11 @@ valid_git_repository_name_pattern = re.compile(
105 r"^(?i)[a-z0-9][a-z0-9+\.\-@_]*\Z")107 r"^(?i)[a-z0-9][a-z0-9+\.\-@_]*\Z")
106108
107109
110# Virtual ref where we automatically put a copy of any open merge proposal.
111GIT_MP_VIRTUAL_REF_FORMAT = 'refs/merge/{mp.id}/head'
112GIT_CREATE_MP_VIRTUAL_REF = 'git.mergeproposal_virtualref.enabled'
113
114
108def valid_git_repository_name(name):115def valid_git_repository_name(name):
109 """Return True iff the name is valid as a Git repository name.116 """Return True iff the name is valid as a Git repository name.
110117
@@ -595,11 +602,15 @@ class IGitRepositoryView(IHasRecipes):
595 def updateLandingTargets(paths):602 def updateLandingTargets(paths):
596 """Update landing targets (MPs where this repository is the source).603 """Update landing targets (MPs where this repository is the source).
597604
598 For each merge proposal, create `UpdatePreviewDiffJob`s.605 For each merge proposal, create `UpdatePreviewDiffJob`s, and runs
606 the appropriate GitHosting.copyRefs operation to allow having
607 virtual merge refs with the updated branch version.
599608
600 :param paths: A list of reference paths. Any merge proposals whose609 :param paths: A list of reference paths. Any merge proposals whose
601 source is this repository and one of these paths will have their610 source is this repository and one of these paths will have their
602 diffs updated.611 diffs updated.
612 :return: Returns a tuple with the list of background jobs created,
613 and the list of ref copies requested to GitHosting.
603 """614 """
604615
605 def markRecipesStale(paths):616 def markRecipesStale(paths):
diff --git a/lib/lp/code/model/branchmergeproposal.py b/lib/lp/code/model/branchmergeproposal.py
index 061fb1e..821b15d 100644
--- a/lib/lp/code/model/branchmergeproposal.py
+++ b/lib/lp/code/model/branchmergeproposal.py
@@ -12,6 +12,7 @@ __all__ = [
1212
13from collections import defaultdict13from collections import defaultdict
14from email.utils import make_msgid14from email.utils import make_msgid
15import logging
15from operator import attrgetter16from operator import attrgetter
16import sys17import sys
1718
@@ -84,7 +85,12 @@ from lp.code.interfaces.codereviewcomment import ICodeReviewComment
84from lp.code.interfaces.codereviewinlinecomment import (85from lp.code.interfaces.codereviewinlinecomment import (
85 ICodeReviewInlineCommentSet,86 ICodeReviewInlineCommentSet,
86 )87 )
88from lp.code.interfaces.githosting import IGitHostingClient
87from lp.code.interfaces.gitref import IGitRef89from lp.code.interfaces.gitref import IGitRef
90from lp.code.interfaces.gitrepository import (
91 GIT_CREATE_MP_VIRTUAL_REF,
92 GIT_MP_VIRTUAL_REF_FORMAT,
93 )
88from lp.code.mail.branch import RecipientReason94from lp.code.mail.branch import RecipientReason
89from lp.code.model.branchrevision import BranchRevision95from lp.code.model.branchrevision import BranchRevision
90from lp.code.model.codereviewcomment import CodeReviewComment96from lp.code.model.codereviewcomment import CodeReviewComment
@@ -94,6 +100,10 @@ from lp.code.model.diff import (
94 IncrementalDiff,100 IncrementalDiff,
95 PreviewDiff,101 PreviewDiff,
96 )102 )
103from lp.code.model.githosting import (
104 RefCopyOperation,
105 RefDeleteOperation,
106 )
97from lp.registry.interfaces.person import (107from lp.registry.interfaces.person import (
98 IPerson,108 IPerson,
99 IPersonSet,109 IPersonSet,
@@ -123,6 +133,7 @@ from lp.services.database.sqlbase import (
123 quote,133 quote,
124 SQLBase,134 SQLBase,
125 )135 )
136from lp.services.features import getFeatureFlag
126from lp.services.helpers import shortlist137from lp.services.helpers import shortlist
127from lp.services.job.interfaces.job import JobStatus138from lp.services.job.interfaces.job import JobStatus
128from lp.services.job.model.job import Job139from lp.services.job.model.job import Job
@@ -144,6 +155,9 @@ from lp.soyuz.enums import (
144 )155 )
145156
146157
158logger = logging.getLogger(__name__)
159
160
147def is_valid_transition(proposal, from_state, next_state, user=None):161def is_valid_transition(proposal, from_state, next_state, user=None):
148 """Is it valid for this user to move this proposal to to next_state?162 """Is it valid for this user to move this proposal to to next_state?
149163
@@ -971,6 +985,7 @@ class BranchMergeProposal(SQLBase, BugLinkTargetMixin):
971 branch_merge_proposal=self.id):985 branch_merge_proposal=self.id):
972 job.destroySelf()986 job.destroySelf()
973 self._preview_diffs.remove()987 self._preview_diffs.remove()
988
974 self.destroySelf()989 self.destroySelf()
975990
976 def getUnlandedSourceBranchRevisions(self):991 def getUnlandedSourceBranchRevisions(self):
@@ -1217,6 +1232,89 @@ class BranchMergeProposal(SQLBase, BugLinkTargetMixin):
1217 for diff in diffs)1232 for diff in diffs)
1218 return [diff_dict.get(revisions) for revisions in revision_list]1233 return [diff_dict.get(revisions) for revisions in revision_list]
12191234
1235 @property
1236 def should_have_git_virtual_refs(self):
1237 """True if this MP should have virtual refs in the target
1238 repository. False otherwise."""
1239 # If the source repository is private and it's opening a MP to
1240 # another repository, we should not risk disclosing the private code to
1241 # everyone with permission to see the target repo:
1242 private_source = self.source_git_repository.private
1243 same_owner = self.source_git_repository.owner.inTeam(
1244 self.target_git_repository.owner)
1245 return not private_source or same_owner
1246
1247 def getGitRefCopyOperations(self):
1248 """See `IBranchMergeProposal`"""
1249 if (not self.target_git_repository
1250 or not getFeatureFlag(GIT_CREATE_MP_VIRTUAL_REF)):
1251 # Not a git MP. Skip.
1252 return []
1253 if not self.should_have_git_virtual_refs:
1254 return []
1255 return [RefCopyOperation(
1256 self.source_git_repository.getInternalPath(),
1257 self.source_git_commit_sha1,
1258 self.target_git_repository.getInternalPath(),
1259 GIT_MP_VIRTUAL_REF_FORMAT.format(mp=self))]
1260
1261 def getGitRefDeleteOperations(self, force_delete=False):
1262 """See `IBranchMergeProposal`"""
1263 if self.source_git_ref is None:
1264 # Not a git MP. Skip.
1265 return []
1266 if not force_delete and self.should_have_git_virtual_refs:
1267 return []
1268 return [RefDeleteOperation(
1269 self.target_git_repository.getInternalPath(),
1270 GIT_MP_VIRTUAL_REF_FORMAT.format(mp=self))]
1271
1272 def syncGitVirtualRefs(self, force_delete=False):
1273 """See `IBranchMergeProposal`"""
1274 if self.source_git_ref is None:
1275 return
1276 if not getFeatureFlag(GIT_CREATE_MP_VIRTUAL_REF):
1277 # XXX pappacena 2020-09-15: Do not try to remove virtual refs if
1278 # the feature is disabled. It might be disabled due to bug on
1279 # the first versions, for example, and we should have a way to
1280 # skip this feature entirely.
1281 # Once the feature is stable, we should actually *always* delete
1282 # the virtual refs, since we don't know if the feature was
1283 # enabled or not when the branch was created.
1284 return
1285 if force_delete:
1286 copy_operations = []
1287 else:
1288 copy_operations = self.getGitRefCopyOperations()
1289 delete_operations = self.getGitRefDeleteOperations(force_delete)
1290
1291 if copy_operations or delete_operations:
1292 hosting_client = getUtility(IGitHostingClient)
1293 hosting_client.bulkSyncRefs(copy_operations, delete_operations)
1294
1295 @classmethod
1296 def bulkSyncGitVirtualRefs(cls, merge_proposals):
1297 """See `IBranchMergeProposal`"""
1298 if not getFeatureFlag(GIT_CREATE_MP_VIRTUAL_REF):
1299 # XXX pappacena 2020-09-15: Do not try to remove virtual refs if
1300 # the feature is disabled. It might be disabled due to bug on
1301 # the first versions, for example, and we should have a way to
1302 # skip this feature entirely.
1303 # Once the feature is stable, we should actually *always* delete
1304 # the virtual refs, since we don't know if the feature was
1305 # enabled or not when the branch was created.
1306 return [], []
1307 copy_operations = []
1308 delete_operations = []
1309 for merge_proposal in merge_proposals:
1310 copy_operations += merge_proposal.getGitRefCopyOperations()
1311 delete_operations += merge_proposal.getGitRefDeleteOperations()
1312
1313 if copy_operations or delete_operations:
1314 hosting_client = getUtility(IGitHostingClient)
1315 hosting_client.bulkSyncRefs(copy_operations, delete_operations)
1316 return copy_operations, delete_operations
1317
1220 def scheduleDiffUpdates(self, return_jobs=True):1318 def scheduleDiffUpdates(self, return_jobs=True):
1221 """See `IBranchMergeProposal`."""1319 """See `IBranchMergeProposal`."""
1222 from lp.code.model.branchmergeproposaljob import (1320 from lp.code.model.branchmergeproposaljob import (
diff --git a/lib/lp/code/model/githosting.py b/lib/lp/code/model/githosting.py
index 74f4ec0..d0e23cf 100644
--- a/lib/lp/code/model/githosting.py
+++ b/lib/lp/code/model/githosting.py
@@ -7,9 +7,11 @@ __metaclass__ = type
7__all__ = [7__all__ = [
8 'GitHostingClient',8 'GitHostingClient',
9 'RefCopyOperation',9 'RefCopyOperation',
10 'RefDeleteOperation',
10 ]11 ]
1112
12import base6413import base64
14from collections import defaultdict
13import json15import json
14import sys16import sys
1517
@@ -33,7 +35,6 @@ from lp.code.errors import (
33 GitRepositoryDeletionFault,35 GitRepositoryDeletionFault,
34 GitRepositoryScanFault,36 GitRepositoryScanFault,
35 GitTargetError,37 GitTargetError,
36 NoSuchGitReference,
37 )38 )
38from lp.code.interfaces.githosting import IGitHostingClient39from lp.code.interfaces.githosting import IGitHostingClient
39from lp.services.config import config40from lp.services.config import config
@@ -55,12 +56,21 @@ class RefCopyOperation:
55 This class is just a helper to define copy operations parameters on56 This class is just a helper to define copy operations parameters on
56 IGitHostingClient.copyRefs method.57 IGitHostingClient.copyRefs method.
57 """58 """
58 def __init__(self, source_ref, target_repo, target_ref):59 def __init__(self, source_repo_path, source_ref, target_repo_path,
60 target_ref):
61 self.source_repo_path = source_repo_path
59 self.source_ref = source_ref62 self.source_ref = source_ref
60 self.target_repo = target_repo63 self.target_repo_path = target_repo_path
61 self.target_ref = target_ref64 self.target_ref = target_ref
6265
6366
67class RefDeleteOperation:
68 """A description of a ref delete operation."""
69 def __init__(self, repo_path, ref):
70 self.repo_path = repo_path
71 self.ref = ref
72
73
64@implementer(IGitHostingClient)74@implementer(IGitHostingClient)
65class GitHostingClient:75class GitHostingClient:
66 """A client for the internal API provided by the Git hosting system."""76 """A client for the internal API provided by the Git hosting system."""
@@ -277,7 +287,7 @@ class GitHostingClient:
277 json_data = {287 json_data = {
278 "operations": [{288 "operations": [{
279 "from": i.source_ref,289 "from": i.source_ref,
280 "to": {"repo": i.target_repo, "ref": i.target_ref}290 "to": {"repo": i.target_repo_path, "ref": i.target_ref}
281 } for i in operations]291 } for i in operations]
282 }292 }
283 try:293 try:
@@ -299,6 +309,8 @@ class GitHostingClient:
299309
300 def deleteRefs(self, refs, logger=None):310 def deleteRefs(self, refs, logger=None):
301 """See `IGitHostingClient`."""311 """See `IGitHostingClient`."""
312 # XXX pappacena: This needs to be done in a bulk operation,
313 # instead of several requests.
302 for path, ref in refs:314 for path, ref in refs:
303 try:315 try:
304 if logger is not None:316 if logger is not None:
@@ -309,3 +321,16 @@ class GitHostingClient:
309 raise GitReferenceDeletionFault(321 raise GitReferenceDeletionFault(
310 "Error deleting %s from repo %s: HTTP %s" %322 "Error deleting %s from repo %s: HTTP %s" %
311 (ref, path, e.response.status_code))323 (ref, path, e.response.status_code))
324
325 def bulkSyncRefs(self, copy_operations, delete_operations):
326 """See `IGitHostingClient`."""
327 # XXX pappacena: This needs to be done in a bulk operation on Turnip
328 # side, instead of several requests.
329 operations_per_path = defaultdict(list)
330 for operation in copy_operations:
331 operations_per_path[operation.source_repo_path].append(operation)
332 for repo_path, operations in operations_per_path.items():
333 self.copyRefs(repo_path, operations)
334
335 self.deleteRefs([(i.repo_path, i.ref) for i in delete_operations])
336
diff --git a/lib/lp/code/model/gitjob.py b/lib/lp/code/model/gitjob.py
index ba7b9f2..4514f08 100644
--- a/lib/lp/code/model/gitjob.py
+++ b/lib/lp/code/model/gitjob.py
@@ -12,6 +12,7 @@ __all__ = [
12 'ReclaimGitRepositorySpaceJob',12 'ReclaimGitRepositorySpaceJob',
13 ]13 ]
1414
15from itertools import chain
1516
16from lazr.delegates import delegate_to17from lazr.delegates import delegate_to
17from lazr.enum import (18from lazr.enum import (
@@ -33,7 +34,10 @@ from zope.interface import (
33 provider,34 provider,
34 )35 )
3536
36from lp.app.enums import InformationType37from lp.app.enums import (
38 InformationType,
39 PRIVATE_INFORMATION_TYPES,
40 )
37from lp.app.errors import NotFoundError41from lp.app.errors import NotFoundError
38from lp.code.enums import (42from lp.code.enums import (
39 GitActivityType,43 GitActivityType,
@@ -54,6 +58,7 @@ from lp.code.interfaces.gitjob import (
54 )58 )
55from lp.code.interfaces.gitrule import describe_git_permissions59from lp.code.interfaces.gitrule import describe_git_permissions
56from lp.code.mail.branch import BranchMailer60from lp.code.mail.branch import BranchMailer
61from lp.code.model.branchmergeproposal import BranchMergeProposal
57from lp.registry.interfaces.person import IPersonSet62from lp.registry.interfaces.person import IPersonSet
58from lp.services.config import config63from lp.services.config import config
59from lp.services.database.enumcol import EnumCol64from lp.services.database.enumcol import EnumCol
@@ -440,6 +445,12 @@ class GitRepositoryTransitionToInformationTypeJob(GitJobDerived):
440 def information_type(self):445 def information_type(self):
441 return InformationType.items[self.metadata["information_type"]]446 return InformationType.items[self.metadata["information_type"]]
442447
448 def updateVirtualRefs(self):
449 merge_proposals = chain(
450 self.repository.landing_targets,
451 self.repository.landing_candidates)
452 BranchMergeProposal.bulkSyncGitVirtualRefs(merge_proposals)
453
443 def run(self):454 def run(self):
444 """See `IGitRepositoryTransitionToInformationTypeJob`."""455 """See `IGitRepositoryTransitionToInformationTypeJob`."""
445 if (self.repository.status !=456 if (self.repository.status !=
@@ -447,7 +458,21 @@ class GitRepositoryTransitionToInformationTypeJob(GitJobDerived):
447 raise AttributeError(458 raise AttributeError(
448 "The repository %s is not pending information type change." %459 "The repository %s is not pending information type change." %
449 self.repository)460 self.repository)
461 is_private = self.repository.private
462 will_be_private = self.information_type in PRIVATE_INFORMATION_TYPES
450463
464 # XXX pappacena 2020-10-29: We might need to run the
465 # _transitionToInformationType as a callback (maybe on
466 # lp.code.xmlrpc.git), as a reaction to git hosting server finishing
467 # the operations to update virtual refs (call bellow).
451 self.repository._transitionToInformationType(468 self.repository._transitionToInformationType(
452 self.information_type, self.user, self.verify_policy)469 self.information_type, self.user, self.verify_policy)
470
471 # When changing privacy type, we need to make sure to not leak
472 # anything though virtual refs, and create new ones that didn't
473 # exist before.
474 if is_private != will_be_private:
475 # XXX pappacena 2020-10-28: We need to implement retry rules here.
476 self.updateVirtualRefs()
477
453 self.repository.status = GitRepositoryStatus.AVAILABLE478 self.repository.status = GitRepositoryStatus.AVAILABLE
diff --git a/lib/lp/code/model/gitrepository.py b/lib/lp/code/model/gitrepository.py
index fb506f1..731ca3f 100644
--- a/lib/lp/code/model/gitrepository.py
+++ b/lib/lp/code/model/gitrepository.py
@@ -1205,9 +1205,12 @@ class GitRepository(StormBase, WebhookTargetMixin, GitIdentityMixin):
1205 def updateLandingTargets(self, paths):1205 def updateLandingTargets(self, paths):
1206 """See `IGitRepository`."""1206 """See `IGitRepository`."""
1207 jobs = []1207 jobs = []
1208 for merge_proposal in self.getActiveLandingTargets(paths):1208 merge_proposals = self.getActiveLandingTargets(paths)
1209 for merge_proposal in merge_proposals:
1209 jobs.extend(merge_proposal.scheduleDiffUpdates())1210 jobs.extend(merge_proposal.scheduleDiffUpdates())
1210 return jobs1211 copy_ops, delete_ops = BranchMergeProposal.bulkSyncGitVirtualRefs(
1212 merge_proposals)
1213 return jobs, copy_ops, delete_ops
12111214
1212 def _getRecipes(self, paths=None):1215 def _getRecipes(self, paths=None):
1213 """Undecorated version of recipes for use by `markRecipesStale`."""1216 """Undecorated version of recipes for use by `markRecipesStale`."""
diff --git a/lib/lp/code/model/tests/test_branchmergeproposal.py b/lib/lp/code/model/tests/test_branchmergeproposal.py
index 81b94b3..db69320 100644
--- a/lib/lp/code/model/tests/test_branchmergeproposal.py
+++ b/lib/lp/code/model/tests/test_branchmergeproposal.py
@@ -67,6 +67,10 @@ from lp.code.interfaces.branchmergeproposal import (
67 IBranchMergeProposalGetter,67 IBranchMergeProposalGetter,
68 IBranchMergeProposalJobSource,68 IBranchMergeProposalJobSource,
69 )69 )
70from lp.code.interfaces.gitrepository import (
71 GIT_CREATE_MP_VIRTUAL_REF,
72 GIT_MP_VIRTUAL_REF_FORMAT,
73 )
70from lp.code.model.branchmergeproposal import (74from lp.code.model.branchmergeproposal import (
71 BranchMergeProposal,75 BranchMergeProposal,
72 BranchMergeProposalGetter,76 BranchMergeProposalGetter,
@@ -97,6 +101,7 @@ from lp.testing import (
97 ExpectedException,101 ExpectedException,
98 launchpadlib_for,102 launchpadlib_for,
99 login,103 login,
104 login_admin,
100 login_person,105 login_person,
101 person_logged_in,106 person_logged_in,
102 TestCaseWithFactory,107 TestCaseWithFactory,
@@ -256,6 +261,196 @@ class TestBranchMergeProposalPrivacy(TestCaseWithFactory):
256 self.assertContentEqual([owner, team], subscriptions)261 self.assertContentEqual([owner, team], subscriptions)
257262
258263
264class TestGitBranchMergeProposalVirtualRefs(TestCaseWithFactory):
265 """Ensure that BranchMergeProposal creation run the appropriate copy
266 and delete of virtual refs, like ref/merge/<id>/head."""
267
268 layer = DatabaseFunctionalLayer
269
270 def setUp(self):
271 super(TestGitBranchMergeProposalVirtualRefs, self).setUp()
272 self.hosting_fixture = self.useFixture(GitHostingFixture())
273
274 def test_should_have_vrefs_public_repos(self):
275 mp = self.factory.makeBranchMergeProposalForGit()
276 self.assertTrue(mp.should_have_git_virtual_refs)
277
278 def test_should_have_vrefs_private_source(self):
279 login_admin()
280 source_repo = self.factory.makeGitRepository(
281 information_type=InformationType.PRIVATESECURITY)
282 target_repo = self.factory.makeGitRepository(target=source_repo.target)
283 [source] = self.factory.makeGitRefs(source_repo)
284 [target] = self.factory.makeGitRefs(target_repo)
285 mp = self.factory.makeBranchMergeProposalForGit(
286 source_ref=source, target_ref=target)
287 self.assertFalse(mp.should_have_git_virtual_refs)
288
289 def test_should_have_vrefs_private_target(self):
290 login_admin()
291 source_repo = self.factory.makeGitRepository()
292 target_repo = self.factory.makeGitRepository(
293 target=source_repo.target,
294 information_type=InformationType.PRIVATESECURITY)
295 [source] = self.factory.makeGitRefs(source_repo)
296 [target] = self.factory.makeGitRefs(target_repo)
297 mp = self.factory.makeBranchMergeProposalForGit(
298 source_ref=source, target_ref=target)
299 self.assertTrue(mp.should_have_git_virtual_refs)
300
301 def test_copy_git_merge_virtual_ref(self):
302 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
303 mp = self.factory.makeBranchMergeProposalForGit()
304
305 copy_operations = mp.getGitRefCopyOperations()
306 self.assertEqual(1, len(copy_operations))
307 self.assertThat(copy_operations[0], MatchesStructure(
308 source_repo_path=Equals(
309 mp.source_git_repository.getInternalPath()),
310 source_ref=Equals(mp.source_git_commit_sha1),
311 target_repo_path=Equals(
312 mp.target_git_repository.getInternalPath()),
313 target_ref=Equals("refs/merge/%s/head" % mp.id),
314 ))
315 delete_operations = mp.getGitRefDeleteOperations()
316 self.assertEqual([], delete_operations)
317
318 self.assertEqual(1, self.hosting_fixture.bulkSyncRefs.call_count)
319 args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
320 copy_ops, delete_ops = args
321 self.assertEqual({}, kwargs)
322 self.assertEqual([], delete_ops)
323 self.assertEqual(1, len(copy_ops))
324 self.assertThat(copy_ops[0], MatchesStructure(
325 source_repo_path=Equals(
326 mp.source_git_repository.getInternalPath()),
327 source_ref=Equals(mp.source_git_commit_sha1),
328 target_repo_path=Equals(
329 mp.target_git_repository.getInternalPath()),
330 target_ref=Equals("refs/merge/%s/head" % mp.id),
331 ))
332
333 def test_getGitRefCopyOperations_private_source(self):
334 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
335 login_admin()
336 source_repo = self.factory.makeGitRepository(
337 information_type=InformationType.PRIVATESECURITY)
338 target_repo = self.factory.makeGitRepository(target=source_repo.target)
339 [source] = self.factory.makeGitRefs(source_repo)
340 [target] = self.factory.makeGitRefs(target_repo)
341 mp = self.factory.makeBranchMergeProposalForGit(
342 source_ref=source, target_ref=target)
343 self.assertEqual([], mp.getGitRefCopyOperations())
344
345 delete_operations = mp.getGitRefDeleteOperations()
346 self.assertEqual(1, len(delete_operations))
347 self.assertThat(delete_operations[0], MatchesStructure(
348 repo_path=Equals(mp.target_git_repository.getInternalPath()),
349 ref=Equals("refs/merge/%s/head" % mp.id),
350 ))
351
352 def test_getGitRefCopyOperations_private_source_same_repo(self):
353 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
354 login_admin()
355 repo = self.factory.makeGitRepository(
356 information_type=InformationType.PRIVATESECURITY)
357 [source, target] = self.factory.makeGitRefs(
358 repo, ['refs/heads/bugfix', 'refs/heads/master'])
359 mp = self.factory.makeBranchMergeProposalForGit(
360 source_ref=source, target_ref=target)
361 operations = mp.getGitRefCopyOperations()
362 self.assertEqual(1, len(operations))
363 self.assertThat(operations[0], MatchesStructure(
364 source_repo_path=Equals(
365 mp.source_git_repository.getInternalPath()),
366 source_ref=Equals(mp.source_git_commit_sha1),
367 target_repo_path=Equals(
368 mp.target_git_repository.getInternalPath()),
369 target_ref=Equals(GIT_MP_VIRTUAL_REF_FORMAT.format(mp=mp))
370 ))
371
372 delete_operations = mp.getGitRefDeleteOperations()
373 self.assertEqual([], delete_operations)
374
375 def test_getGitRefCopyOperations_private_source_same_owner(self):
376 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
377 login_admin()
378 source_repo = self.factory.makeGitRepository(
379 information_type=InformationType.PRIVATESECURITY)
380 target_repo = self.factory.makeGitRepository(
381 target=source_repo.target, owner=source_repo.owner)
382 [source] = self.factory.makeGitRefs(source_repo)
383 [target] = self.factory.makeGitRefs(target_repo)
384 mp = self.factory.makeBranchMergeProposalForGit(
385 source_ref=source, target_ref=target)
386 operations = mp.getGitRefCopyOperations()
387 self.assertEqual(1, len(operations))
388 self.assertThat(operations[0], MatchesStructure(
389 source_repo_path=Equals(
390 mp.source_git_repository.getInternalPath()),
391 source_ref=Equals(mp.source_git_commit_sha1),
392 target_repo_path=Equals(
393 mp.target_git_repository.getInternalPath()),
394 target_ref=Equals(GIT_MP_VIRTUAL_REF_FORMAT.format(mp=mp))
395 ))
396
397 delete_operations = mp.getGitRefDeleteOperations()
398 self.assertEqual([], delete_operations)
399
400 def test_syncGitVirtualRefs(self):
401 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
402 login_admin()
403 login_admin()
404 source_repo = self.factory.makeGitRepository()
405 target_repo = self.factory.makeGitRepository(target=source_repo.target)
406 [source] = self.factory.makeGitRefs(source_repo)
407 [target] = self.factory.makeGitRefs(target_repo)
408 mp = self.factory.makeBranchMergeProposalForGit(
409 source_ref=source, target_ref=target)
410
411 # mp.syncGitVirtualRefs should have been triggered by event.
412 # Check lp.code.subscribers.branchmergeproposal.merge_proposal_created.
413 self.assertEqual(1, self.hosting_fixture.bulkSyncRefs.call_count)
414 args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
415 copy_ops, delete_ops = args
416 self.assertEquals({}, kwargs)
417 self.assertEqual([], delete_ops)
418 self.assertEqual(1, len(copy_ops))
419 self.assertThat(copy_ops[0], MatchesStructure(
420 source_repo_path=Equals(
421 mp.source_git_repository.getInternalPath()),
422 source_ref=Equals(mp.source_git_commit_sha1),
423 target_repo_path=Equals(
424 mp.target_git_repository.getInternalPath()),
425 target_ref=Equals("refs/merge/%s/head" % mp.id),
426 ))
427
428 self.assertEqual(0, self.hosting_fixture.deleteRefs.call_count)
429
430 def test_syncGitVirtualRefs_private_source_deletes_ref(self):
431 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
432 login_admin()
433 source_repo = self.factory.makeGitRepository(
434 information_type=InformationType.PRIVATESECURITY)
435 target_repo = self.factory.makeGitRepository(target=source_repo.target)
436 [source] = self.factory.makeGitRefs(source_repo)
437 [target] = self.factory.makeGitRefs(target_repo)
438 mp = self.factory.makeBranchMergeProposalForGit(
439 source_ref=source, target_ref=target)
440
441 # mp.syncGitVirtualRefs should have been triggered by event.
442 # Check lp.code.subscribers.branchmergeproposal.merge_proposal_deleted.
443 self.assertEqual(1, self.hosting_fixture.bulkSyncRefs.call_count)
444 args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
445 copy_ops, delete_ops = args
446 self.assertEqual({}, kwargs)
447 self.assertEqual([], copy_ops)
448 self.assertEqual(1, len(delete_ops))
449 self.assertThat(delete_ops[0], MatchesStructure(
450 repo_path=Equals(target_repo.getInternalPath()),
451 ref=Equals("refs/merge/%s/head" % mp.id)))
452
453
259class TestBranchMergeProposalTransitions(TestCaseWithFactory):454class TestBranchMergeProposalTransitions(TestCaseWithFactory):
260 """Test the state transitions of branch merge proposals."""455 """Test the state transitions of branch merge proposals."""
261456
@@ -1185,6 +1380,10 @@ class TestMergeProposalWebhooks(WithVCSScenarios, TestCaseWithFactory):
1185 def test_delete_triggers_webhooks(self):1380 def test_delete_triggers_webhooks(self):
1186 # When an existing merge proposal is deleted, any relevant webhooks1381 # When an existing merge proposal is deleted, any relevant webhooks
1187 # are triggered.1382 # are triggered.
1383 self.useFixture(FeatureFixture({
1384 GIT_CREATE_MP_VIRTUAL_REF: "on",
1385 BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG: "on"}))
1386 hosting_fixture = self.useFixture(GitHostingFixture())
1188 logger = self.useFixture(FakeLogger())1387 logger = self.useFixture(FakeLogger())
1189 source = self.makeBranch()1388 source = self.makeBranch()
1190 target = self.makeBranch(same_target_as=source)1389 target = self.makeBranch(same_target_as=source)
@@ -1203,10 +1402,25 @@ class TestMergeProposalWebhooks(WithVCSScenarios, TestCaseWithFactory):
1203 expected_redacted_payload = dict(1402 expected_redacted_payload = dict(
1204 expected_payload,1403 expected_payload,
1205 old=MatchesDict(self.getExpectedPayload(proposal, redact=True)))1404 old=MatchesDict(self.getExpectedPayload(proposal, redact=True)))
1405 hosting_fixture = self.useFixture(GitHostingFixture())
1206 proposal.deleteProposal()1406 proposal.deleteProposal()
1207 delivery = hook.deliveries.one()1407 delivery = hook.deliveries.one()
1408 self.assertIsNotNone(delivery)
1208 self.assertCorrectDelivery(expected_payload, hook, delivery)1409 self.assertCorrectDelivery(expected_payload, hook, delivery)
1209 self.assertCorrectLogging(expected_redacted_payload, hook, logger)1410 self.assertCorrectLogging(expected_redacted_payload, hook, logger)
1411 if not self.git:
1412 self.assertEqual(0, hosting_fixture.bulkSyncRefs.call_count)
1413 else:
1414 self.assertEqual(1, hosting_fixture.bulkSyncRefs.call_count)
1415 args, kwargs = hosting_fixture.bulkSyncRefs.calls[0]
1416 self.assertEqual({}, kwargs)
1417 copy_ops, delete_ops = args
1418 self.assertEqual(0, len(copy_ops))
1419 self.assertEqual(1, len(delete_ops))
1420 self.assertThat(delete_ops[0], MatchesStructure(
1421 repo_path=Equals(target.repository.getInternalPath()),
1422 ref=Equals("refs/merge/%s/head" % proposal.id)
1423 ))
12101424
12111425
1212class TestGetAddress(TestCaseWithFactory):1426class TestGetAddress(TestCaseWithFactory):
@@ -1507,6 +1721,26 @@ class TestBranchMergeProposalDeletion(TestCaseWithFactory):
1507 self.assertRaises(1721 self.assertRaises(
1508 SQLObjectNotFound, BranchMergeProposalJob.get, job_id)1722 SQLObjectNotFound, BranchMergeProposalJob.get, job_id)
15091723
1724 def test_deleteProposal_for_git_removes_virtual_ref(self):
1725 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
1726 self.useFixture(GitHostingFixture())
1727 proposal = self.factory.makeBranchMergeProposalForGit()
1728
1729 # Clean up fixture calls.
1730 hosting_fixture = self.useFixture(GitHostingFixture())
1731 proposal.deleteProposal()
1732
1733 self.assertEqual(1, hosting_fixture.bulkSyncRefs.call_count)
1734 args, kwargs = hosting_fixture.bulkSyncRefs.calls[0]
1735 self.assertEqual({}, kwargs)
1736 copy_ops, delete_ops = args
1737 self.assertEqual([], copy_ops)
1738 self.assertEqual(1, len(delete_ops))
1739 self.assertThat(delete_ops[0], MatchesStructure(
1740 repo_path=Equals(proposal.target_git_repository.getInternalPath()),
1741 ref=Equals('refs/merge/%s/head' % proposal.id)
1742 ))
1743
15101744
1511class TestBranchMergeProposalBugs(WithVCSScenarios, TestCaseWithFactory):1745class TestBranchMergeProposalBugs(WithVCSScenarios, TestCaseWithFactory):
15121746
diff --git a/lib/lp/code/model/tests/test_githosting.py b/lib/lp/code/model/tests/test_githosting.py
index 23d6063..fad0df3 100644
--- a/lib/lp/code/model/tests/test_githosting.py
+++ b/lib/lp/code/model/tests/test_githosting.py
@@ -42,7 +42,6 @@ from lp.code.errors import (
42 GitRepositoryDeletionFault,42 GitRepositoryDeletionFault,
43 GitRepositoryScanFault,43 GitRepositoryScanFault,
44 GitTargetError,44 GitTargetError,
45 NoSuchGitReference,
46 )45 )
47from lp.code.interfaces.githosting import IGitHostingClient46from lp.code.interfaces.githosting import IGitHostingClient
48from lp.code.model.githosting import RefCopyOperation47from lp.code.model.githosting import RefCopyOperation
@@ -414,8 +413,8 @@ class TestGitHostingClient(TestCase):
414413
415 def getCopyRefOperations(self):414 def getCopyRefOperations(self):
416 return [415 return [
417 RefCopyOperation("1a2b3c4", "999", "refs/merge/123"),416 RefCopyOperation("1", "1a2b3c4", "999", "refs/merge/123"),
418 RefCopyOperation("9a8b7c6", "666", "refs/merge/989"),417 RefCopyOperation("1", "9a8b7c6", "666", "refs/merge/989"),
419 ]418 ]
420419
421 def test_copyRefs(self):420 def test_copyRefs(self):
diff --git a/lib/lp/code/model/tests/test_gitrepository.py b/lib/lp/code/model/tests/test_gitrepository.py
index 08887f5..2296352 100644
--- a/lib/lp/code/model/tests/test_gitrepository.py
+++ b/lib/lp/code/model/tests/test_gitrepository.py
@@ -97,6 +97,7 @@ from lp.code.interfaces.gitnamespace import (
97 IGitNamespaceSet,97 IGitNamespaceSet,
98 )98 )
99from lp.code.interfaces.gitrepository import (99from lp.code.interfaces.gitrepository import (
100 GIT_CREATE_MP_VIRTUAL_REF,
100 IGitRepository,101 IGitRepository,
101 IGitRepositorySet,102 IGitRepositorySet,
102 IGitRepositoryView,103 IGitRepositoryView,
@@ -783,6 +784,7 @@ class TestGitRepositoryDeletion(TestCaseWithFactory):
783 # Make sure that the tests all flush the database changes.784 # Make sure that the tests all flush the database changes.
784 self.addCleanup(Store.of(self.repository).flush)785 self.addCleanup(Store.of(self.repository).flush)
785 login_person(self.user)786 login_person(self.user)
787 self.hosting_fixture = self.useFixture(GitHostingFixture())
786788
787 def test_deletable(self):789 def test_deletable(self):
788 # A newly created repository can be deleted without any problems.790 # A newly created repository can be deleted without any problems.
@@ -824,7 +826,6 @@ class TestGitRepositoryDeletion(TestCaseWithFactory):
824826
825 def test_code_import_does_not_disable_deletion(self):827 def test_code_import_does_not_disable_deletion(self):
826 # A repository that has an attached code import can be deleted.828 # A repository that has an attached code import can be deleted.
827 self.useFixture(GitHostingFixture())
828 code_import = self.factory.makeCodeImport(829 code_import = self.factory.makeCodeImport(
829 target_rcs_type=TargetRevisionControlSystems.GIT)830 target_rcs_type=TargetRevisionControlSystems.GIT)
830 repository = code_import.git_repository831 repository = code_import.git_repository
@@ -984,6 +985,7 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
984 # unsubscribe the repository owner here.985 # unsubscribe the repository owner here.
985 self.repository.unsubscribe(986 self.repository.unsubscribe(
986 self.repository.owner, self.repository.owner)987 self.repository.owner, self.repository.owner)
988 self.hosting_fixture = self.useFixture(GitHostingFixture())
987989
988 def test_plain_repository(self):990 def test_plain_repository(self):
989 # A fresh repository has no deletion requirements.991 # A fresh repository has no deletion requirements.
@@ -1115,7 +1117,6 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
11151117
1116 def test_code_import_requirements(self):1118 def test_code_import_requirements(self):
1117 # Code imports are not included explicitly in deletion requirements.1119 # Code imports are not included explicitly in deletion requirements.
1118 self.useFixture(GitHostingFixture())
1119 code_import = self.factory.makeCodeImport(1120 code_import = self.factory.makeCodeImport(
1120 target_rcs_type=TargetRevisionControlSystems.GIT)1121 target_rcs_type=TargetRevisionControlSystems.GIT)
1121 # Remove the implicit repository subscription first.1122 # Remove the implicit repository subscription first.
@@ -1126,7 +1127,6 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
11261127
1127 def test_code_import_deletion(self):1128 def test_code_import_deletion(self):
1128 # break_references allows deleting a code import repository.1129 # break_references allows deleting a code import repository.
1129 self.useFixture(GitHostingFixture())
1130 code_import = self.factory.makeCodeImport(1130 code_import = self.factory.makeCodeImport(
1131 target_rcs_type=TargetRevisionControlSystems.GIT)1131 target_rcs_type=TargetRevisionControlSystems.GIT)
1132 code_import_id = code_import.id1132 code_import_id = code_import.id
@@ -1182,7 +1182,6 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
11821182
1183 def test_DeleteCodeImport(self):1183 def test_DeleteCodeImport(self):
1184 # DeleteCodeImport.__call__ must delete the CodeImport.1184 # DeleteCodeImport.__call__ must delete the CodeImport.
1185 self.useFixture(GitHostingFixture())
1186 code_import = self.factory.makeCodeImport(1185 code_import = self.factory.makeCodeImport(
1187 target_rcs_type=TargetRevisionControlSystems.GIT)1186 target_rcs_type=TargetRevisionControlSystems.GIT)
1188 code_import_id = code_import.id1187 code_import_id = code_import.id
@@ -1521,6 +1520,10 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
15211520
1522 layer = DatabaseFunctionalLayer1521 layer = DatabaseFunctionalLayer
15231522
1523 def setUp(self):
1524 super(TestGitRepositoryRefs, self).setUp()
1525 self.hosting_fixture = self.useFixture(GitHostingFixture())
1526
1524 def test__convertRefInfo(self):1527 def test__convertRefInfo(self):
1525 # _convertRefInfo converts a valid info dictionary.1528 # _convertRefInfo converts a valid info dictionary.
1526 sha1 = six.ensure_text(hashlib.sha1("").hexdigest())1529 sha1 = six.ensure_text(hashlib.sha1("").hexdigest())
@@ -1791,18 +1794,17 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1791 # planRefChanges excludes some ref prefixes by default, and can be1794 # planRefChanges excludes some ref prefixes by default, and can be
1792 # configured otherwise.1795 # configured otherwise.
1793 repository = self.factory.makeGitRepository()1796 repository = self.factory.makeGitRepository()
1794 hosting_fixture = self.useFixture(GitHostingFixture())
1795 repository.planRefChanges("dummy")1797 repository.planRefChanges("dummy")
1796 self.assertEqual(1798 self.assertEqual(
1797 [{"exclude_prefixes": ["refs/changes/"]}],1799 [{"exclude_prefixes": ["refs/changes/"]}],
1798 hosting_fixture.getRefs.extract_kwargs())1800 self.hosting_fixture.getRefs.extract_kwargs())
1799 hosting_fixture.getRefs.calls = []1801 self.hosting_fixture.getRefs.calls = []
1800 self.pushConfig(1802 self.pushConfig(
1801 "codehosting", git_exclude_ref_prefixes="refs/changes/ refs/pull/")1803 "codehosting", git_exclude_ref_prefixes="refs/changes/ refs/pull/")
1802 repository.planRefChanges("dummy")1804 repository.planRefChanges("dummy")
1803 self.assertEqual(1805 self.assertEqual(
1804 [{"exclude_prefixes": ["refs/changes/", "refs/pull/"]}],1806 [{"exclude_prefixes": ["refs/changes/", "refs/pull/"]}],
1805 hosting_fixture.getRefs.extract_kwargs())1807 self.hosting_fixture.getRefs.extract_kwargs())
18061808
1807 def test_fetchRefCommits(self):1809 def test_fetchRefCommits(self):
1808 # fetchRefCommits fetches detailed tip commit metadata for the1810 # fetchRefCommits fetches detailed tip commit metadata for the
@@ -1875,9 +1877,8 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1875 def test_fetchRefCommits_empty(self):1877 def test_fetchRefCommits_empty(self):
1876 # If given an empty refs dictionary, fetchRefCommits returns early1878 # If given an empty refs dictionary, fetchRefCommits returns early
1877 # without contacting the hosting service.1879 # without contacting the hosting service.
1878 hosting_fixture = self.useFixture(GitHostingFixture())
1879 GitRepository.fetchRefCommits("dummy", {})1880 GitRepository.fetchRefCommits("dummy", {})
1880 self.assertEqual([], hosting_fixture.getCommits.calls)1881 self.assertEqual([], self.hosting_fixture.getCommits.calls)
18811882
1882 def test_synchroniseRefs(self):1883 def test_synchroniseRefs(self):
1883 # synchroniseRefs copes with synchronising a repository where some1884 # synchroniseRefs copes with synchronising a repository where some
@@ -1920,7 +1921,6 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1920 self.assertThat(repository.refs, MatchesSetwise(*matchers))1921 self.assertThat(repository.refs, MatchesSetwise(*matchers))
19211922
1922 def test_set_default_branch(self):1923 def test_set_default_branch(self):
1923 hosting_fixture = self.useFixture(GitHostingFixture())
1924 repository = self.factory.makeGitRepository()1924 repository = self.factory.makeGitRepository()
1925 self.factory.makeGitRefs(1925 self.factory.makeGitRefs(
1926 repository=repository,1926 repository=repository,
@@ -1931,22 +1931,20 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1931 self.assertEqual(1931 self.assertEqual(
1932 [((repository.getInternalPath(),),1932 [((repository.getInternalPath(),),
1933 {"default_branch": "refs/heads/new"})],1933 {"default_branch": "refs/heads/new"})],
1934 hosting_fixture.setProperties.calls)1934 self.hosting_fixture.setProperties.calls)
1935 self.assertEqual("refs/heads/new", repository.default_branch)1935 self.assertEqual("refs/heads/new", repository.default_branch)
19361936
1937 def test_set_default_branch_unchanged(self):1937 def test_set_default_branch_unchanged(self):
1938 hosting_fixture = self.useFixture(GitHostingFixture())
1939 repository = self.factory.makeGitRepository()1938 repository = self.factory.makeGitRepository()
1940 self.factory.makeGitRefs(1939 self.factory.makeGitRefs(
1941 repository=repository, paths=["refs/heads/master"])1940 repository=repository, paths=["refs/heads/master"])
1942 removeSecurityProxy(repository)._default_branch = "refs/heads/master"1941 removeSecurityProxy(repository)._default_branch = "refs/heads/master"
1943 with person_logged_in(repository.owner):1942 with person_logged_in(repository.owner):
1944 repository.default_branch = "master"1943 repository.default_branch = "master"
1945 self.assertEqual([], hosting_fixture.setProperties.calls)1944 self.assertEqual([], self.hosting_fixture.setProperties.calls)
1946 self.assertEqual("refs/heads/master", repository.default_branch)1945 self.assertEqual("refs/heads/master", repository.default_branch)
19471946
1948 def test_set_default_branch_imported(self):1947 def test_set_default_branch_imported(self):
1949 hosting_fixture = self.useFixture(GitHostingFixture())
1950 repository = self.factory.makeGitRepository(1948 repository = self.factory.makeGitRepository(
1951 repository_type=GitRepositoryType.IMPORTED)1949 repository_type=GitRepositoryType.IMPORTED)
1952 self.factory.makeGitRefs(1950 self.factory.makeGitRefs(
@@ -1959,7 +1957,7 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1959 "Cannot modify non-hosted Git repository %s." %1957 "Cannot modify non-hosted Git repository %s." %
1960 repository.display_name,1958 repository.display_name,
1961 setattr, repository, "default_branch", "new")1959 setattr, repository, "default_branch", "new")
1962 self.assertEqual([], hosting_fixture.setProperties.calls)1960 self.assertEqual([], self.hosting_fixture.setProperties.calls)
1963 self.assertEqual("refs/heads/master", repository.default_branch)1961 self.assertEqual("refs/heads/master", repository.default_branch)
19641962
1965 def test_exception_unset_default_branch(self):1963 def test_exception_unset_default_branch(self):
@@ -2581,18 +2579,66 @@ class TestGitRepositoryUpdateLandingTargets(TestCaseWithFactory):
25812579
2582 layer = DatabaseFunctionalLayer2580 layer = DatabaseFunctionalLayer
25832581
2584 def test_schedules_diff_updates(self):2582 def setUp(self):
2583 super(TestGitRepositoryUpdateLandingTargets, self).setUp()
2584 self.hosting_fixture = self.useFixture(GitHostingFixture())
2585
2586 def assertSchedulesDiffUpdate(self, with_mp_virtual_ref):
2585 """Create jobs for all merge proposals."""2587 """Create jobs for all merge proposals."""
2586 bmp1 = self.factory.makeBranchMergeProposalForGit()2588 bmp1 = self.factory.makeBranchMergeProposalForGit()
2587 bmp2 = self.factory.makeBranchMergeProposalForGit(2589 bmp2 = self.factory.makeBranchMergeProposalForGit(
2588 source_ref=bmp1.source_git_ref)2590 source_ref=bmp1.source_git_ref)
2589 jobs = bmp1.source_git_repository.updateLandingTargets(2591
2590 [bmp1.source_git_path])2592 # Only enable this virtual ref here, since
2593 # self.factory.makeBranchMergeProposalForGit also tries to create
2594 # the virtual refs.
2595 if with_mp_virtual_ref:
2596 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
2597 else:
2598 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: ""}))
2599 jobs, ref_copies, ref_deletes = (
2600 bmp1.source_git_repository.updateLandingTargets(
2601 [bmp1.source_git_path]))
2591 self.assertEqual(2, len(jobs))2602 self.assertEqual(2, len(jobs))
2592 bmps_to_update = [2603 bmps_to_update = [
2593 removeSecurityProxy(job).branch_merge_proposal for job in jobs]2604 removeSecurityProxy(job).branch_merge_proposal for job in jobs]
2594 self.assertContentEqual([bmp1, bmp2], bmps_to_update)2605 self.assertContentEqual([bmp1, bmp2], bmps_to_update)
25952606
2607 if not with_mp_virtual_ref:
2608 self.assertEqual(
2609 0, self.hosting_fixture.bulkSyncRefs.call_count)
2610 else:
2611 self.assertEqual(
2612 1, self.hosting_fixture.bulkSyncRefs.call_count)
2613 args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
2614 self.assertEqual({}, kwargs)
2615 self.assertEqual(2, len(args))
2616 copy_ops, delete_ops = args
2617 self.assertEqual(2, len(copy_ops))
2618 self.assertEqual(0, len(delete_ops))
2619 self.assertThat(copy_ops[0], MatchesStructure(
2620 source_repo_path=Equals(
2621 bmp1.source_git_repository.getInternalPath()),
2622 source_ref=Equals(bmp1.source_git_commit_sha1),
2623 target_repo_path=Equals(
2624 bmp1.target_git_repository.getInternalPath()),
2625 target_ref=Equals("refs/merge/%s/head" % bmp1.id),
2626 ))
2627 self.assertThat(copy_ops[1], MatchesStructure(
2628 source_repo_path=Equals(
2629 bmp2.source_git_repository.getInternalPath()),
2630 source_ref=Equals(bmp2.source_git_commit_sha1),
2631 target_repo_path=Equals(
2632 bmp2.target_git_repository.getInternalPath()),
2633 target_ref=Equals("refs/merge/%s/head" % bmp2.id),
2634 ))
2635
2636 def test_schedules_diff_updates_with_mp_virtual_ref(self):
2637 self.assertSchedulesDiffUpdate(with_mp_virtual_ref=True)
2638
2639 def test_schedules_diff_updates_without_mp_virtual_ref(self):
2640 self.assertSchedulesDiffUpdate(with_mp_virtual_ref=False)
2641
2596 def test_ignores_final(self):2642 def test_ignores_final(self):
2597 """Diffs for proposals in final states aren't updated."""2643 """Diffs for proposals in final states aren't updated."""
2598 [source_ref] = self.factory.makeGitRefs()2644 [source_ref] = self.factory.makeGitRefs()
@@ -2604,8 +2650,17 @@ class TestGitRepositoryUpdateLandingTargets(TestCaseWithFactory):
2604 for bmp in source_ref.landing_targets:2650 for bmp in source_ref.landing_targets:
2605 if bmp.queue_status not in FINAL_STATES:2651 if bmp.queue_status not in FINAL_STATES:
2606 removeSecurityProxy(bmp).deleteProposal()2652 removeSecurityProxy(bmp).deleteProposal()
2607 jobs = source_ref.repository.updateLandingTargets([source_ref.path])2653
2654 # Enable the feature here, since factory.makeBranchMergeProposalForGit
2655 # would also trigger the copy refs call.
2656 self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
2657 jobs, ref_copies, ref_deletes = (
2658 source_ref.repository.updateLandingTargets(
2659 [source_ref.path]))
2608 self.assertEqual(0, len(jobs))2660 self.assertEqual(0, len(jobs))
2661 self.assertEqual(0, len(ref_copies))
2662 self.assertEqual(0, len(ref_deletes))
2663 self.assertEqual(0, self.hosting_fixture.copyRefs.call_count)
26092664
26102665
2611class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):2666class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
diff --git a/lib/lp/code/subscribers/branchmergeproposal.py b/lib/lp/code/subscribers/branchmergeproposal.py
index a214c8f..a34ca2a 100644
--- a/lib/lp/code/subscribers/branchmergeproposal.py
+++ b/lib/lp/code/subscribers/branchmergeproposal.py
@@ -1,4 +1,4 @@
1# Copyright 2010-2016 Canonical Ltd. This software is licensed under the1# Copyright 2010-2020 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).2# GNU Affero General Public License version 3 (see the file LICENSE).
33
4"""Event subscribers for branch merge proposals."""4"""Event subscribers for branch merge proposals."""
@@ -63,9 +63,13 @@ def _trigger_webhook(merge_proposal, payload):
63def merge_proposal_created(merge_proposal, event):63def merge_proposal_created(merge_proposal, event):
64 """A new merge proposal has been created.64 """A new merge proposal has been created.
6565
66 Create a job to update the diff for the merge proposal; trigger webhooks.66 Create a job to update the diff for the merge proposal; trigger webhooks
67 and copy virtual refs as needed.
67 """68 """
68 getUtility(IUpdatePreviewDiffJobSource).create(merge_proposal)69 getUtility(IUpdatePreviewDiffJobSource).create(merge_proposal)
70
71 merge_proposal.syncGitVirtualRefs()
72
69 if getFeatureFlag(BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG):73 if getFeatureFlag(BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG):
70 payload = {74 payload = {
71 "action": "created",75 "action": "created",
@@ -149,3 +153,5 @@ def merge_proposal_deleted(merge_proposal, event):
149 "old": _compose_merge_proposal_webhook_payload(merge_proposal),153 "old": _compose_merge_proposal_webhook_payload(merge_proposal),
150 }154 }
151 _trigger_webhook(merge_proposal, payload)155 _trigger_webhook(merge_proposal, payload)
156
157 merge_proposal.syncGitVirtualRefs(force_delete=True)
diff --git a/lib/lp/code/tests/helpers.py b/lib/lp/code/tests/helpers.py
index 4ef843d..3704b06 100644
--- a/lib/lp/code/tests/helpers.py
+++ b/lib/lp/code/tests/helpers.py
@@ -369,6 +369,9 @@ class GitHostingFixture(fixtures.Fixture):
369 self.getBlob = FakeMethod(result=blob)369 self.getBlob = FakeMethod(result=blob)
370 self.delete = FakeMethod()370 self.delete = FakeMethod()
371 self.disable_memcache = disable_memcache371 self.disable_memcache = disable_memcache
372 self.copyRefs = FakeMethod()
373 self.deleteRefs = FakeMethod()
374 self.bulkSyncRefs = FakeMethod()
372375
373 def _setUp(self):376 def _setUp(self):
374 self.useFixture(ZopeUtilityFixture(self, IGitHostingClient))377 self.useFixture(ZopeUtilityFixture(self, IGitHostingClient))