Merge ~pappacena/launchpad:git-repo-async-privacy-virtualrefs into launchpad:master

Proposed by Thiago F. Pappacena
Status: Superseded
Proposed branch: ~pappacena/launchpad:git-repo-async-privacy-virtualrefs
Merge into: launchpad:master
Diff against target: 1843 lines (+964/-53)
28 files modified
database/schema/security.cfg (+18/-0)
lib/lp/app/widgets/itemswidgets.py (+6/-2)
lib/lp/code/browser/gitrepository.py (+13/-1)
lib/lp/code/browser/tests/test_branchmergeproposal.py (+1/-1)
lib/lp/code/browser/tests/test_gitrepository.py (+67/-2)
lib/lp/code/configure.zcml (+9/-0)
lib/lp/code/enums.py (+6/-0)
lib/lp/code/interfaces/branchmergeproposal.py (+45/-1)
lib/lp/code/interfaces/githosting.py (+8/-0)
lib/lp/code/interfaces/gitjob.py (+20/-1)
lib/lp/code/interfaces/gitrepository.py (+12/-1)
lib/lp/code/model/branchmergeproposal.py (+98/-0)
lib/lp/code/model/githosting.py (+29/-4)
lib/lp/code/model/gitjob.py (+84/-0)
lib/lp/code/model/gitref.py (+1/-1)
lib/lp/code/model/gitrepository.py (+27/-3)
lib/lp/code/model/tests/test_branchmergeproposal.py (+234/-0)
lib/lp/code/model/tests/test_gitcollection.py (+1/-1)
lib/lp/code/model/tests/test_githosting.py (+2/-3)
lib/lp/code/model/tests/test_gitjob.py (+65/-0)
lib/lp/code/model/tests/test_gitrepository.py (+81/-24)
lib/lp/code/subscribers/branchmergeproposal.py (+8/-2)
lib/lp/code/tests/helpers.py (+3/-0)
lib/lp/code/xmlrpc/tests/test_git.py (+5/-5)
lib/lp/scripts/garbo.py (+54/-0)
lib/lp/scripts/tests/test_garbo.py (+60/-0)
lib/lp/services/config/schema-lazr.conf (+6/-0)
lib/lp/testing/factory.py (+1/-1)
Reviewer Review Type Date Requested Status
Launchpad code reviewers Pending
Review via email: mp+393074@code.launchpad.net

This proposal has been superseded by a proposal from 2020-10-29.

Commit message

Doing virtual refs sync when running git repository privacy change background task

Description of the change

This branch carries code from https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/390581, but it does a big chunch of refactoring of the code contained there. Mostly, the bulk operations got encapsulated in the GitHostingClient, so we would have a single point of change once Turnip will be able to run the copy/delete refs operations in bulk (instead of doing a bunch of requests).

To post a comment you must log in.

Unmerged commits

99d323d... by Thiago F. Pappacena

Using also branchscanner db permission when running repo privacy change job

e954159... by Thiago F. Pappacena

Merge branch 'git-repo-async-privacy' into git-repo-async-privacy-virtualrefs

974b22a... by Thiago F. Pappacena

Fixing test

6fde200... by Thiago F. Pappacena

Refactoring the way we deal with copy/delete virtual refs operations

7793c43... by Thiago F. Pappacena

Merge branch 'create-mp-refs' into git-repo-async-privacy-virtualrefs

b4af659... by Thiago F. Pappacena

Merge branch 'master' into create-mp-refs

1688ced... by Thiago F. Pappacena

Running the branch sync when changing repository privacy

6dbcc2a... by Thiago F. Pappacena

Merge branch 'create-mp-refs' into git-repo-async-privacy-virtualrefs

6118345... by Thiago F. Pappacena

Improving tests for more scenarios

20ea95f... by Thiago F. Pappacena

Adding garbo to keep GitRepo status in line with info type change job

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/database/schema/security.cfg b/database/schema/security.cfg
2index 11aba6f..94cb548 100644
3--- a/database/schema/security.cfg
4+++ b/database/schema/security.cfg
5@@ -2082,6 +2082,24 @@ public.webhookjob = SELECT, INSERT
6 public.xref = SELECT, INSERT, DELETE
7 type=user
8
9+[privacy-change-jobs]
10+groups=script, branchscanner
11+public.accessartifact = SELECT, UPDATE, DELETE, INSERT
12+public.accessartifactgrant = SELECT, UPDATE, DELETE, INSERT
13+public.accesspolicyartifact = SELECT, UPDATE, DELETE, INSERT
14+public.accesspolicygrant = SELECT, UPDATE, DELETE
15+public.account = SELECT
16+public.distribution = SELECT
17+public.gitjob = SELECT, UPDATE
18+public.gitrepository = SELECT, UPDATE
19+public.gitsubscription = SELECT, UPDATE, DELETE
20+public.job = SELECT, INSERT, UPDATE
21+public.person = SELECT
22+public.product = SELECT
23+public.sharingjob = SELECT, INSERT, UPDATE
24+public.teamparticipation = SELECT
25+type=user
26+
27 [sharing-jobs]
28 groups=script
29 public.accessartifactgrant = SELECT, UPDATE, DELETE
30diff --git a/lib/lp/app/widgets/itemswidgets.py b/lib/lp/app/widgets/itemswidgets.py
31index 1dbb59f..a644a96 100644
32--- a/lib/lp/app/widgets/itemswidgets.py
33+++ b/lib/lp/app/widgets/itemswidgets.py
34@@ -189,25 +189,29 @@ class LaunchpadRadioWidgetWithDescription(LaunchpadRadioWidget):
35 """Render an item of the list."""
36 text = html_escape(text)
37 id = '%s.%s' % (name, index)
38+ extra_attr = {"disabled": "disabled"} if self.context.readonly else {}
39 elem = renderElement(u'input',
40 value=value,
41 name=name,
42 id=id,
43 cssClass=cssClass,
44- type='radio')
45+ type='radio',
46+ **extra_attr)
47 return self._renderRow(text, value, id, elem)
48
49 def renderSelectedItem(self, index, text, value, name, cssClass):
50 """Render a selected item of the list."""
51 text = html_escape(text)
52 id = '%s.%s' % (name, index)
53+ extra_attr = {"disabled": "disabled"} if self.context.readonly else {}
54 elem = renderElement(u'input',
55 value=value,
56 name=name,
57 id=id,
58 cssClass=cssClass,
59 checked="checked",
60- type='radio')
61+ type='radio',
62+ **extra_attr)
63 return self._renderRow(text, value, id, elem)
64
65 def renderExtraHint(self):
66diff --git a/lib/lp/code/browser/gitrepository.py b/lib/lp/code/browser/gitrepository.py
67index 2552ffb..029e7ba 100644
68--- a/lib/lp/code/browser/gitrepository.py
69+++ b/lib/lp/code/browser/gitrepository.py
70@@ -483,6 +483,9 @@ class GitRepositoryView(InformationTypePortletMixin, LaunchpadView,
71 def warning_message(self):
72 if self.context.status == GitRepositoryStatus.CREATING:
73 return "This repository is being created."
74+ if (self.context.status ==
75+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION):
76+ return "This repository's information type is being changed."
77 return None
78
79 @property
80@@ -573,6 +576,9 @@ class GitRepositoryEditFormView(LaunchpadEditFormView):
81 @cachedproperty
82 def schema(self):
83 info_types = self.getInformationTypesToShow()
84+ read_only_info_type = (
85+ self.context.status ==
86+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION)
87
88 class GitRepositoryEditSchema(Interface):
89 """Defines the fields for the edit form.
90@@ -582,7 +588,8 @@ class GitRepositoryEditFormView(LaunchpadEditFormView):
91 """
92 use_template(IGitRepository, include=["default_branch"])
93 information_type = copy_field(
94- IGitRepository["information_type"], readonly=False,
95+ IGitRepository["information_type"],
96+ readonly=read_only_info_type,
97 vocabulary=InformationTypeVocabulary(types=info_types))
98 name = copy_field(IGitRepository["name"], readonly=False)
99 owner = copy_field(IGitRepository["owner"], readonly=False)
100@@ -785,6 +792,11 @@ class GitRepositoryEditView(CodeEditOwnerMixin, GitRepositoryEditFormView):
101 self.widgets["target"].hint = (
102 "This is the default repository for this target, so it "
103 "cannot be moved to another target.")
104+ if (self.context.status ==
105+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION):
106+ self.widgets["information_type"].hint = (
107+ "Information type is being changed. The operation needs to "
108+ "finish before you can changing it again.")
109 if self.context.default_branch:
110 self.widgets['default_branch'].context.required = True
111
112diff --git a/lib/lp/code/browser/tests/test_branchmergeproposal.py b/lib/lp/code/browser/tests/test_branchmergeproposal.py
113index f2ec4ec..83c51dc 100644
114--- a/lib/lp/code/browser/tests/test_branchmergeproposal.py
115+++ b/lib/lp/code/browser/tests/test_branchmergeproposal.py
116@@ -2336,7 +2336,7 @@ class TestLatestProposalsForEachBranchGit(
117
118 @staticmethod
119 def _setBranchInvisible(branch):
120- removeSecurityProxy(branch.repository).transitionToInformationType(
121+ removeSecurityProxy(branch.repository)._transitionToInformationType(
122 InformationType.USERDATA, branch.owner, verify_policy=False)
123
124
125diff --git a/lib/lp/code/browser/tests/test_gitrepository.py b/lib/lp/code/browser/tests/test_gitrepository.py
126index 912965d..8a9af9f 100644
127--- a/lib/lp/code/browser/tests/test_gitrepository.py
128+++ b/lib/lp/code/browser/tests/test_gitrepository.py
129@@ -60,6 +60,9 @@ from lp.code.enums import (
130 GitRepositoryType,
131 )
132 from lp.code.interfaces.gitcollection import IGitCollection
133+from lp.code.interfaces.gitjob import (
134+ IGitRepositoryTransitionToInformationTypeJobSource,
135+ )
136 from lp.code.interfaces.gitrepository import IGitRepositorySet
137 from lp.code.interfaces.revision import IRevisionSet
138 from lp.code.model.gitjob import GitRefScanJob
139@@ -161,6 +164,15 @@ class TestGitRepositoryView(BrowserTestCase):
140 self.assertTextMatchesExpressionIgnoreWhitespace(
141 r"""This repository is being created\..*""", text)
142
143+ def test_changing_info_type_warning_message_is_present(self):
144+ repository = removeSecurityProxy(self.factory.makeGitRepository())
145+ repository.status = (
146+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION)
147+ text = self.getMainText(repository, "+index", user=repository.owner)
148+ self.assertTextMatchesExpressionIgnoreWhitespace(
149+ r"""This repository's information type is being changed\..*""",
150+ text)
151+
152 def test_creating_warning_message_is_not_shown(self):
153 repository = removeSecurityProxy(self.factory.makeGitRepository())
154 repository.status = GitRepositoryStatus.AVAILABLE
155@@ -1161,7 +1173,50 @@ class TestGitRepositoryEditView(TestCaseWithFactory):
156 browser.getControl("Change Git Repository").click()
157 with person_logged_in(person):
158 self.assertEqual(
159- InformationType.USERDATA, repository.information_type)
160+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
161+ repository.status)
162+ job_util = getUtility(
163+ IGitRepositoryTransitionToInformationTypeJobSource)
164+ jobs = list(job_util.iterReady())
165+ self.assertEqual(1, len(jobs))
166+ job = removeSecurityProxy(jobs[0])
167+ self.assertEqual(repository, job.repository)
168+ self.assertEqual(InformationType.USERDATA, job.information_type)
169+ self.assertEqual(admin, job.user)
170+
171+ def test_information_type_in_ui_blocked_if_already_changing(self):
172+ # The information_type of a repository can't be changed via the UI
173+ # if the repository is already pending a info type change.
174+ person = self.factory.makePerson()
175+ repository = self.factory.makeGitRepository(owner=person)
176+ removeSecurityProxy(repository).status = (
177+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION)
178+ admin = getUtility(ILaunchpadCelebrities).admin.teamowner
179+ browser = self.getUserBrowser(
180+ canonical_url(repository) + "/+edit", user=admin)
181+ # Make sure the privacy controls are all disabled in the UI.
182+ controls = [
183+ "Public", "Public Security", "Private Security", "Private",
184+ "Proprietary", "Embargoed"]
185+ self.assertTrue(
186+ all(browser.getControl(i, index=0).disabled for i in controls))
187+ expected_msg = (
188+ "Information type is being changed. The operation needs to "
189+ "finish before you can changing it again.")
190+ self.assertIn(expected_msg, extract_text(browser.contents))
191+
192+ # Trying to change should have no effect in the backend, since the
193+ # repository is already changing info type and this field is read-only.
194+ browser.getControl("Private", index=1).click()
195+ browser.getControl("Change Git Repository").click()
196+ with person_logged_in(person):
197+ self.assertEqual(
198+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
199+ repository.status)
200+ job_util = getUtility(
201+ IGitRepositoryTransitionToInformationTypeJobSource)
202+ jobs = list(job_util.iterReady())
203+ self.assertEqual(0, len(jobs))
204
205 def test_edit_view_ajax_render(self):
206 # An information type change request is processed as expected when
207@@ -1184,7 +1239,17 @@ class TestGitRepositoryEditView(TestCaseWithFactory):
208 result = view.render()
209 self.assertEqual("", result)
210 self.assertEqual(
211- repository.information_type, InformationType.PUBLICSECURITY)
212+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
213+ repository.status)
214+ job_util = getUtility(
215+ IGitRepositoryTransitionToInformationTypeJobSource)
216+ jobs = list(job_util.iterReady())
217+ self.assertEqual(1, len(jobs))
218+ job = removeSecurityProxy(jobs[0])
219+ self.assertEqual(repository, job.repository)
220+ self.assertEqual(
221+ InformationType.PUBLICSECURITY, job.information_type)
222+ self.assertEqual(person, job.user)
223
224 def test_change_default_branch(self):
225 # An authorised user can change the default branch to one that
226diff --git a/lib/lp/code/configure.zcml b/lib/lp/code/configure.zcml
227index 898e645..fee38a9 100644
228--- a/lib/lp/code/configure.zcml
229+++ b/lib/lp/code/configure.zcml
230@@ -1111,6 +1111,11 @@
231 provides="lp.code.interfaces.gitjob.IGitRepositoryModifiedMailJobSource">
232 <allow interface="lp.code.interfaces.gitjob.IGitRepositoryModifiedMailJobSource" />
233 </securedutility>
234+ <securedutility
235+ component="lp.code.model.gitjob.GitRepositoryTransitionToInformationTypeJob"
236+ provides="lp.code.interfaces.gitjob.IGitRepositoryTransitionToInformationTypeJobSource">
237+ <allow interface="lp.code.interfaces.gitjob.IGitRepositoryTransitionToInformationTypeJobSource" />
238+ </securedutility>
239 <class class="lp.code.model.gitjob.GitRefScanJob">
240 <allow interface="lp.code.interfaces.gitjob.IGitJob" />
241 <allow interface="lp.code.interfaces.gitjob.IGitRefScanJob" />
242@@ -1123,6 +1128,10 @@
243 <allow interface="lp.code.interfaces.gitjob.IGitJob" />
244 <allow interface="lp.code.interfaces.gitjob.IGitRepositoryModifiedMailJob" />
245 </class>
246+ <class class="lp.code.model.gitjob.GitRepositoryTransitionToInformationTypeJob">
247+ <allow interface="lp.code.interfaces.gitjob.IGitJob" />
248+ <allow interface="lp.code.interfaces.gitjob.IGitRepositoryTransitionToInformationTypeJob" />
249+ </class>
250
251 <lp:help-folder folder="help" name="+help-code" />
252
253diff --git a/lib/lp/code/enums.py b/lib/lp/code/enums.py
254index d2a9262..2edf401 100644
255--- a/lib/lp/code/enums.py
256+++ b/lib/lp/code/enums.py
257@@ -167,6 +167,12 @@ class GitRepositoryStatus(DBEnumeratedType):
258 This repository is available to be used.
259 """)
260
261+ PENDING_INFORMATION_TYPE_TRANSITION = DBItem(3, """
262+ Information type transition pending
263+
264+ This repository's privacy setting is being changed.
265+ """)
266+
267
268 class GitObjectType(DBEnumeratedType):
269 """Git Object Type
270diff --git a/lib/lp/code/interfaces/branchmergeproposal.py b/lib/lp/code/interfaces/branchmergeproposal.py
271index bf481c9..efe1e4a 100644
272--- a/lib/lp/code/interfaces/branchmergeproposal.py
273+++ b/lib/lp/code/interfaces/branchmergeproposal.py
274@@ -1,4 +1,4 @@
275-# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
276+# Copyright 2009-2020 Canonical Ltd. This software is licensed under the
277 # GNU Affero General Public License version 3 (see the file LICENSE).
278
279 """The interface for branch merge proposals."""
280@@ -519,6 +519,50 @@ class IBranchMergeProposalView(Interface):
281 def getLatestDiffUpdateJob():
282 """Return the latest IUpdatePreviewDiffJob for this MP."""
283
284+ def getGitRefCopyOperations():
285+ """Gets the list of GitHosting copy operations that should be done
286+ in order to keep this MP's virtual refs up-to-date.
287+
288+ Note that, for now, the only possible virtual ref that could
289+ possibly be copied is GIT_CREATE_MP_VIRTUAL_REF. So, this method
290+ will return at most 1 copy operation. Once new virtual refs will be
291+ created, this method should be extended to add more copy operations
292+ too.
293+
294+ :return: A list of RefCopyOperation objects.
295+ """
296+
297+ def getGitRefDeleteOperations(force_delete=False):
298+ """Gets the list of git refs that should be deleted in order to keep
299+ this MP's virtual refs up-to-date.
300+
301+ Note that, for now, the only existing virtual ref to be deleted is
302+ GIT_CREATE_MP_VIRTUAL_REF. So this method will only ever delete at
303+ most 1 virtual ref. Once new virtual refs will be created, this method
304+ should be extended to delete them also.
305+
306+ :param force_delete: True if we should get delete operation
307+ regardless of any check. False otherwise.
308+ :return: A list of ref names.
309+ """
310+
311+ def syncGitVirtualRefs(force_delete=False):
312+ """Requests all copies and deletion of virtual refs to make git code
313+ hosting in sync with this MP.
314+
315+ :param force_delete: Do not try to copy any new virtual ref. Just
316+ delete all virtual refs possibly created.
317+ """
318+
319+ def bulkSyncGitVirtualRefs(merge_proposals):
320+ """Synchronizes a set of merge proposals' virtual refs.
321+
322+ :params merge_proposals: The set of merge proposals that should have
323+ the virtual refs synced.
324+ :return: A tuple with the list of (copy operations, delete operations)
325+ executed.
326+ """
327+
328
329 class IBranchMergeProposalEdit(Interface):
330
331diff --git a/lib/lp/code/interfaces/githosting.py b/lib/lp/code/interfaces/githosting.py
332index 441e850..7fe0d0e 100644
333--- a/lib/lp/code/interfaces/githosting.py
334+++ b/lib/lp/code/interfaces/githosting.py
335@@ -148,3 +148,11 @@ class IGitHostingClient(Interface):
336 :param refs: A list of tuples like (repo_path, ref_name) to be deleted.
337 :param logger: An optional logger.
338 """
339+
340+ def bulkSyncRefs(copy_operations, delete_operations):
341+ """Executes all copy operations and delete operations on Turnip
342+ side.
343+
344+ :param copy_operations: The list of RefCopyOperation to run.
345+ :param delete_operations: The list of RefDeleteOperation to run.
346+ """
347diff --git a/lib/lp/code/interfaces/gitjob.py b/lib/lp/code/interfaces/gitjob.py
348index 4f31b19..bfe5525 100644
349--- a/lib/lp/code/interfaces/gitjob.py
350+++ b/lib/lp/code/interfaces/gitjob.py
351@@ -1,4 +1,4 @@
352-# Copyright 2015 Canonical Ltd. This software is licensed under the
353+# Copyright 2015-2020 Canonical Ltd. This software is licensed under the
354 # GNU Affero General Public License version 3 (see the file LICENSE).
355
356 """GitJob interfaces."""
357@@ -11,6 +11,8 @@ __all__ = [
358 'IGitRefScanJobSource',
359 'IGitRepositoryModifiedMailJob',
360 'IGitRepositoryModifiedMailJobSource',
361+ 'IGitRepositoryTransitionToInformationTypeJob',
362+ 'IGitRepositoryTransitionToInformationTypeJobSource',
363 'IReclaimGitRepositorySpaceJob',
364 'IReclaimGitRepositorySpaceJobSource',
365 ]
366@@ -93,3 +95,20 @@ class IGitRepositoryModifiedMailJobSource(IJobSource):
367 :param repository_delta: An `IGitRepositoryDelta` describing the
368 changes.
369 """
370+
371+
372+class IGitRepositoryTransitionToInformationTypeJob(IRunnableJob):
373+ """A Job to change repository's information type."""
374+
375+
376+class IGitRepositoryTransitionToInformationTypeJobSource(IJobSource):
377+
378+ def create(repository, user, information_type, verify_policy=True):
379+ """Create a job to change git repository's information type.
380+
381+ :param repository: The `IGitRepository` that was modified.
382+ :param information_type: The `InformationType` to transition to.
383+ :param user: The `IPerson` who is making the change.
384+ :param verify_policy: Check if the new information type complies
385+ with the `IGitNamespacePolicy`.
386+ """
387diff --git a/lib/lp/code/interfaces/gitrepository.py b/lib/lp/code/interfaces/gitrepository.py
388index 192133a..48e7dd3 100644
389--- a/lib/lp/code/interfaces/gitrepository.py
390+++ b/lib/lp/code/interfaces/gitrepository.py
391@@ -8,6 +8,8 @@ __metaclass__ = type
392 __all__ = [
393 'ContributorGitIdentity',
394 'GitIdentityMixin',
395+ 'GIT_CREATE_MP_VIRTUAL_REF',
396+ 'GIT_MP_VIRTUAL_REF_FORMAT',
397 'GIT_REPOSITORY_NAME_VALIDATION_ERROR_MESSAGE',
398 'git_repository_name_validator',
399 'IGitRepository',
400@@ -105,6 +107,11 @@ valid_git_repository_name_pattern = re.compile(
401 r"^(?i)[a-z0-9][a-z0-9+\.\-@_]*\Z")
402
403
404+# Virtual ref where we automatically put a copy of any open merge proposal.
405+GIT_MP_VIRTUAL_REF_FORMAT = 'refs/merge/{mp.id}/head'
406+GIT_CREATE_MP_VIRTUAL_REF = 'git.mergeproposal_virtualref.enabled'
407+
408+
409 def valid_git_repository_name(name):
410 """Return True iff the name is valid as a Git repository name.
411
412@@ -595,11 +602,15 @@ class IGitRepositoryView(IHasRecipes):
413 def updateLandingTargets(paths):
414 """Update landing targets (MPs where this repository is the source).
415
416- For each merge proposal, create `UpdatePreviewDiffJob`s.
417+ For each merge proposal, create `UpdatePreviewDiffJob`s, and runs
418+ the appropriate GitHosting.copyRefs operation to allow having
419+ virtual merge refs with the updated branch version.
420
421 :param paths: A list of reference paths. Any merge proposals whose
422 source is this repository and one of these paths will have their
423 diffs updated.
424+ :return: Returns a tuple with the list of background jobs created,
425+ and the list of ref copies requested to GitHosting.
426 """
427
428 def markRecipesStale(paths):
429diff --git a/lib/lp/code/model/branchmergeproposal.py b/lib/lp/code/model/branchmergeproposal.py
430index 061fb1e..821b15d 100644
431--- a/lib/lp/code/model/branchmergeproposal.py
432+++ b/lib/lp/code/model/branchmergeproposal.py
433@@ -12,6 +12,7 @@ __all__ = [
434
435 from collections import defaultdict
436 from email.utils import make_msgid
437+import logging
438 from operator import attrgetter
439 import sys
440
441@@ -84,7 +85,12 @@ from lp.code.interfaces.codereviewcomment import ICodeReviewComment
442 from lp.code.interfaces.codereviewinlinecomment import (
443 ICodeReviewInlineCommentSet,
444 )
445+from lp.code.interfaces.githosting import IGitHostingClient
446 from lp.code.interfaces.gitref import IGitRef
447+from lp.code.interfaces.gitrepository import (
448+ GIT_CREATE_MP_VIRTUAL_REF,
449+ GIT_MP_VIRTUAL_REF_FORMAT,
450+ )
451 from lp.code.mail.branch import RecipientReason
452 from lp.code.model.branchrevision import BranchRevision
453 from lp.code.model.codereviewcomment import CodeReviewComment
454@@ -94,6 +100,10 @@ from lp.code.model.diff import (
455 IncrementalDiff,
456 PreviewDiff,
457 )
458+from lp.code.model.githosting import (
459+ RefCopyOperation,
460+ RefDeleteOperation,
461+ )
462 from lp.registry.interfaces.person import (
463 IPerson,
464 IPersonSet,
465@@ -123,6 +133,7 @@ from lp.services.database.sqlbase import (
466 quote,
467 SQLBase,
468 )
469+from lp.services.features import getFeatureFlag
470 from lp.services.helpers import shortlist
471 from lp.services.job.interfaces.job import JobStatus
472 from lp.services.job.model.job import Job
473@@ -144,6 +155,9 @@ from lp.soyuz.enums import (
474 )
475
476
477+logger = logging.getLogger(__name__)
478+
479+
480 def is_valid_transition(proposal, from_state, next_state, user=None):
481 """Is it valid for this user to move this proposal to to next_state?
482
483@@ -971,6 +985,7 @@ class BranchMergeProposal(SQLBase, BugLinkTargetMixin):
484 branch_merge_proposal=self.id):
485 job.destroySelf()
486 self._preview_diffs.remove()
487+
488 self.destroySelf()
489
490 def getUnlandedSourceBranchRevisions(self):
491@@ -1217,6 +1232,89 @@ class BranchMergeProposal(SQLBase, BugLinkTargetMixin):
492 for diff in diffs)
493 return [diff_dict.get(revisions) for revisions in revision_list]
494
495+ @property
496+ def should_have_git_virtual_refs(self):
497+ """True if this MP should have virtual refs in the target
498+ repository. False otherwise."""
499+ # If the source repository is private and it's opening a MP to
500+ # another repository, we should not risk disclosing the private code to
501+ # everyone with permission to see the target repo:
502+ private_source = self.source_git_repository.private
503+ same_owner = self.source_git_repository.owner.inTeam(
504+ self.target_git_repository.owner)
505+ return not private_source or same_owner
506+
507+ def getGitRefCopyOperations(self):
508+ """See `IBranchMergeProposal`"""
509+ if (not self.target_git_repository
510+ or not getFeatureFlag(GIT_CREATE_MP_VIRTUAL_REF)):
511+ # Not a git MP. Skip.
512+ return []
513+ if not self.should_have_git_virtual_refs:
514+ return []
515+ return [RefCopyOperation(
516+ self.source_git_repository.getInternalPath(),
517+ self.source_git_commit_sha1,
518+ self.target_git_repository.getInternalPath(),
519+ GIT_MP_VIRTUAL_REF_FORMAT.format(mp=self))]
520+
521+ def getGitRefDeleteOperations(self, force_delete=False):
522+ """See `IBranchMergeProposal`"""
523+ if self.source_git_ref is None:
524+ # Not a git MP. Skip.
525+ return []
526+ if not force_delete and self.should_have_git_virtual_refs:
527+ return []
528+ return [RefDeleteOperation(
529+ self.target_git_repository.getInternalPath(),
530+ GIT_MP_VIRTUAL_REF_FORMAT.format(mp=self))]
531+
532+ def syncGitVirtualRefs(self, force_delete=False):
533+ """See `IBranchMergeProposal`"""
534+ if self.source_git_ref is None:
535+ return
536+ if not getFeatureFlag(GIT_CREATE_MP_VIRTUAL_REF):
537+ # XXX pappacena 2020-09-15: Do not try to remove virtual refs if
538+ # the feature is disabled. It might be disabled due to bug on
539+ # the first versions, for example, and we should have a way to
540+ # skip this feature entirely.
541+ # Once the feature is stable, we should actually *always* delete
542+ # the virtual refs, since we don't know if the feature was
543+ # enabled or not when the branch was created.
544+ return
545+ if force_delete:
546+ copy_operations = []
547+ else:
548+ copy_operations = self.getGitRefCopyOperations()
549+ delete_operations = self.getGitRefDeleteOperations(force_delete)
550+
551+ if copy_operations or delete_operations:
552+ hosting_client = getUtility(IGitHostingClient)
553+ hosting_client.bulkSyncRefs(copy_operations, delete_operations)
554+
555+ @classmethod
556+ def bulkSyncGitVirtualRefs(cls, merge_proposals):
557+ """See `IBranchMergeProposal`"""
558+ if not getFeatureFlag(GIT_CREATE_MP_VIRTUAL_REF):
559+ # XXX pappacena 2020-09-15: Do not try to remove virtual refs if
560+ # the feature is disabled. It might be disabled due to bug on
561+ # the first versions, for example, and we should have a way to
562+ # skip this feature entirely.
563+ # Once the feature is stable, we should actually *always* delete
564+ # the virtual refs, since we don't know if the feature was
565+ # enabled or not when the branch was created.
566+ return [], []
567+ copy_operations = []
568+ delete_operations = []
569+ for merge_proposal in merge_proposals:
570+ copy_operations += merge_proposal.getGitRefCopyOperations()
571+ delete_operations += merge_proposal.getGitRefDeleteOperations()
572+
573+ if copy_operations or delete_operations:
574+ hosting_client = getUtility(IGitHostingClient)
575+ hosting_client.bulkSyncRefs(copy_operations, delete_operations)
576+ return copy_operations, delete_operations
577+
578 def scheduleDiffUpdates(self, return_jobs=True):
579 """See `IBranchMergeProposal`."""
580 from lp.code.model.branchmergeproposaljob import (
581diff --git a/lib/lp/code/model/githosting.py b/lib/lp/code/model/githosting.py
582index 74f4ec0..b2632f6 100644
583--- a/lib/lp/code/model/githosting.py
584+++ b/lib/lp/code/model/githosting.py
585@@ -7,9 +7,11 @@ __metaclass__ = type
586 __all__ = [
587 'GitHostingClient',
588 'RefCopyOperation',
589+ 'RefDeleteOperation',
590 ]
591
592 import base64
593+from collections import defaultdict
594 import json
595 import sys
596
597@@ -33,7 +35,6 @@ from lp.code.errors import (
598 GitRepositoryDeletionFault,
599 GitRepositoryScanFault,
600 GitTargetError,
601- NoSuchGitReference,
602 )
603 from lp.code.interfaces.githosting import IGitHostingClient
604 from lp.services.config import config
605@@ -55,12 +56,21 @@ class RefCopyOperation:
606 This class is just a helper to define copy operations parameters on
607 IGitHostingClient.copyRefs method.
608 """
609- def __init__(self, source_ref, target_repo, target_ref):
610+ def __init__(self, source_repo_path, source_ref, target_repo_path,
611+ target_ref):
612+ self.source_repo_path = source_repo_path
613 self.source_ref = source_ref
614- self.target_repo = target_repo
615+ self.target_repo_path = target_repo_path
616 self.target_ref = target_ref
617
618
619+class RefDeleteOperation:
620+ """A description of a ref delete operation."""
621+ def __init__(self, repo_path, ref):
622+ self.repo_path = repo_path
623+ self.ref = ref
624+
625+
626 @implementer(IGitHostingClient)
627 class GitHostingClient:
628 """A client for the internal API provided by the Git hosting system."""
629@@ -277,7 +287,7 @@ class GitHostingClient:
630 json_data = {
631 "operations": [{
632 "from": i.source_ref,
633- "to": {"repo": i.target_repo, "ref": i.target_ref}
634+ "to": {"repo": i.target_repo_path, "ref": i.target_ref}
635 } for i in operations]
636 }
637 try:
638@@ -299,6 +309,8 @@ class GitHostingClient:
639
640 def deleteRefs(self, refs, logger=None):
641 """See `IGitHostingClient`."""
642+ # XXX pappacena: This needs to be done in a bulk operation,
643+ # instead of several requests.
644 for path, ref in refs:
645 try:
646 if logger is not None:
647@@ -309,3 +321,16 @@ class GitHostingClient:
648 raise GitReferenceDeletionFault(
649 "Error deleting %s from repo %s: HTTP %s" %
650 (ref, path, e.response.status_code))
651+
652+ def bulkSyncRefs(self, copy_operations, delete_operations):
653+ """See `IGitHostingClient`."""
654+ # XXX pappacena: This needs to be done in a bulk operation on Turnip
655+ # side, instead of several requests.
656+ operations_per_path = defaultdict(list)
657+ for operation in copy_operations:
658+ operations_per_path[operation.source_repo_path].append(operation)
659+ for repo_path, operations in operations_per_path:
660+ self.copyRefs(repo_path, operations)
661+
662+ self.deleteRefs([(i.repo_path, i.ref) for i in delete_operations])
663+
664diff --git a/lib/lp/code/model/gitjob.py b/lib/lp/code/model/gitjob.py
665index 1eb87da..77135b3 100644
666--- a/lib/lp/code/model/gitjob.py
667+++ b/lib/lp/code/model/gitjob.py
668@@ -12,6 +12,7 @@ __all__ = [
669 'ReclaimGitRepositorySpaceJob',
670 ]
671
672+from itertools import chain
673
674 from lazr.delegates import delegate_to
675 from lazr.enum import (
676@@ -33,10 +34,15 @@ from zope.interface import (
677 provider,
678 )
679
680+from lp.app.enums import (
681+ InformationType,
682+ PRIVATE_INFORMATION_TYPES,
683+ )
684 from lp.app.errors import NotFoundError
685 from lp.code.enums import (
686 GitActivityType,
687 GitPermissionType,
688+ GitRepositoryStatus,
689 )
690 from lp.code.interfaces.githosting import IGitHostingClient
691 from lp.code.interfaces.gitjob import (
692@@ -45,6 +51,8 @@ from lp.code.interfaces.gitjob import (
693 IGitRefScanJobSource,
694 IGitRepositoryModifiedMailJob,
695 IGitRepositoryModifiedMailJobSource,
696+ IGitRepositoryTransitionToInformationTypeJob,
697+ IGitRepositoryTransitionToInformationTypeJobSource,
698 IReclaimGitRepositorySpaceJob,
699 IReclaimGitRepositorySpaceJobSource,
700 )
701@@ -100,6 +108,13 @@ class GitJobType(DBEnumeratedType):
702 modifications.
703 """)
704
705+ REPOSITORY_TRANSITION_TO_INFO_TYPE = DBItem(3, """
706+ Change repository's information type
707+
708+ This job runs when a user requests to change privacy settings of a
709+ repository.
710+ """)
711+
712
713 @implementer(IGitJob)
714 class GitJob(StormBase):
715@@ -393,3 +408,72 @@ class GitRepositoryModifiedMailJob(GitJobDerived):
716 def run(self):
717 """See `IGitRepositoryModifiedMailJob`."""
718 self.getMailer().sendAll()
719+
720+
721+@implementer(IGitRepositoryTransitionToInformationTypeJob)
722+@provider(IGitRepositoryTransitionToInformationTypeJobSource)
723+class GitRepositoryTransitionToInformationTypeJob(GitJobDerived):
724+ """A Job to change git repository's information type."""
725+
726+ class_job_type = GitJobType.REPOSITORY_TRANSITION_TO_INFO_TYPE
727+
728+ config = config.IGitRepositoryTransitionToInformationTypeJobSource
729+
730+ @classmethod
731+ def create(cls, repository, information_type, user, verify_policy=True):
732+ """See `IGitRepositoryTransitionToInformationTypeJobSource`."""
733+ metadata = {
734+ "user": user.id,
735+ "information_type": information_type.value,
736+ "verify_policy": verify_policy,
737+ }
738+ git_job = GitJob(repository, cls.class_job_type, metadata)
739+ job = cls(git_job)
740+ job.celeryRunOnCommit()
741+ return job
742+
743+ @property
744+ def user(self):
745+ return getUtility(IPersonSet).get(self.metadata["user"])
746+
747+ @property
748+ def verify_policy(self):
749+ return self.metadata["verify_policy"]
750+
751+ @property
752+ def information_type(self):
753+ return InformationType.items[self.metadata["information_type"]]
754+
755+ def updateVirtualRefs(self):
756+ copy_operations = []
757+ delete_operations = []
758+ merge_proposals = chain(
759+ self.repository.landing_targets,
760+ self.repository.landing_candidates)
761+ for mp in merge_proposals:
762+ copy_operations += mp.getGitRefCopyOperations()
763+ delete_operations += mp.getGitRefDeleteOperations()
764+ hosting_client = getUtility(IGitHostingClient)
765+ hosting_client.bulkSyncRefs(copy_operations, delete_operations)
766+
767+ def run(self):
768+ """See `IGitRepositoryTransitionToInformationTypeJob`."""
769+ if (self.repository.status !=
770+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION):
771+ raise AttributeError(
772+ "The repository %s is not pending information type change." %
773+ self.repository)
774+ is_private = self.repository.private
775+ will_be_private = self.information_type in PRIVATE_INFORMATION_TYPES
776+
777+ self.repository._transitionToInformationType(
778+ self.information_type, self.user, self.verify_policy)
779+
780+ # When changing privacy type, we need to make sure to not leak
781+ # anything though virtual refs, and create new ones that didn't
782+ # exist before.
783+ if is_private != will_be_private:
784+ # XXX pappacena 2020-10-28: We need to implement retry rules here.
785+ self.updateVirtualRefs()
786+
787+ self.repository.status = GitRepositoryStatus.AVAILABLE
788diff --git a/lib/lp/code/model/gitref.py b/lib/lp/code/model/gitref.py
789index f7dc142..e2f788d 100644
790--- a/lib/lp/code/model/gitref.py
791+++ b/lib/lp/code/model/gitref.py
792@@ -180,7 +180,7 @@ class GitRefMixin:
793
794 def transitionToInformationType(self, information_type, user,
795 verify_policy=True):
796- return self.repository.transitionToInformationType(
797+ return self.repository._transitionToInformationType(
798 information_type, user, verify_policy=verify_policy)
799
800 @property
801diff --git a/lib/lp/code/model/gitrepository.py b/lib/lp/code/model/gitrepository.py
802index fb5e703..731ca3f 100644
803--- a/lib/lp/code/model/gitrepository.py
804+++ b/lib/lp/code/model/gitrepository.py
805@@ -117,7 +117,10 @@ from lp.code.interfaces.gitcollection import (
806 IGitCollection,
807 )
808 from lp.code.interfaces.githosting import IGitHostingClient
809-from lp.code.interfaces.gitjob import IGitRefScanJobSource
810+from lp.code.interfaces.gitjob import (
811+ IGitRefScanJobSource,
812+ IGitRepositoryTransitionToInformationTypeJobSource,
813+ )
814 from lp.code.interfaces.gitlookup import IGitLookup
815 from lp.code.interfaces.gitnamespace import (
816 get_git_namespace,
817@@ -882,6 +885,24 @@ class GitRepository(StormBase, WebhookTargetMixin, GitIdentityMixin):
818 def transitionToInformationType(self, information_type, user,
819 verify_policy=True):
820 """See `IGitRepository`."""
821+ if self.status != GitRepositoryStatus.AVAILABLE:
822+ raise CannotChangeInformationType(
823+ "Cannot change privacy settings while git repository is "
824+ "being changed.")
825+ self.status = GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION
826+ util = getUtility(
827+ IGitRepositoryTransitionToInformationTypeJobSource)
828+ return util.create(self, information_type, user, verify_policy)
829+
830+ def _transitionToInformationType(self, information_type, user,
831+ verify_policy=True):
832+ """Synchronously make the change in this repository's information
833+ type.
834+
835+ External callers should use the async, public version of this
836+ method, since it deals with the side effects of changing
837+ repository's privacy changes.
838+ """
839 if self.information_type == information_type:
840 return
841 if (verify_policy and
842@@ -1184,9 +1205,12 @@ class GitRepository(StormBase, WebhookTargetMixin, GitIdentityMixin):
843 def updateLandingTargets(self, paths):
844 """See `IGitRepository`."""
845 jobs = []
846- for merge_proposal in self.getActiveLandingTargets(paths):
847+ merge_proposals = self.getActiveLandingTargets(paths)
848+ for merge_proposal in merge_proposals:
849 jobs.extend(merge_proposal.scheduleDiffUpdates())
850- return jobs
851+ copy_ops, delete_ops = BranchMergeProposal.bulkSyncGitVirtualRefs(
852+ merge_proposals)
853+ return jobs, copy_ops, delete_ops
854
855 def _getRecipes(self, paths=None):
856 """Undecorated version of recipes for use by `markRecipesStale`."""
857diff --git a/lib/lp/code/model/tests/test_branchmergeproposal.py b/lib/lp/code/model/tests/test_branchmergeproposal.py
858index 81b94b3..b601ad8 100644
859--- a/lib/lp/code/model/tests/test_branchmergeproposal.py
860+++ b/lib/lp/code/model/tests/test_branchmergeproposal.py
861@@ -67,6 +67,10 @@ from lp.code.interfaces.branchmergeproposal import (
862 IBranchMergeProposalGetter,
863 IBranchMergeProposalJobSource,
864 )
865+from lp.code.interfaces.gitrepository import (
866+ GIT_CREATE_MP_VIRTUAL_REF,
867+ GIT_MP_VIRTUAL_REF_FORMAT,
868+ )
869 from lp.code.model.branchmergeproposal import (
870 BranchMergeProposal,
871 BranchMergeProposalGetter,
872@@ -97,6 +101,7 @@ from lp.testing import (
873 ExpectedException,
874 launchpadlib_for,
875 login,
876+ login_admin,
877 login_person,
878 person_logged_in,
879 TestCaseWithFactory,
880@@ -256,6 +261,196 @@ class TestBranchMergeProposalPrivacy(TestCaseWithFactory):
881 self.assertContentEqual([owner, team], subscriptions)
882
883
884+class TestGitBranchMergeProposalVirtualRefs(TestCaseWithFactory):
885+ """Ensure that BranchMergeProposal creation run the appropriate copy
886+ and delete of virtual refs, like ref/merge/<id>/head."""
887+
888+ layer = DatabaseFunctionalLayer
889+
890+ def setUp(self):
891+ super(TestGitBranchMergeProposalVirtualRefs, self).setUp()
892+ self.hosting_fixture = self.useFixture(GitHostingFixture())
893+
894+ def test_should_have_vrefs_public_repos(self):
895+ mp = self.factory.makeBranchMergeProposalForGit()
896+ self.assertTrue(mp.should_have_git_virtual_refs)
897+
898+ def test_should_have_vrefs_private_source(self):
899+ login_admin()
900+ source_repo = self.factory.makeGitRepository(
901+ information_type=InformationType.PRIVATESECURITY)
902+ target_repo = self.factory.makeGitRepository(target=source_repo.target)
903+ [source] = self.factory.makeGitRefs(source_repo)
904+ [target] = self.factory.makeGitRefs(target_repo)
905+ mp = self.factory.makeBranchMergeProposalForGit(
906+ source_ref=source, target_ref=target)
907+ self.assertFalse(mp.should_have_git_virtual_refs)
908+
909+ def test_should_have_vrefs_private_target(self):
910+ login_admin()
911+ source_repo = self.factory.makeGitRepository()
912+ target_repo = self.factory.makeGitRepository(
913+ target=source_repo.target,
914+ information_type=InformationType.PRIVATESECURITY)
915+ [source] = self.factory.makeGitRefs(source_repo)
916+ [target] = self.factory.makeGitRefs(target_repo)
917+ mp = self.factory.makeBranchMergeProposalForGit(
918+ source_ref=source, target_ref=target)
919+ self.assertTrue(mp.should_have_git_virtual_refs)
920+
921+ def test_copy_git_merge_virtual_ref(self):
922+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
923+ mp = self.factory.makeBranchMergeProposalForGit()
924+
925+ copy_operations = mp.getGitRefCopyOperations()
926+ self.assertEqual(1, len(copy_operations))
927+ self.assertThat(copy_operations[0], MatchesStructure(
928+ source_repo_path=Equals(
929+ mp.source_git_repository.getInternalPath()),
930+ source_ref=Equals(mp.source_git_commit_sha1),
931+ target_repo_path=Equals(
932+ mp.target_git_repository.getInternalPath()),
933+ target_ref=Equals("refs/merge/%s/head" % mp.id),
934+ ))
935+ delete_operations = mp.getGitRefDeleteOperations()
936+ self.assertEqual([], delete_operations)
937+
938+ self.assertEqual(1, self.hosting_fixture.bulkSyncRefs.call_count)
939+ args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
940+ copy_ops, delete_ops = args
941+ self.assertEqual({}, kwargs)
942+ self.assertEqual([], delete_ops)
943+ self.assertEqual(1, len(copy_ops))
944+ self.assertThat(copy_ops[0], MatchesStructure(
945+ source_repo_path=Equals(
946+ mp.source_git_repository.getInternalPath()),
947+ source_ref=Equals(mp.source_git_commit_sha1),
948+ target_repo_path=Equals(
949+ mp.target_git_repository.getInternalPath()),
950+ target_ref=Equals("refs/merge/%s/head" % mp.id),
951+ ))
952+
953+ def test_getGitRefCopyOperations_private_source(self):
954+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
955+ login_admin()
956+ source_repo = self.factory.makeGitRepository(
957+ information_type=InformationType.PRIVATESECURITY)
958+ target_repo = self.factory.makeGitRepository(target=source_repo.target)
959+ [source] = self.factory.makeGitRefs(source_repo)
960+ [target] = self.factory.makeGitRefs(target_repo)
961+ mp = self.factory.makeBranchMergeProposalForGit(
962+ source_ref=source, target_ref=target)
963+ self.assertEqual([], mp.getGitRefCopyOperations())
964+
965+ delete_operations = mp.getGitRefDeleteOperations()
966+ self.assertEqual(1, len(delete_operations))
967+ self.assertThat(delete_operations[0], MatchesStructure(
968+ repo_path=Equals(mp.target_git_repository.getInternalPath()),
969+ ref=Equals("refs/merge/%s/head" % mp.id),
970+ ))
971+
972+ def test_getGitRefCopyOperations_private_source_same_repo(self):
973+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
974+ login_admin()
975+ repo = self.factory.makeGitRepository(
976+ information_type=InformationType.PRIVATESECURITY)
977+ [source, target] = self.factory.makeGitRefs(
978+ repo, ['refs/heads/bugfix', 'refs/heads/master'])
979+ mp = self.factory.makeBranchMergeProposalForGit(
980+ source_ref=source, target_ref=target)
981+ operations = mp.getGitRefCopyOperations()
982+ self.assertEqual(1, len(operations))
983+ self.assertThat(operations[0], MatchesStructure(
984+ source_repo_path=Equals(
985+ mp.source_git_repository.getInternalPath()),
986+ source_ref=Equals(mp.source_git_commit_sha1),
987+ target_repo_path=Equals(
988+ mp.target_git_repository.getInternalPath()),
989+ target_ref=Equals(GIT_MP_VIRTUAL_REF_FORMAT.format(mp=mp))
990+ ))
991+
992+ delete_operations = mp.getGitRefDeleteOperations()
993+ self.assertEqual([], delete_operations)
994+
995+ def test_getGitRefCopyOperations_private_source_same_owner(self):
996+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
997+ login_admin()
998+ source_repo = self.factory.makeGitRepository(
999+ information_type=InformationType.PRIVATESECURITY)
1000+ target_repo = self.factory.makeGitRepository(
1001+ target=source_repo.target, owner=source_repo.owner)
1002+ [source] = self.factory.makeGitRefs(source_repo)
1003+ [target] = self.factory.makeGitRefs(target_repo)
1004+ mp = self.factory.makeBranchMergeProposalForGit(
1005+ source_ref=source, target_ref=target)
1006+ operations = mp.getGitRefCopyOperations()
1007+ self.assertEqual(1, len(operations))
1008+ self.assertThat(operations[0], MatchesStructure(
1009+ source_repo_path=Equals(
1010+ mp.source_git_repository.getInternalPath()),
1011+ source_ref=Equals(mp.source_git_commit_sha1),
1012+ target_repo_path=Equals(
1013+ mp.target_git_repository.getInternalPath()),
1014+ target_ref=Equals(GIT_MP_VIRTUAL_REF_FORMAT.format(mp=mp))
1015+ ))
1016+
1017+ delete_operations = mp.getGitRefDeleteOperations()
1018+ self.assertEqual([], delete_operations)
1019+
1020+ def test_syncGitVirtualRefs(self):
1021+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
1022+ login_admin()
1023+ login_admin()
1024+ source_repo = self.factory.makeGitRepository()
1025+ target_repo = self.factory.makeGitRepository(target=source_repo.target)
1026+ [source] = self.factory.makeGitRefs(source_repo)
1027+ [target] = self.factory.makeGitRefs(target_repo)
1028+ mp = self.factory.makeBranchMergeProposalForGit(
1029+ source_ref=source, target_ref=target)
1030+
1031+ # mp.syncGitVirtualRefs should have been triggered by event.
1032+ # Check lp.code.subscribers.branchmergeproposal.merge_proposal_created.
1033+ self.assertEqual(1, self.hosting_fixture.bulkSyncRefs.call_count)
1034+ args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
1035+ copy_ops, delete_ops = args
1036+ self.assertEquals({}, kwargs)
1037+ self.assertEqual([], delete_ops)
1038+ self.assertEqual(1, len(copy_ops))
1039+ self.assertThat(copy_ops[0], MatchesStructure(
1040+ source_repo_path=Equals(
1041+ mp.source_git_repository.getInternalPath()),
1042+ source_ref=Equals(mp.source_git_commit_sha1),
1043+ target_repo_path=Equals(
1044+ mp.target_git_repository.getInternalPath()),
1045+ target_ref=Equals("refs/merge/%s/head" % mp.id),
1046+ ))
1047+
1048+ self.assertEqual(0, self.hosting_fixture.deleteRefs.call_count)
1049+
1050+ def test_syncGitVirtualRefs_private_source_deletes_ref(self):
1051+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
1052+ login_admin()
1053+ source_repo = self.factory.makeGitRepository(
1054+ information_type=InformationType.PRIVATESECURITY)
1055+ target_repo = self.factory.makeGitRepository(target=source_repo.target)
1056+ [source] = self.factory.makeGitRefs(source_repo)
1057+ [target] = self.factory.makeGitRefs(target_repo)
1058+ mp = self.factory.makeBranchMergeProposalForGit(
1059+ source_ref=source, target_ref=target)
1060+
1061+ # mp.syncGitVirtualRefs should have been triggered by event.
1062+ # Check lp.code.subscribers.branchmergeproposal.merge_proposal_created.
1063+ self.assertEqual(1, self.hosting_fixture.bulkSyncRefs.call_count)
1064+ args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
1065+ copy_ops, delete_ops = args
1066+ self.assertEqual({}, kwargs)
1067+ self.assertEqual([], copy_ops)
1068+ self.assertEqual(1, len(delete_ops))
1069+ self.assertThat(delete_ops[0], MatchesStructure(
1070+ repo_path=Equals(target_repo.getInternalPath()),
1071+ ref=Equals("refs/merge/%s/head" % mp.id)))
1072+
1073+
1074 class TestBranchMergeProposalTransitions(TestCaseWithFactory):
1075 """Test the state transitions of branch merge proposals."""
1076
1077@@ -1185,6 +1380,10 @@ class TestMergeProposalWebhooks(WithVCSScenarios, TestCaseWithFactory):
1078 def test_delete_triggers_webhooks(self):
1079 # When an existing merge proposal is deleted, any relevant webhooks
1080 # are triggered.
1081+ self.useFixture(FeatureFixture({
1082+ GIT_CREATE_MP_VIRTUAL_REF: "on",
1083+ BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG: "on"}))
1084+ hosting_fixture = self.useFixture(GitHostingFixture())
1085 logger = self.useFixture(FakeLogger())
1086 source = self.makeBranch()
1087 target = self.makeBranch(same_target_as=source)
1088@@ -1203,10 +1402,25 @@ class TestMergeProposalWebhooks(WithVCSScenarios, TestCaseWithFactory):
1089 expected_redacted_payload = dict(
1090 expected_payload,
1091 old=MatchesDict(self.getExpectedPayload(proposal, redact=True)))
1092+ hosting_fixture = self.useFixture(GitHostingFixture())
1093 proposal.deleteProposal()
1094 delivery = hook.deliveries.one()
1095+ self.assertIsNotNone(delivery)
1096 self.assertCorrectDelivery(expected_payload, hook, delivery)
1097 self.assertCorrectLogging(expected_redacted_payload, hook, logger)
1098+ if not self.git:
1099+ self.assertEqual(0, hosting_fixture.bulkSyncRefs.call_count)
1100+ else:
1101+ self.assertEqual(1, hosting_fixture.bulkSyncRefs.call_count)
1102+ args, kwargs = hosting_fixture.bulkSyncRefs.calls[0]
1103+ self.assertEqual({}, kwargs)
1104+ copy_ops, delete_ops = args
1105+ self.assertEqual(0, len(copy_ops))
1106+ self.assertEqual(1, len(delete_ops))
1107+ self.assertThat(delete_ops[0], MatchesStructure(
1108+ repo_path=Equals(target.repository.getInternalPath()),
1109+ ref=Equals("refs/merge/%s/head" % proposal.id)
1110+ ))
1111
1112
1113 class TestGetAddress(TestCaseWithFactory):
1114@@ -1507,6 +1721,26 @@ class TestBranchMergeProposalDeletion(TestCaseWithFactory):
1115 self.assertRaises(
1116 SQLObjectNotFound, BranchMergeProposalJob.get, job_id)
1117
1118+ def test_deleteProposal_for_git_removes_virtual_ref(self):
1119+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
1120+ self.useFixture(GitHostingFixture())
1121+ proposal = self.factory.makeBranchMergeProposalForGit()
1122+
1123+ # Clean up fixture calls.
1124+ hosting_fixture = self.useFixture(GitHostingFixture())
1125+ proposal.deleteProposal()
1126+
1127+ self.assertEqual(1, hosting_fixture.bulkSyncRefs.call_count)
1128+ args, kwargs = hosting_fixture.bulkSyncRefs.calls[0]
1129+ self.assertEqual({}, kwargs)
1130+ copy_ops, delete_ops = args
1131+ self.assertEqual([], copy_ops)
1132+ self.assertEqual(1, len(delete_ops))
1133+ self.assertThat(delete_ops[0], MatchesStructure(
1134+ repo_path=Equals(proposal.target_git_repository.getInternalPath()),
1135+ ref=Equals('refs/merge/%s/head' % proposal.id)
1136+ ))
1137+
1138
1139 class TestBranchMergeProposalBugs(WithVCSScenarios, TestCaseWithFactory):
1140
1141diff --git a/lib/lp/code/model/tests/test_gitcollection.py b/lib/lp/code/model/tests/test_gitcollection.py
1142index 447fcb4..372a633 100644
1143--- a/lib/lp/code/model/tests/test_gitcollection.py
1144+++ b/lib/lp/code/model/tests/test_gitcollection.py
1145@@ -599,7 +599,7 @@ class TestBranchMergeProposals(TestCaseWithFactory):
1146 registrant = self.factory.makePerson()
1147 mp1 = self.factory.makeBranchMergeProposalForGit(registrant=registrant)
1148 naked_repository = removeSecurityProxy(mp1.target_git_repository)
1149- naked_repository.transitionToInformationType(
1150+ naked_repository._transitionToInformationType(
1151 InformationType.USERDATA, registrant, verify_policy=False)
1152 collection = self.all_repositories.visibleByUser(None)
1153 proposals = collection.getMergeProposals()
1154diff --git a/lib/lp/code/model/tests/test_githosting.py b/lib/lp/code/model/tests/test_githosting.py
1155index 23d6063..fad0df3 100644
1156--- a/lib/lp/code/model/tests/test_githosting.py
1157+++ b/lib/lp/code/model/tests/test_githosting.py
1158@@ -42,7 +42,6 @@ from lp.code.errors import (
1159 GitRepositoryDeletionFault,
1160 GitRepositoryScanFault,
1161 GitTargetError,
1162- NoSuchGitReference,
1163 )
1164 from lp.code.interfaces.githosting import IGitHostingClient
1165 from lp.code.model.githosting import RefCopyOperation
1166@@ -414,8 +413,8 @@ class TestGitHostingClient(TestCase):
1167
1168 def getCopyRefOperations(self):
1169 return [
1170- RefCopyOperation("1a2b3c4", "999", "refs/merge/123"),
1171- RefCopyOperation("9a8b7c6", "666", "refs/merge/989"),
1172+ RefCopyOperation("1", "1a2b3c4", "999", "refs/merge/123"),
1173+ RefCopyOperation("1", "9a8b7c6", "666", "refs/merge/989"),
1174 ]
1175
1176 def test_copyRefs(self):
1177diff --git a/lib/lp/code/model/tests/test_gitjob.py b/lib/lp/code/model/tests/test_gitjob.py
1178index 3f606c0..69cb653 100644
1179--- a/lib/lp/code/model/tests/test_gitjob.py
1180+++ b/lib/lp/code/model/tests/test_gitjob.py
1181@@ -25,13 +25,16 @@ from testtools.matchers import (
1182 MatchesStructure,
1183 )
1184 import transaction
1185+from zope.component import getUtility
1186 from zope.interface import providedBy
1187 from zope.security.proxy import removeSecurityProxy
1188
1189+from lp.app.enums import InformationType
1190 from lp.code.adapters.gitrepository import GitRepositoryDelta
1191 from lp.code.enums import (
1192 GitGranteeType,
1193 GitObjectType,
1194+ GitRepositoryStatus,
1195 )
1196 from lp.code.interfaces.branchmergeproposal import (
1197 BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG,
1198@@ -39,6 +42,7 @@ from lp.code.interfaces.branchmergeproposal import (
1199 from lp.code.interfaces.gitjob import (
1200 IGitJob,
1201 IGitRefScanJob,
1202+ IGitRepositoryTransitionToInformationTypeJobSource,
1203 IReclaimGitRepositorySpaceJob,
1204 )
1205 from lp.code.model.gitjob import (
1206@@ -47,9 +51,11 @@ from lp.code.model.gitjob import (
1207 GitJobDerived,
1208 GitJobType,
1209 GitRefScanJob,
1210+ GitRepositoryTransitionToInformationTypeJob,
1211 ReclaimGitRepositorySpaceJob,
1212 )
1213 from lp.code.tests.helpers import GitHostingFixture
1214+from lp.registry.errors import CannotChangeInformationType
1215 from lp.services.config import config
1216 from lp.services.database.constants import UTC_NOW
1217 from lp.services.features.testing import FeatureFixture
1218@@ -362,6 +368,65 @@ class TestReclaimGitRepositorySpaceJob(TestCaseWithFactory):
1219 self.assertEqual([(path,)], hosting_fixture.delete.extract_args())
1220
1221
1222+class TestGitRepositoryTransitionInformationType(TestCaseWithFactory):
1223+
1224+ layer = ZopelessDatabaseLayer
1225+
1226+ def test_block_multiple_requests_to_change_info_type(self):
1227+ repo = self.factory.makeGitRepository()
1228+ repo.transitionToInformationType(
1229+ InformationType.PRIVATESECURITY, repo.owner)
1230+ self.assertEqual(
1231+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
1232+ repo.status)
1233+ expected_msg = (
1234+ "Cannot change privacy settings while git repository is "
1235+ "being changed.")
1236+ self.assertRaisesRegex(
1237+ CannotChangeInformationType, expected_msg,
1238+ repo.transitionToInformationType,
1239+ InformationType.PROPRIETARY, repo.owner)
1240+
1241+ def test_avoid_transitioning_while_creating(self):
1242+ repo = self.factory.makeGitRepository()
1243+ removeSecurityProxy(repo).status = GitRepositoryStatus.CREATING
1244+ expected_msg = (
1245+ "Cannot change privacy settings while git repository is "
1246+ "being changed.")
1247+ self.assertRaisesRegex(
1248+ CannotChangeInformationType, expected_msg,
1249+ repo.transitionToInformationType,
1250+ InformationType.PROPRIETARY, repo.owner)
1251+
1252+ def test_run_changes_info_type(self):
1253+ repo = self.factory.makeGitRepository(
1254+ information_type=InformationType.PUBLIC)
1255+ # Change to a private info type and with verify_policy, so we hit as
1256+ # many database tables as possible.
1257+ repo.transitionToInformationType(
1258+ InformationType.PRIVATESECURITY, repo.owner, verify_policy=True)
1259+ self.assertEqual(
1260+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
1261+ repo.status)
1262+
1263+ job_util = getUtility(
1264+ IGitRepositoryTransitionToInformationTypeJobSource)
1265+ jobs = list(job_util.iterReady())
1266+ self.assertEqual(1, len(jobs))
1267+ with dbuser(GitRepositoryTransitionToInformationTypeJob.config.dbuser):
1268+ JobRunner(jobs).runAll()
1269+
1270+ self.assertEqual(GitRepositoryStatus.AVAILABLE, repo.status)
1271+ self.assertEqual(
1272+ InformationType.PRIVATESECURITY, repo.information_type)
1273+
1274+ # After the job finished, another change is possible.
1275+ repo.transitionToInformationType(InformationType.PUBLIC, repo.owner)
1276+ self.assertEqual(
1277+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
1278+ repo.status)
1279+
1280+
1281 class TestDescribeRepositoryDelta(TestCaseWithFactory):
1282 """Tests for `describe_repository_delta`."""
1283
1284diff --git a/lib/lp/code/model/tests/test_gitrepository.py b/lib/lp/code/model/tests/test_gitrepository.py
1285index b83f9da..2296352 100644
1286--- a/lib/lp/code/model/tests/test_gitrepository.py
1287+++ b/lib/lp/code/model/tests/test_gitrepository.py
1288@@ -97,6 +97,7 @@ from lp.code.interfaces.gitnamespace import (
1289 IGitNamespaceSet,
1290 )
1291 from lp.code.interfaces.gitrepository import (
1292+ GIT_CREATE_MP_VIRTUAL_REF,
1293 IGitRepository,
1294 IGitRepositorySet,
1295 IGitRepositoryView,
1296@@ -783,6 +784,7 @@ class TestGitRepositoryDeletion(TestCaseWithFactory):
1297 # Make sure that the tests all flush the database changes.
1298 self.addCleanup(Store.of(self.repository).flush)
1299 login_person(self.user)
1300+ self.hosting_fixture = self.useFixture(GitHostingFixture())
1301
1302 def test_deletable(self):
1303 # A newly created repository can be deleted without any problems.
1304@@ -809,7 +811,7 @@ class TestGitRepositoryDeletion(TestCaseWithFactory):
1305
1306 def test_private_subscription_does_not_disable_deletion(self):
1307 # A private repository that has a subscription can be deleted.
1308- self.repository.transitionToInformationType(
1309+ removeSecurityProxy(self.repository)._transitionToInformationType(
1310 InformationType.USERDATA, self.repository.owner,
1311 verify_policy=False)
1312 self.repository.subscribe(
1313@@ -824,7 +826,6 @@ class TestGitRepositoryDeletion(TestCaseWithFactory):
1314
1315 def test_code_import_does_not_disable_deletion(self):
1316 # A repository that has an attached code import can be deleted.
1317- self.useFixture(GitHostingFixture())
1318 code_import = self.factory.makeCodeImport(
1319 target_rcs_type=TargetRevisionControlSystems.GIT)
1320 repository = code_import.git_repository
1321@@ -984,6 +985,7 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
1322 # unsubscribe the repository owner here.
1323 self.repository.unsubscribe(
1324 self.repository.owner, self.repository.owner)
1325+ self.hosting_fixture = self.useFixture(GitHostingFixture())
1326
1327 def test_plain_repository(self):
1328 # A fresh repository has no deletion requirements.
1329@@ -1115,7 +1117,6 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
1330
1331 def test_code_import_requirements(self):
1332 # Code imports are not included explicitly in deletion requirements.
1333- self.useFixture(GitHostingFixture())
1334 code_import = self.factory.makeCodeImport(
1335 target_rcs_type=TargetRevisionControlSystems.GIT)
1336 # Remove the implicit repository subscription first.
1337@@ -1126,7 +1127,6 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
1338
1339 def test_code_import_deletion(self):
1340 # break_references allows deleting a code import repository.
1341- self.useFixture(GitHostingFixture())
1342 code_import = self.factory.makeCodeImport(
1343 target_rcs_type=TargetRevisionControlSystems.GIT)
1344 code_import_id = code_import.id
1345@@ -1182,7 +1182,6 @@ class TestGitRepositoryDeletionConsequences(TestCaseWithFactory):
1346
1347 def test_DeleteCodeImport(self):
1348 # DeleteCodeImport.__call__ must delete the CodeImport.
1349- self.useFixture(GitHostingFixture())
1350 code_import = self.factory.makeCodeImport(
1351 target_rcs_type=TargetRevisionControlSystems.GIT)
1352 code_import_id = code_import.id
1353@@ -1521,6 +1520,10 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1354
1355 layer = DatabaseFunctionalLayer
1356
1357+ def setUp(self):
1358+ super(TestGitRepositoryRefs, self).setUp()
1359+ self.hosting_fixture = self.useFixture(GitHostingFixture())
1360+
1361 def test__convertRefInfo(self):
1362 # _convertRefInfo converts a valid info dictionary.
1363 sha1 = six.ensure_text(hashlib.sha1("").hexdigest())
1364@@ -1791,18 +1794,17 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1365 # planRefChanges excludes some ref prefixes by default, and can be
1366 # configured otherwise.
1367 repository = self.factory.makeGitRepository()
1368- hosting_fixture = self.useFixture(GitHostingFixture())
1369 repository.planRefChanges("dummy")
1370 self.assertEqual(
1371 [{"exclude_prefixes": ["refs/changes/"]}],
1372- hosting_fixture.getRefs.extract_kwargs())
1373- hosting_fixture.getRefs.calls = []
1374+ self.hosting_fixture.getRefs.extract_kwargs())
1375+ self.hosting_fixture.getRefs.calls = []
1376 self.pushConfig(
1377 "codehosting", git_exclude_ref_prefixes="refs/changes/ refs/pull/")
1378 repository.planRefChanges("dummy")
1379 self.assertEqual(
1380 [{"exclude_prefixes": ["refs/changes/", "refs/pull/"]}],
1381- hosting_fixture.getRefs.extract_kwargs())
1382+ self.hosting_fixture.getRefs.extract_kwargs())
1383
1384 def test_fetchRefCommits(self):
1385 # fetchRefCommits fetches detailed tip commit metadata for the
1386@@ -1875,9 +1877,8 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1387 def test_fetchRefCommits_empty(self):
1388 # If given an empty refs dictionary, fetchRefCommits returns early
1389 # without contacting the hosting service.
1390- hosting_fixture = self.useFixture(GitHostingFixture())
1391 GitRepository.fetchRefCommits("dummy", {})
1392- self.assertEqual([], hosting_fixture.getCommits.calls)
1393+ self.assertEqual([], self.hosting_fixture.getCommits.calls)
1394
1395 def test_synchroniseRefs(self):
1396 # synchroniseRefs copes with synchronising a repository where some
1397@@ -1920,7 +1921,6 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1398 self.assertThat(repository.refs, MatchesSetwise(*matchers))
1399
1400 def test_set_default_branch(self):
1401- hosting_fixture = self.useFixture(GitHostingFixture())
1402 repository = self.factory.makeGitRepository()
1403 self.factory.makeGitRefs(
1404 repository=repository,
1405@@ -1931,22 +1931,20 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1406 self.assertEqual(
1407 [((repository.getInternalPath(),),
1408 {"default_branch": "refs/heads/new"})],
1409- hosting_fixture.setProperties.calls)
1410+ self.hosting_fixture.setProperties.calls)
1411 self.assertEqual("refs/heads/new", repository.default_branch)
1412
1413 def test_set_default_branch_unchanged(self):
1414- hosting_fixture = self.useFixture(GitHostingFixture())
1415 repository = self.factory.makeGitRepository()
1416 self.factory.makeGitRefs(
1417 repository=repository, paths=["refs/heads/master"])
1418 removeSecurityProxy(repository)._default_branch = "refs/heads/master"
1419 with person_logged_in(repository.owner):
1420 repository.default_branch = "master"
1421- self.assertEqual([], hosting_fixture.setProperties.calls)
1422+ self.assertEqual([], self.hosting_fixture.setProperties.calls)
1423 self.assertEqual("refs/heads/master", repository.default_branch)
1424
1425 def test_set_default_branch_imported(self):
1426- hosting_fixture = self.useFixture(GitHostingFixture())
1427 repository = self.factory.makeGitRepository(
1428 repository_type=GitRepositoryType.IMPORTED)
1429 self.factory.makeGitRefs(
1430@@ -1959,7 +1957,7 @@ class TestGitRepositoryRefs(TestCaseWithFactory):
1431 "Cannot modify non-hosted Git repository %s." %
1432 repository.display_name,
1433 setattr, repository, "default_branch", "new")
1434- self.assertEqual([], hosting_fixture.setProperties.calls)
1435+ self.assertEqual([], self.hosting_fixture.setProperties.calls)
1436 self.assertEqual("refs/heads/master", repository.default_branch)
1437
1438 def test_exception_unset_default_branch(self):
1439@@ -2055,7 +2053,8 @@ class TestGitRepositoryModerate(TestCaseWithFactory):
1440 repository.transitionToInformationType(
1441 InformationType.PRIVATESECURITY, project.owner)
1442 self.assertEqual(
1443- InformationType.PRIVATESECURITY, repository.information_type)
1444+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
1445+ repository.status)
1446
1447 def test_attribute_smoketest(self):
1448 # Users with launchpad.Moderate can set attributes.
1449@@ -2580,18 +2579,66 @@ class TestGitRepositoryUpdateLandingTargets(TestCaseWithFactory):
1450
1451 layer = DatabaseFunctionalLayer
1452
1453- def test_schedules_diff_updates(self):
1454+ def setUp(self):
1455+ super(TestGitRepositoryUpdateLandingTargets, self).setUp()
1456+ self.hosting_fixture = self.useFixture(GitHostingFixture())
1457+
1458+ def assertSchedulesDiffUpdate(self, with_mp_virtual_ref):
1459 """Create jobs for all merge proposals."""
1460 bmp1 = self.factory.makeBranchMergeProposalForGit()
1461 bmp2 = self.factory.makeBranchMergeProposalForGit(
1462 source_ref=bmp1.source_git_ref)
1463- jobs = bmp1.source_git_repository.updateLandingTargets(
1464- [bmp1.source_git_path])
1465+
1466+ # Only enable this virtual ref here, since
1467+ # self.factory.makeBranchMergeProposalForGit also tries to create
1468+ # the virtual refs.
1469+ if with_mp_virtual_ref:
1470+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
1471+ else:
1472+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: ""}))
1473+ jobs, ref_copies, ref_deletes = (
1474+ bmp1.source_git_repository.updateLandingTargets(
1475+ [bmp1.source_git_path]))
1476 self.assertEqual(2, len(jobs))
1477 bmps_to_update = [
1478 removeSecurityProxy(job).branch_merge_proposal for job in jobs]
1479 self.assertContentEqual([bmp1, bmp2], bmps_to_update)
1480
1481+ if not with_mp_virtual_ref:
1482+ self.assertEqual(
1483+ 0, self.hosting_fixture.bulkSyncRefs.call_count)
1484+ else:
1485+ self.assertEqual(
1486+ 1, self.hosting_fixture.bulkSyncRefs.call_count)
1487+ args, kwargs = self.hosting_fixture.bulkSyncRefs.calls[0]
1488+ self.assertEqual({}, kwargs)
1489+ self.assertEqual(2, len(args))
1490+ copy_ops, delete_ops = args
1491+ self.assertEqual(2, len(copy_ops))
1492+ self.assertEqual(0, len(delete_ops))
1493+ self.assertThat(copy_ops[0], MatchesStructure(
1494+ source_repo_path=Equals(
1495+ bmp1.source_git_repository.getInternalPath()),
1496+ source_ref=Equals(bmp1.source_git_commit_sha1),
1497+ target_repo_path=Equals(
1498+ bmp1.target_git_repository.getInternalPath()),
1499+ target_ref=Equals("refs/merge/%s/head" % bmp1.id),
1500+ ))
1501+ self.assertThat(copy_ops[1], MatchesStructure(
1502+ source_repo_path=Equals(
1503+ bmp2.source_git_repository.getInternalPath()),
1504+ source_ref=Equals(bmp2.source_git_commit_sha1),
1505+ target_repo_path=Equals(
1506+ bmp2.target_git_repository.getInternalPath()),
1507+ target_ref=Equals("refs/merge/%s/head" % bmp2.id),
1508+ ))
1509+
1510+ def test_schedules_diff_updates_with_mp_virtual_ref(self):
1511+ self.assertSchedulesDiffUpdate(with_mp_virtual_ref=True)
1512+
1513+ def test_schedules_diff_updates_without_mp_virtual_ref(self):
1514+ self.assertSchedulesDiffUpdate(with_mp_virtual_ref=False)
1515+
1516 def test_ignores_final(self):
1517 """Diffs for proposals in final states aren't updated."""
1518 [source_ref] = self.factory.makeGitRefs()
1519@@ -2603,8 +2650,17 @@ class TestGitRepositoryUpdateLandingTargets(TestCaseWithFactory):
1520 for bmp in source_ref.landing_targets:
1521 if bmp.queue_status not in FINAL_STATES:
1522 removeSecurityProxy(bmp).deleteProposal()
1523- jobs = source_ref.repository.updateLandingTargets([source_ref.path])
1524+
1525+ # Enable the feature here, since factory.makeBranchMergeProposalForGit
1526+ # would also trigger the copy refs call.
1527+ self.useFixture(FeatureFixture({GIT_CREATE_MP_VIRTUAL_REF: "on"}))
1528+ jobs, ref_copies, ref_deletes = (
1529+ source_ref.repository.updateLandingTargets(
1530+ [source_ref.path]))
1531 self.assertEqual(0, len(jobs))
1532+ self.assertEqual(0, len(ref_copies))
1533+ self.assertEqual(0, len(ref_deletes))
1534+ self.assertEqual(0, self.hosting_fixture.copyRefs.call_count)
1535
1536
1537 class TestGitRepositoryMarkRecipesStale(TestCaseWithFactory):
1538@@ -3454,7 +3510,7 @@ class TestGitRepositorySet(TestCaseWithFactory):
1539 ]
1540 for repository, modified_date in zip(repositories, modified_dates):
1541 removeSecurityProxy(repository).date_last_modified = modified_date
1542- removeSecurityProxy(repositories[0]).transitionToInformationType(
1543+ removeSecurityProxy(repositories[0])._transitionToInformationType(
1544 InformationType.PRIVATESECURITY, repositories[0].registrant)
1545 self.assertEqual(
1546 [repositories[3], repositories[4], repositories[1],
1547@@ -3967,7 +4023,8 @@ class TestGitRepositoryWebservice(TestCaseWithFactory):
1548 self.assertEqual(209, response.status)
1549 with person_logged_in(ANONYMOUS):
1550 self.assertEqual(
1551- InformationType.PUBLICSECURITY, repository_db.information_type)
1552+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION,
1553+ repository_db.status)
1554
1555 def test_set_information_type_other_person(self):
1556 # An unrelated user cannot change the information type.
1557diff --git a/lib/lp/code/subscribers/branchmergeproposal.py b/lib/lp/code/subscribers/branchmergeproposal.py
1558index a214c8f..a34ca2a 100644
1559--- a/lib/lp/code/subscribers/branchmergeproposal.py
1560+++ b/lib/lp/code/subscribers/branchmergeproposal.py
1561@@ -1,4 +1,4 @@
1562-# Copyright 2010-2016 Canonical Ltd. This software is licensed under the
1563+# Copyright 2010-2020 Canonical Ltd. This software is licensed under the
1564 # GNU Affero General Public License version 3 (see the file LICENSE).
1565
1566 """Event subscribers for branch merge proposals."""
1567@@ -63,9 +63,13 @@ def _trigger_webhook(merge_proposal, payload):
1568 def merge_proposal_created(merge_proposal, event):
1569 """A new merge proposal has been created.
1570
1571- Create a job to update the diff for the merge proposal; trigger webhooks.
1572+ Create a job to update the diff for the merge proposal; trigger webhooks
1573+ and copy virtual refs as needed.
1574 """
1575 getUtility(IUpdatePreviewDiffJobSource).create(merge_proposal)
1576+
1577+ merge_proposal.syncGitVirtualRefs()
1578+
1579 if getFeatureFlag(BRANCH_MERGE_PROPOSAL_WEBHOOKS_FEATURE_FLAG):
1580 payload = {
1581 "action": "created",
1582@@ -149,3 +153,5 @@ def merge_proposal_deleted(merge_proposal, event):
1583 "old": _compose_merge_proposal_webhook_payload(merge_proposal),
1584 }
1585 _trigger_webhook(merge_proposal, payload)
1586+
1587+ merge_proposal.syncGitVirtualRefs(force_delete=True)
1588diff --git a/lib/lp/code/tests/helpers.py b/lib/lp/code/tests/helpers.py
1589index 4ef843d..3704b06 100644
1590--- a/lib/lp/code/tests/helpers.py
1591+++ b/lib/lp/code/tests/helpers.py
1592@@ -369,6 +369,9 @@ class GitHostingFixture(fixtures.Fixture):
1593 self.getBlob = FakeMethod(result=blob)
1594 self.delete = FakeMethod()
1595 self.disable_memcache = disable_memcache
1596+ self.copyRefs = FakeMethod()
1597+ self.deleteRefs = FakeMethod()
1598+ self.bulkSyncRefs = FakeMethod()
1599
1600 def _setUp(self):
1601 self.useFixture(ZopeUtilityFixture(self, IGitHostingClient))
1602diff --git a/lib/lp/code/xmlrpc/tests/test_git.py b/lib/lp/code/xmlrpc/tests/test_git.py
1603index dcbcee6..e884cd2 100644
1604--- a/lib/lp/code/xmlrpc/tests/test_git.py
1605+++ b/lib/lp/code/xmlrpc/tests/test_git.py
1606@@ -899,7 +899,7 @@ class TestGitAPI(TestGitAPIMixin, TestCaseWithFactory):
1607 for _ in range(2)]
1608 private_repository = code_imports[0].git_repository
1609 removeSecurityProxy(
1610- private_repository).transitionToInformationType(
1611+ private_repository)._transitionToInformationType(
1612 InformationType.PRIVATESECURITY, private_repository.owner)
1613 with celebrity_logged_in("vcs_imports"):
1614 jobs = [
1615@@ -1077,7 +1077,7 @@ class TestGitAPI(TestGitAPIMixin, TestCaseWithFactory):
1616 for _ in range(2)]
1617 private_repository = code_imports[0].git_repository
1618 removeSecurityProxy(
1619- private_repository).transitionToInformationType(
1620+ private_repository)._transitionToInformationType(
1621 InformationType.PRIVATESECURITY, private_repository.owner)
1622 with celebrity_logged_in("vcs_imports"):
1623 jobs = [
1624@@ -1687,7 +1687,7 @@ class TestGitAPI(TestGitAPIMixin, TestCaseWithFactory):
1625 target_rcs_type=TargetRevisionControlSystems.GIT)
1626 for _ in range(2)]
1627 private_repository = code_imports[0].git_repository
1628- removeSecurityProxy(private_repository).transitionToInformationType(
1629+ removeSecurityProxy(private_repository)._transitionToInformationType(
1630 InformationType.PRIVATESECURITY, private_repository.owner)
1631 with celebrity_logged_in("vcs_imports"):
1632 jobs = [
1633@@ -2012,7 +2012,7 @@ class TestGitAPI(TestGitAPIMixin, TestCaseWithFactory):
1634 target_rcs_type=TargetRevisionControlSystems.GIT)
1635 for _ in range(2)]
1636 private_repository = code_imports[0].git_repository
1637- removeSecurityProxy(private_repository).transitionToInformationType(
1638+ removeSecurityProxy(private_repository)._transitionToInformationType(
1639 InformationType.PRIVATESECURITY, private_repository.owner)
1640 with celebrity_logged_in("vcs_imports"):
1641 jobs = [
1642@@ -2268,7 +2268,7 @@ class TestGitAPI(TestGitAPIMixin, TestCaseWithFactory):
1643 target_rcs_type=TargetRevisionControlSystems.GIT)
1644 for _ in range(2)]
1645 private_repository = code_imports[0].git_repository
1646- removeSecurityProxy(private_repository).transitionToInformationType(
1647+ removeSecurityProxy(private_repository)._transitionToInformationType(
1648 InformationType.PRIVATESECURITY, private_repository.owner)
1649 with celebrity_logged_in("vcs_imports"):
1650 jobs = [
1651diff --git a/lib/lp/scripts/garbo.py b/lib/lp/scripts/garbo.py
1652index 6e0feac..ea248a4 100644
1653--- a/lib/lp/scripts/garbo.py
1654+++ b/lib/lp/scripts/garbo.py
1655@@ -38,6 +38,7 @@ from storm.expr import (
1656 Cast,
1657 In,
1658 Join,
1659+ LeftJoin,
1660 Max,
1661 Min,
1662 Or,
1663@@ -68,6 +69,10 @@ from lp.code.model.diff import (
1664 Diff,
1665 PreviewDiff,
1666 )
1667+from lp.code.model.gitjob import (
1668+ GitJob,
1669+ GitJobType,
1670+ )
1671 from lp.code.model.gitrepository import GitRepository
1672 from lp.code.model.revision import (
1673 RevisionAuthor,
1674@@ -1554,6 +1559,54 @@ class GitRepositoryPruner(TunableLoop):
1675 transaction.commit()
1676
1677
1678+class GitRepositoryBrokenInfoTypeTransition(TunableLoop):
1679+ """Put back to "AVAILABLE" repositories that are pending information
1680+ type changes, but we don't have any git job that will actually do that
1681+ in the upcoming future.
1682+ """
1683+
1684+ maximum_chunk_size = 500
1685+
1686+ def __init__(self, log, abort_time=None):
1687+ super(GitRepositoryBrokenInfoTypeTransition, self).__init__(
1688+ log, abort_time)
1689+ self.store = IMasterStore(GitRepository)
1690+
1691+ def findRepositories(self):
1692+ pending_change = (
1693+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION)
1694+ job_type = GitJobType.REPOSITORY_TRANSITION_TO_INFO_TYPE
1695+ job_pending_statuses = (JobStatus.WAITING, JobStatus.RUNNING)
1696+ # Get git repositories left-joining with
1697+ # REPOSITORY_TRANSITION_TO_INFO_TYPE GitJobs waiting to be run (or
1698+ # already running).
1699+ join = [
1700+ GitRepository,
1701+ LeftJoin(
1702+ GitJob,
1703+ And(GitJob.repository_id == GitRepository.id,
1704+ GitJob.job_type == job_type)),
1705+ LeftJoin(
1706+ Job,
1707+ And(GitJob.job_id == Job.id,
1708+ Job._status.is_in(job_pending_statuses)))]
1709+ # We get only the repositories pending change without associated job.
1710+ result_set = self.store.using(*join).find(
1711+ GitRepository,
1712+ GitRepository.status == pending_change,
1713+ Job._status == None)
1714+ return result_set.order_by(GitRepository.date_created)
1715+
1716+ def isDone(self):
1717+ return self.findRepositories().is_empty()
1718+
1719+ def __call__(self, chunk_size):
1720+ repositories = self.findRepositories()[:chunk_size]
1721+ for repository in repositories:
1722+ repository.status = GitRepositoryStatus.AVAILABLE
1723+ transaction.commit()
1724+
1725+
1726 class BaseDatabaseGarbageCollector(LaunchpadCronScript):
1727 """Abstract base class to run a collection of TunableLoops."""
1728 script_name = None # Script name for locking and database user. Override.
1729@@ -1806,6 +1859,7 @@ class HourlyDatabaseGarbageCollector(BaseDatabaseGarbageCollector):
1730 BugHeatUpdater,
1731 DuplicateSessionPruner,
1732 GitRepositoryPruner,
1733+ GitRepositoryBrokenInfoTypeTransition,
1734 RevisionCachePruner,
1735 UnusedSessionPruner,
1736 ]
1737diff --git a/lib/lp/scripts/tests/test_garbo.py b/lib/lp/scripts/tests/test_garbo.py
1738index 2fbc80d..ae3b0f1 100644
1739--- a/lib/lp/scripts/tests/test_garbo.py
1740+++ b/lib/lp/scripts/tests/test_garbo.py
1741@@ -1126,6 +1126,66 @@ class TestGarbo(FakeAdapterMixin, TestCaseWithFactory):
1742 {old_available, recent_available, recent_creating},
1743 set(remaining_repos))
1744
1745+ def test_GitRepositoryBrokenInfoTypeTransition_changes_status(self):
1746+ self.useFixture(FeatureFixture({OCI_RECIPE_ALLOW_CREATE: 'on'}))
1747+ # Shortcuts.
1748+ available = GitRepositoryStatus.AVAILABLE
1749+ pending_transition = (
1750+ GitRepositoryStatus.PENDING_INFORMATION_TYPE_TRANSITION)
1751+
1752+ switch_dbuser('testadmin')
1753+ store = IMasterStore(GitRepository)
1754+ created_repos = [self.factory.makeGitRepository() for i in range(5)]
1755+
1756+ pending_without_job_repos = []
1757+ for i in range(2):
1758+ repo = self.factory.makeGitRepository()
1759+ removeSecurityProxy(repo)._status = pending_transition
1760+
1761+ pending_with_failed_jobs = []
1762+ for i in range(3):
1763+ repo = self.factory.makeGitRepository()
1764+ # For some repos, create the job but force them to fail
1765+ job = repo.transitionToInformationType(
1766+ InformationType.PRIVATESECURITY, repo.owner)
1767+ job.start()
1768+ job.fail()
1769+ pending_with_failed_jobs.append(repo)
1770+
1771+ pending_with_started_job_repos = []
1772+ for i in range(2):
1773+ repo = self.factory.makeGitRepository()
1774+ job = repo.transitionToInformationType(
1775+ InformationType.PRIVATESECURITY, repo.owner)
1776+ job.start()
1777+ pending_with_started_job_repos.append(repo)
1778+
1779+ pending_with_waiting_jobs = []
1780+ for i in range(3):
1781+ repo = self.factory.makeGitRepository()
1782+ repo.transitionToInformationType(
1783+ InformationType.PRIVATESECURITY, repo.owner)
1784+ pending_with_started_job_repos.append(repo)
1785+
1786+ self.assertEqual(15, store.find(GitRepository).count())
1787+
1788+ self.runHourly(maximum_chunk_size=2)
1789+
1790+ switch_dbuser('testadmin')
1791+ self.assertEqual(15, store.find(GitRepository).count())
1792+ self.assertTrue(
1793+ all(i.status == available for i in created_repos))
1794+ self.assertTrue(
1795+ all(i.status == available for i in pending_without_job_repos))
1796+ self.assertTrue(
1797+ all(i.status == available for i in pending_with_failed_jobs))
1798+ self.assertTrue(
1799+ all(i.status == pending_transition
1800+ for i in pending_with_started_job_repos))
1801+ self.assertTrue(
1802+ all(i.status == pending_transition
1803+ for i in pending_with_waiting_jobs))
1804+
1805 def test_WebhookJobPruner(self):
1806 # Garbo should remove jobs completed over 30 days ago.
1807 switch_dbuser('testadmin')
1808diff --git a/lib/lp/services/config/schema-lazr.conf b/lib/lp/services/config/schema-lazr.conf
1809index aaa9d30..1a51d27 100644
1810--- a/lib/lp/services/config/schema-lazr.conf
1811+++ b/lib/lp/services/config/schema-lazr.conf
1812@@ -1768,6 +1768,7 @@ job_sources:
1813 ICommercialExpiredJobSource,
1814 IExpiringMembershipNotificationJobSource,
1815 IGitRepositoryModifiedMailJobSource,
1816+ IGitRepositoryTransitionToInformationTypeJobSource,
1817 IMembershipNotificationJobSource,
1818 IOCIRecipeRequestBuildsJobSource,
1819 IOCIRegistryUploadJobSource,
1820@@ -1829,6 +1830,11 @@ module: lp.code.interfaces.gitjob
1821 dbuser: send-branch-mail
1822 crontab_group: MAIN
1823
1824+[IGitRepositoryTransitionToInformationTypeJobSource]
1825+module: lp.code.interfaces.gitjob
1826+dbuser: privacy-change-jobs
1827+crontab_group: MAIN
1828+
1829 [IInitializeDistroSeriesJobSource]
1830 module: lp.soyuz.interfaces.distributionjob
1831 dbuser: initializedistroseries
1832diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
1833index 57babc5..9d1f53b 100644
1834--- a/lib/lp/testing/factory.py
1835+++ b/lib/lp/testing/factory.py
1836@@ -1815,7 +1815,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
1837 reviewer=reviewer, **optional_repository_args)
1838 naked_repository = removeSecurityProxy(repository)
1839 if information_type is not None:
1840- naked_repository.transitionToInformationType(
1841+ naked_repository._transitionToInformationType(
1842 information_type, registrant, verify_policy=False)
1843 return repository
1844