Launchpad itself

Merge ~twom/launchpad:offline-oci-builds into launchpad:master

  1. Git
  2. lp:~twom/launchpad
  3. offline-oci-builds
  4. Merge into master
Proposed by Tom Wardill
Status: Merged
Approved by: Tom Wardill
Approved revision: 9a39372794474be26e72babbf1333d2334e6c8dc
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~twom/launchpad:offline-oci-builds
Merge into: launchpad:master
Diff against target: 290 lines (+53/-17)
11 files modified
lib/lp/app/stories/basics/xx-pagetest-logging.txt (+1/-1)
lib/lp/oci/browser/ocirecipe.py (+2/-1)
lib/lp/oci/browser/tests/test_ocirecipe.py (+3/-0)
lib/lp/oci/interfaces/ocirecipe.py (+9/-1)
lib/lp/oci/model/ocirecipe.py (+8/-3)
lib/lp/oci/model/ocirecipebuildbehaviour.py (+1/-1)
lib/lp/oci/tests/test_ocirecipe.py (+1/-0)
lib/lp/oci/tests/test_ocirecipebuildbehaviour.py (+17/-5)
lib/lp/testing/factory.py (+6/-4)
lib/lp/testing/layers.py (+4/-1)
lib/lp/testing/pages.py (+1/-0)
Reviewer Review Type Date Requested Status
Colin Watson (community) Approve
Review via email: mp+387376@code.launchpad.net

Commit message

Add allow_internet to OCIRecipe

Description of the change

Enable control over whether OCIRecipe builds should have access to the internet.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) :
review: Approve
Revision history for this message
Colin Watson (cjwatson) wrote :

Also, just for the record, this of course can't land until the corresponding DB patch is on production.

Revision history for this message
Otto Co-Pilot (otto-copilot) wrote :

Merging failed
https://jenkins.ols.canonical.com/online-services/job/launchpad/627/

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lib/lp/app/stories/basics/xx-pagetest-logging.txt b/lib/lp/app/stories/basics/xx-pagetest-logging.txt
2index c140d45..b700a8b 100644
3--- a/lib/lp/app/stories/basics/xx-pagetest-logging.txt
4+++ b/lib/lp/app/stories/basics/xx-pagetest-logging.txt
5@@ -5,7 +5,7 @@ pagetests-access.log file.
6
7 >>> browser.open('http://launchpad.test/')
8
9- >>> log = open('logs/pagetests-access.log').read()
10+ >>> log = open(page_log_location).read()
11 >>> print(log.strip().split('\n')[-1])
12 127.0.0.88 - ... "launchpad.test" [...] "GET / HTTP/1.1" 200 ...
13 "Anonymous" "RootObject:index.html" "" "Python-urllib/..."
14diff --git a/lib/lp/oci/browser/ocirecipe.py b/lib/lp/oci/browser/ocirecipe.py
15index 2fe41af..e18d990 100644
16--- a/lib/lp/oci/browser/ocirecipe.py
17+++ b/lib/lp/oci/browser/ocirecipe.py
18@@ -499,6 +499,7 @@ class IOCIRecipeEditSchema(Interface):
19 "build_file",
20 "build_daily",
21 "require_virtualized",
22+ "allow_internet",
23 "push_rules",
24 ])
25
26@@ -612,7 +613,7 @@ class OCIRecipeAdminView(BaseOCIRecipeEditView):
27
28 page_title = "Administer"
29
30- field_names = ("require_virtualized",)
31+ field_names = ("require_virtualized", "allow_internet")
32
33
34 class OCIRecipeEditView(BaseOCIRecipeEditView, EnableProcessorsMixin):
35diff --git a/lib/lp/oci/browser/tests/test_ocirecipe.py b/lib/lp/oci/browser/tests/test_ocirecipe.py
36index 0d4923c..f85df0f 100644
37--- a/lib/lp/oci/browser/tests/test_ocirecipe.py
38+++ b/lib/lp/oci/browser/tests/test_ocirecipe.py
39@@ -295,14 +295,17 @@ class TestOCIRecipeAdminView(BaseTestOCIRecipeView):
40 login_person(self.person)
41 recipe = self.factory.makeOCIRecipe(registrant=self.person)
42 self.assertTrue(recipe.require_virtualized)
43+ self.assertTrue(recipe.allow_internet)
44
45 browser = self.getViewBrowser(recipe, user=commercial_admin)
46 browser.getLink("Administer OCI recipe").click()
47 browser.getControl("Require virtualized builders").selected = False
48+ browser.getControl("Allow external network access").selected = False
49 browser.getControl("Update OCI recipe").click()
50
51 login_person(self.person)
52 self.assertFalse(recipe.require_virtualized)
53+ self.assertFalse(recipe.allow_internet)
54
55 def test_admin_recipe_sets_date_last_modified(self):
56 # Administering an OCI recipe sets the date_last_modified property.
57diff --git a/lib/lp/oci/interfaces/ocirecipe.py b/lib/lp/oci/interfaces/ocirecipe.py
58index 64c5387..37f74a2 100644
59--- a/lib/lp/oci/interfaces/ocirecipe.py
60+++ b/lib/lp/oci/interfaces/ocirecipe.py
61@@ -408,6 +408,13 @@ class IOCIRecipeAdminAttributes(Interface):
62 value_type=Reference(schema=IProcessor),
63 readonly=False))
64
65+ allow_internet = exported(Bool(
66+ title=_("Allow external network access"),
67+ required=True, readonly=False,
68+ description=_(
69+ "Allow access to external network resources via a proxy. "
70+ "Resources hosted on Launchpad itself are always allowed.")))
71+
72
73 @exported_as_webservice_entry(
74 publish_web_link=True, as_of="devel", singular_name="oci_recipe")
75@@ -421,7 +428,8 @@ class IOCIRecipeSet(Interface):
76
77 def new(name, registrant, owner, oci_project, git_ref, build_file,
78 description=None, official=False, require_virtualized=True,
79- build_daily=False, processors=None, date_created=DEFAULT):
80+ build_daily=False, processors=None, date_created=DEFAULT,
81+ allow_internet=True):
82 """Create an IOCIRecipe."""
83
84 def exists(owner, oci_project, name):
85diff --git a/lib/lp/oci/model/ocirecipe.py b/lib/lp/oci/model/ocirecipe.py
86index 38e5a9a..45691c5 100644
87--- a/lib/lp/oci/model/ocirecipe.py
88+++ b/lib/lp/oci/model/ocirecipe.py
89@@ -145,11 +145,14 @@ class OCIRecipe(Storm, WebhookTargetMixin):
90 require_virtualized = Bool(name="require_virtualized", default=True,
91 allow_none=False)
92
93+ allow_internet = Bool(name='allow_internet', allow_none=False)
94+
95 build_daily = Bool(name="build_daily", default=False)
96
97 def __init__(self, name, registrant, owner, oci_project, git_ref,
98 description=None, official=False, require_virtualized=True,
99- build_file=None, build_daily=False, date_created=DEFAULT):
100+ build_file=None, build_daily=False, date_created=DEFAULT,
101+ allow_internet=True):
102 if not getFeatureFlag(OCI_RECIPE_ALLOW_CREATE):
103 raise OCIRecipeFeatureDisabled()
104 super(OCIRecipe, self).__init__()
105@@ -165,6 +168,7 @@ class OCIRecipe(Storm, WebhookTargetMixin):
106 self.date_created = date_created
107 self.date_last_modified = date_created
108 self.git_ref = git_ref
109+ self.allow_internet = allow_internet
110
111 def __repr__(self):
112 return "<OCIRecipe ~%s/%s/+oci/%s/+recipe/%s>" % (
113@@ -502,7 +506,8 @@ class OCIRecipeSet:
114
115 def new(self, name, registrant, owner, oci_project, git_ref, build_file,
116 description=None, official=False, require_virtualized=True,
117- build_daily=False, processors=None, date_created=DEFAULT):
118+ build_daily=False, processors=None, date_created=DEFAULT,
119+ allow_internet=True):
120 """See `IOCIRecipeSet`."""
121 if not registrant.inTeam(owner):
122 if owner.is_team:
123@@ -524,7 +529,7 @@ class OCIRecipeSet:
124 oci_recipe = OCIRecipe(
125 name, registrant, owner, oci_project, git_ref, description,
126 official, require_virtualized, build_file, build_daily,
127- date_created)
128+ date_created, allow_internet)
129 store.add(oci_recipe)
130
131 if processors is None:
132diff --git a/lib/lp/oci/model/ocirecipebuildbehaviour.py b/lib/lp/oci/model/ocirecipebuildbehaviour.py
133index 8b5f5e0..fce6dd3 100644
134--- a/lib/lp/oci/model/ocirecipebuildbehaviour.py
135+++ b/lib/lp/oci/model/ocirecipebuildbehaviour.py
136@@ -85,7 +85,7 @@ class OCIRecipeBuildBehaviour(SnapProxyMixin, BuildFarmJobBehaviourBase):
137 build = self.build
138 args = yield super(OCIRecipeBuildBehaviour, self).extraBuildArgs(
139 logger=logger)
140- yield self.addProxyArgs(args)
141+ yield self.addProxyArgs(args, build.recipe.allow_internet)
142 # XXX twom 2020-02-17 This may need to be more complex, and involve
143 # distribution name.
144 args["name"] = build.recipe.name
145diff --git a/lib/lp/oci/tests/test_ocirecipe.py b/lib/lp/oci/tests/test_ocirecipe.py
146index 1d07913..880c351 100644
147--- a/lib/lp/oci/tests/test_ocirecipe.py
148+++ b/lib/lp/oci/tests/test_ocirecipe.py
149@@ -806,6 +806,7 @@ class TestOCIRecipeSet(TestCaseWithFactory):
150 self.assertEqual(target.official, False)
151 self.assertEqual(target.require_virtualized, False)
152 self.assertEqual(target.git_ref, git_ref)
153+ self.assertTrue(target.allow_internet)
154
155 def test_already_exists(self):
156 owner = self.factory.makePerson()
157diff --git a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
158index d30254e..10e0139 100644
159--- a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
160+++ b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
161@@ -106,13 +106,14 @@ class MakeOCIBuildMixin:
162 build.queueBuild()
163 return build
164
165- def makeJob(self, git_ref, recipe=None, build=None):
166+ def makeJob(self, git_ref, recipe=None, build=None, **kwargs):
167 """Create a sample `IOCIRecipeBuildBehaviour`."""
168 if build is None:
169 if recipe is None:
170- build = self.factory.makeOCIRecipeBuild()
171+ build = self.factory.makeOCIRecipeBuild(**kwargs)
172 else:
173- build = self.factory.makeOCIRecipeBuild(recipe=recipe)
174+ build = self.factory.makeOCIRecipeBuild(
175+ recipe=recipe, **kwargs)
176 build.recipe.git_ref = git_ref
177
178 job = IBuildFarmJobBehaviour(build)
179@@ -330,7 +331,7 @@ class TestAsyncOCIRecipeBuildBehaviour(MakeOCIBuildMixin, TestCaseWithFactory):
180 def test_dispatchBuildToSlave_prefers_lxd(self):
181 self.pushConfig("snappy", builder_proxy_host=None)
182 [ref] = self.factory.makeGitRefs()
183- job = self.makeJob(git_ref=ref)
184+ job = self.makeJob(git_ref=ref, allow_internet=False)
185 builder = MockBuilder()
186 builder.processor = job.build.processor
187 slave = OkSlave()
188@@ -349,7 +350,7 @@ class TestAsyncOCIRecipeBuildBehaviour(MakeOCIBuildMixin, TestCaseWithFactory):
189 def test_dispatchBuildToSlave_falls_back_to_chroot(self):
190 self.pushConfig("snappy", builder_proxy_host=None)
191 [ref] = self.factory.makeGitRefs()
192- job = self.makeJob(git_ref=ref)
193+ job = self.makeJob(git_ref=ref, allow_internet=False)
194 builder = MockBuilder()
195 builder.processor = job.build.processor
196 slave = OkSlave()
197@@ -401,6 +402,17 @@ class TestAsyncOCIRecipeBuildBehaviour(MakeOCIBuildMixin, TestCaseWithFactory):
198 # for Distro series
199 self.assertEqual(distroseries.name, slave.call_log[1][5]['series'])
200
201+ @defer.inlineCallbacks
202+ def test_extraBuildArgs_disallow_internet(self):
203+ # If external network access is not allowed for the OCI Recipe,
204+ # extraBuildArgs does not dispatch a proxy token.
205+ [ref] = self.factory.makeGitRefs()
206+ job = self.makeJob(git_ref=ref, allow_internet=False)
207+ with dbuser(config.builddmaster.dbuser):
208+ args = yield job.extraBuildArgs()
209+ self.assertNotIn("proxy_url", args)
210+ self.assertNotIn("revocation_endpoint", args)
211+
212
213 class TestHandleStatusForOCIRecipeBuild(WithScenarios,
214 MakeOCIBuildMixin,
215diff --git a/lib/lp/testing/factory.py b/lib/lp/testing/factory.py
216index b57632d..6c6c9cb 100644
217--- a/lib/lp/testing/factory.py
218+++ b/lib/lp/testing/factory.py
219@@ -4987,7 +4987,8 @@ class BareLaunchpadObjectFactory(ObjectFactory):
220 def makeOCIRecipe(self, name=None, registrant=None, owner=None,
221 oci_project=None, git_ref=None, description=None,
222 official=False, require_virtualized=True,
223- build_file=None, date_created=DEFAULT):
224+ build_file=None, date_created=DEFAULT,
225+ allow_internet=True):
226 """Make a new OCIRecipe."""
227 if name is None:
228 name = self.getUniqueString(u"oci-recipe-name")
229@@ -5013,7 +5014,8 @@ class BareLaunchpadObjectFactory(ObjectFactory):
230 description=description,
231 official=official,
232 require_virtualized=require_virtualized,
233- date_created=date_created)
234+ date_created=date_created,
235+ allow_internet=allow_internet)
236
237 def makeOCIRecipeArch(self, recipe=None, processor=None):
238 """Make a new OCIRecipeArch."""
239@@ -5026,7 +5028,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
240 def makeOCIRecipeBuild(self, requester=None, registrant=None, recipe=None,
241 distro_arch_series=None, date_created=DEFAULT,
242 status=BuildStatus.NEEDSBUILD, builder=None,
243- duration=None):
244+ duration=None, **kwargs):
245 """Make a new OCIRecipeBuild."""
246 if requester is None:
247 requester = self.makePerson()
248@@ -5047,7 +5049,7 @@ class BareLaunchpadObjectFactory(ObjectFactory):
249 if registrant is None:
250 registrant = requester
251 recipe = self.makeOCIRecipe(
252- registrant=registrant, oci_project=oci_project)
253+ registrant=registrant, oci_project=oci_project, **kwargs)
254 oci_build = getUtility(IOCIRecipeBuildSet).new(
255 requester, recipe, distro_arch_series, date_created)
256 if duration is not None:
257diff --git a/lib/lp/testing/layers.py b/lib/lp/testing/layers.py
258index 6cf747c..8c9241c 100644
259--- a/lib/lp/testing/layers.py
260+++ b/lib/lp/testing/layers.py
261@@ -1649,6 +1649,8 @@ class PageTestLayer(LaunchpadFunctionalLayer, BingServiceLayer):
262 """Environment for page tests.
263 """
264
265+ log_location = None
266+
267 @classmethod
268 @profiled
269 def setUp(cls):
270@@ -1656,7 +1658,8 @@ class PageTestLayer(LaunchpadFunctionalLayer, BingServiceLayer):
271 PageTestLayer.profiler = Profile()
272 else:
273 PageTestLayer.profiler = None
274- file_handler = logging.FileHandler('logs/pagetests-access.log', 'w')
275+ cls.log_location = tempfile.NamedTemporaryFile().name
276+ file_handler = logging.FileHandler(cls.log_location, 'w')
277 file_handler.setFormatter(logging.Formatter())
278 logger = PythonLogger('pagetests-access')
279 logger.logger.addHandler(file_handler)
280diff --git a/lib/lp/testing/pages.py b/lib/lp/testing/pages.py
281index 77a4059..949dae6 100644
282--- a/lib/lp/testing/pages.py
283+++ b/lib/lp/testing/pages.py
284@@ -903,6 +903,7 @@ def setUpGlobs(test, future=False):
285 test.globs['print_self_link_of_entries'] = print_self_link_of_entries
286 test.globs['print_tag_with_id'] = print_tag_with_id
287 test.globs['PageTestLayer'] = PageTestLayer
288+ test.globs['page_log_location'] = PageTestLayer.log_location
289 test.globs['stop'] = stop
290 test.globs['six'] = six
291

Subscribers

People subscribed via source and target branches

to status/vote changes: