Merge launchpad:master into launchpad:db-devel

Proposed by Colin Watson
Status: Merged
Approved by: Colin Watson
Approved revision: 8d28b7c56f3ac7f0d39ec1c263268ffb0b42c676
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: launchpad:master
Merge into: launchpad:db-devel
Diff against target: 1621 lines (+273/-801)
18 files modified
dev/null (+0/-139)
lib/lp/archivepublisher/publishing.py (+3/-28)
lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py (+11/-206)
lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py (+1/-351)
lib/lp/archivepublisher/tests/test_publisher.py (+0/-39)
lib/lp/oci/model/ocirecipebuildjob.py (+8/-4)
lib/lp/oci/tests/test_ocirecipebuildjob.py (+5/-10)
lib/lp/registry/browser/distribution.py (+0/-3)
lib/lp/registry/interfaces/distribution.py (+42/-1)
lib/lp/registry/model/distribution.py (+28/-0)
lib/lp/registry/scripts/closeaccount.py (+3/-6)
lib/lp/registry/tests/test_distribution.py (+91/-1)
lib/lp/registry/tests/test_personmerge.py (+2/-2)
lib/lp/services/scripts/base.py (+0/-4)
lib/lp/snappy/model/snap.py (+0/-7)
lib/lp/soyuz/scripts/expire_archive_files.py (+3/-0)
lib/lp/testing/layers.py (+18/-0)
utilities/manage-celery-workers.sh (+58/-0)
Reviewer Review Type Date Requested Status
Colin Watson Approve
Review via email: mp+402439@code.launchpad.net

Commit message

Manually merge from master to fix test failure on Python 2

Description of the change

The bug fixed in https://code.launchpad.net/~twom/launchpad/+git/launchpad/+merge/402340 managed to get into db-devel, so db-devel can't pass tests until this is merged.

To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/lib/lp/archivepublisher/htaccess.py b/lib/lp/archivepublisher/htaccess.py
0deleted file mode 1006440deleted file mode 100644
index 613cfde..0000000
--- a/lib/lp/archivepublisher/htaccess.py
+++ /dev/null
@@ -1,124 +0,0 @@
1#!/usr/bin/python2
2#
3# Copyright 2010-2017 Canonical Ltd. This software is licensed under the
4# GNU Affero General Public License version 3 (see the file LICENSE).
5
6"""Writing of htaccess and htpasswd files."""
7
8__metaclass__ = type
9
10__all__ = [
11 'htpasswd_credentials_for_archive',
12 'write_htaccess',
13 'write_htpasswd',
14 ]
15
16import base64
17import crypt
18import os
19
20from lp.registry.model.person import Person
21from lp.services.database.interfaces import IStore
22from lp.soyuz.model.archiveauthtoken import ArchiveAuthToken
23
24
25HTACCESS_TEMPLATE = """
26AuthType Basic
27AuthName "Token Required"
28AuthUserFile %(path)s/.htpasswd
29Require valid-user
30"""
31
32BUILDD_USER_NAME = "buildd"
33
34
35def write_htaccess(htaccess_filename, distroot):
36 """Write a htaccess file for a private archive.
37
38 :param htaccess_filename: Filename of the htaccess file.
39 :param distroot: Archive root path
40 """
41 interpolations = {"path": distroot}
42 file = open(htaccess_filename, "w")
43 try:
44 file.write(HTACCESS_TEMPLATE % interpolations)
45 finally:
46 file.close()
47
48
49def write_htpasswd(filename, users):
50 """Write out a new htpasswd file.
51
52 :param filename: The file to create.
53 :param users: Iterable over (user, password, salt) tuples.
54 """
55 if os.path.isfile(filename):
56 os.remove(filename)
57
58 file = open(filename, "a")
59 try:
60 for user, password, salt in users:
61 encrypted = crypt.crypt(password, salt)
62 file.write("%s:%s\n" % (user, encrypted))
63 finally:
64 file.close()
65
66
67# XXX cjwatson 2017-10-09: This whole mechanism of writing password files to
68# disk (as opposed to e.g. using a WSGI authentication provider that checks
69# passwords against the database) is terrible, but as long as we're using it
70# we should use something like bcrypt rather than DES-based crypt.
71def make_salt(s):
72 """Produce a salt from an input string.
73
74 This ensures that salts are drawn from the correct alphabet
75 ([./a-zA-Z0-9]).
76 """
77 # As long as the input string is at least one character long, there will
78 # be no padding within the first two characters.
79 return base64.b64encode(
80 (s or " ").encode("UTF-8"), altchars=b"./")[:2].decode("ASCII")
81
82
83def htpasswd_credentials_for_archive(archive):
84 """Return credentials for an archive for use with write_htpasswd.
85
86 :param archive: An `IArchive` (must be private)
87 :return: Iterable of tuples with (user, password, salt) for use with
88 write_htpasswd.
89 """
90 assert archive.private, "Archive %r must be private" % archive
91
92 tokens = IStore(ArchiveAuthToken).find(
93 (ArchiveAuthToken.person_id, ArchiveAuthToken.name,
94 ArchiveAuthToken.token),
95 ArchiveAuthToken.archive == archive,
96 ArchiveAuthToken.date_deactivated == None)
97 # We iterate tokens more than once - materialise it.
98 tokens = list(tokens)
99
100 # Preload map with person ID to person name.
101 person_ids = {token[0] for token in tokens}
102 names = dict(
103 IStore(Person).find(
104 (Person.id, Person.name), Person.id.is_in(person_ids)))
105
106 # Format the user field by combining the token list with the person list
107 # (when token has person_id) or prepending a '+' (for named tokens).
108 output = []
109 for person_id, token_name, token in tokens:
110 if token_name:
111 # A named auth token.
112 output.append(('+' + token_name, token, make_salt(token_name)))
113 else:
114 # A subscription auth token.
115 output.append(
116 (names[person_id], token, make_salt(names[person_id])))
117
118 # The first .htpasswd entry is the buildd_secret.
119 yield (BUILDD_USER_NAME, archive.buildd_secret, BUILDD_USER_NAME[:2])
120
121 # Iterate over tokens and write the appropriate htpasswd entries for them.
122 # Sort by name/person ID so the file can be compared later.
123 for user, password, salt in sorted(output):
124 yield (user, password, salt)
diff --git a/lib/lp/archivepublisher/publishing.py b/lib/lp/archivepublisher/publishing.py
index 55614f9..b87c7ce 100644
--- a/lib/lp/archivepublisher/publishing.py
+++ b/lib/lp/archivepublisher/publishing.py
@@ -50,17 +50,14 @@ from lp.archivepublisher import HARDCODED_COMPONENT_ORDER
50from lp.archivepublisher.config import getPubConfig50from lp.archivepublisher.config import getPubConfig
51from lp.archivepublisher.diskpool import DiskPool51from lp.archivepublisher.diskpool import DiskPool
52from lp.archivepublisher.domination import Dominator52from lp.archivepublisher.domination import Dominator
53from lp.archivepublisher.htaccess import (
54 htpasswd_credentials_for_archive,
55 write_htaccess,
56 write_htpasswd,
57 )
58from lp.archivepublisher.indices import (53from lp.archivepublisher.indices import (
59 build_binary_stanza_fields,54 build_binary_stanza_fields,
60 build_source_stanza_fields,55 build_source_stanza_fields,
61 build_translations_stanza_fields,56 build_translations_stanza_fields,
62 )57 )
63from lp.archivepublisher.interfaces.archivegpgsigningkey import ISignableArchive58from lp.archivepublisher.interfaces.archivegpgsigningkey import (
59 ISignableArchive,
60 )
64from lp.archivepublisher.model.ftparchive import FTPArchiveHandler61from lp.archivepublisher.model.ftparchive import FTPArchiveHandler
65from lp.archivepublisher.utils import (62from lp.archivepublisher.utils import (
66 get_ppa_reference,63 get_ppa_reference,
@@ -166,27 +163,6 @@ def _getDiskPool(pubconf, log):
166 return dp163 return dp
167164
168165
169def _setupHtaccess(archive, pubconf, log):
170 """Setup .htaccess/.htpasswd files for an archive.
171 """
172 if not archive.private:
173 # FIXME: JRV 20101108 leftover .htaccess and .htpasswd files
174 # should be removed when support for making existing 3PA's public
175 # is added; bug=376072
176 return
177
178 htaccess_path = os.path.join(pubconf.archiveroot, ".htaccess")
179 htpasswd_path = os.path.join(pubconf.archiveroot, ".htpasswd")
180 # After the initial htaccess/htpasswd files
181 # are created generate_ppa_htaccess is responsible for
182 # updating the tokens.
183 if not os.path.exists(htaccess_path):
184 log.debug("Writing htaccess file.")
185 write_htaccess(htaccess_path, pubconf.archiveroot)
186 passwords = htpasswd_credentials_for_archive(archive)
187 write_htpasswd(htpasswd_path, passwords)
188
189
190def getPublisher(archive, allowed_suites, log, distsroot=None):166def getPublisher(archive, allowed_suites, log, distsroot=None):
191 """Return an initialized Publisher instance for the given context.167 """Return an initialized Publisher instance for the given context.
192168
@@ -472,7 +448,6 @@ class Publisher(object):
472 def setupArchiveDirs(self):448 def setupArchiveDirs(self):
473 self.log.debug("Setting up archive directories.")449 self.log.debug("Setting up archive directories.")
474 self._config.setupArchiveDirs()450 self._config.setupArchiveDirs()
475 _setupHtaccess(self.archive, self._config, self.log)
476451
477 def isDirty(self, distroseries, pocket):452 def isDirty(self, distroseries, pocket):
478 """True if a publication has happened in this release and pocket."""453 """True if a publication has happened in this release and pocket."""
diff --git a/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py b/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py
index a272540..26e8db8 100644
--- a/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py
+++ b/lib/lp/archivepublisher/scripts/generate_ppa_htaccess.py
@@ -3,22 +3,10 @@
3# Copyright 2009-2011 Canonical Ltd. This software is licensed under the3# Copyright 2009-2011 Canonical Ltd. This software is licensed under the
4# GNU Affero General Public License version 3 (see the file LICENSE).4# GNU Affero General Public License version 3 (see the file LICENSE).
55
6from datetime import (6from datetime import datetime
7 datetime,
8 timedelta,
9 )
10import filecmp
11import os
12import tempfile
137
14import pytz8import pytz
159
16from lp.archivepublisher.config import getPubConfig
17from lp.archivepublisher.htaccess import (
18 htpasswd_credentials_for_archive,
19 write_htaccess,
20 write_htpasswd,
21 )
22from lp.registry.model.teammembership import TeamParticipation10from lp.registry.model.teammembership import TeamParticipation
23from lp.services.config import config11from lp.services.config import config
24from lp.services.database.interfaces import IStore12from lp.services.database.interfaces import IStore
@@ -30,23 +18,19 @@ from lp.services.mail.sendmail import (
30 )18 )
31from lp.services.scripts.base import LaunchpadCronScript19from lp.services.scripts.base import LaunchpadCronScript
32from lp.services.webapp import canonical_url20from lp.services.webapp import canonical_url
33from lp.soyuz.enums import (21from lp.soyuz.enums import ArchiveSubscriberStatus
34 ArchiveStatus,
35 ArchiveSubscriberStatus,
36 )
37from lp.soyuz.model.archive import Archive
38from lp.soyuz.model.archiveauthtoken import ArchiveAuthToken22from lp.soyuz.model.archiveauthtoken import ArchiveAuthToken
39from lp.soyuz.model.archivesubscriber import ArchiveSubscriber23from lp.soyuz.model.archivesubscriber import ArchiveSubscriber
4024
41# These PPAs should never have their htaccess/pwd files touched.
42BLACKLISTED_PPAS = {
43 'ubuntuone': ['ppa'],
44 }
45
4625
47class HtaccessTokenGenerator(LaunchpadCronScript):26class HtaccessTokenGenerator(LaunchpadCronScript):
48 """Helper class for generating .htaccess files for private PPAs."""27 """Expire archive subscriptions and deactivate invalid tokens."""
49 blacklist = BLACKLISTED_PPAS28
29 # XXX cjwatson 2021-04-21: This script and class are now misnamed, as we
30 # no longer generate .htaccess or .htpasswd files, but instead check
31 # archive authentication dynamically. We can remove this script once we
32 # stop running it on production and move its remaining functions
33 # elsewhere (probably garbo).
5034
51 def add_my_options(self):35 def add_my_options(self):
52 """Add script command line options."""36 """Add script command line options."""
@@ -60,68 +44,6 @@ class HtaccessTokenGenerator(LaunchpadCronScript):
60 dest="no_deactivation", default=False,44 dest="no_deactivation", default=False,
61 help="If set, tokens are not deactivated.")45 help="If set, tokens are not deactivated.")
6246
63 def ensureHtaccess(self, ppa):
64 """Generate a .htaccess for `ppa`."""
65 if self.options.dryrun:
66 return
67
68 # The publisher Config object does not have an
69 # interface, so we need to remove the security wrapper.
70 pub_config = getPubConfig(ppa)
71 htaccess_filename = os.path.join(pub_config.archiveroot, ".htaccess")
72 if not os.path.exists(htaccess_filename):
73 # It's not there, so create it.
74 if not os.path.exists(pub_config.archiveroot):
75 os.makedirs(pub_config.archiveroot)
76 write_htaccess(htaccess_filename, pub_config.archiveroot)
77 self.logger.debug("Created .htaccess for %s" % ppa.displayname)
78
79 def generateHtpasswd(self, ppa):
80 """Generate a htpasswd file for `ppa`s `tokens`.
81
82 :param ppa: The context PPA (an `IArchive`).
83 :return: The filename of the htpasswd file that was generated.
84 """
85 # Create a temporary file that will be a new .htpasswd.
86 pub_config = getPubConfig(ppa)
87 if not os.path.exists(pub_config.temproot):
88 os.makedirs(pub_config.temproot)
89 fd, temp_filename = tempfile.mkstemp(dir=pub_config.temproot)
90 os.close(fd)
91
92 write_htpasswd(temp_filename, htpasswd_credentials_for_archive(ppa))
93
94 return temp_filename
95
96 def replaceUpdatedHtpasswd(self, ppa, temp_htpasswd_file):
97 """Compare the new and the old htpasswd and replace if changed.
98
99 :return: True if the file was replaced.
100 """
101 try:
102 if self.options.dryrun:
103 return False
104
105 # The publisher Config object does not have an
106 # interface, so we need to remove the security wrapper.
107 pub_config = getPubConfig(ppa)
108 if not os.path.exists(pub_config.archiveroot):
109 os.makedirs(pub_config.archiveroot)
110 htpasswd_filename = os.path.join(
111 pub_config.archiveroot, ".htpasswd")
112
113 if (not os.path.isfile(htpasswd_filename) or
114 not filecmp.cmp(htpasswd_filename, temp_htpasswd_file)):
115 # Atomically replace the old file or create a new file.
116 os.rename(temp_htpasswd_file, htpasswd_filename)
117 self.logger.debug("Replaced htpasswd for %s" % ppa.displayname)
118 return True
119
120 return False
121 finally:
122 if os.path.exists(temp_htpasswd_file):
123 os.unlink(temp_htpasswd_file)
124
125 def sendCancellationEmail(self, token):47 def sendCancellationEmail(self, token):
126 """Send an email to the person whose subscription was cancelled."""48 """Send an email to the person whose subscription was cancelled."""
127 if token.archive.suppress_subscription_notifications:49 if token.archive.suppress_subscription_notifications:
@@ -220,8 +142,7 @@ class HtaccessTokenGenerator(LaunchpadCronScript):
220 :param send_email: Whether to send a cancellation email to the owner142 :param send_email: Whether to send a cancellation email to the owner
221 of the token. This defaults to False to speed up the test143 of the token. This defaults to False to speed up the test
222 suite.144 suite.
223 :return: the set of ppas affected by token deactivations so that we145 :return: the set of ppas affected by token deactivations.
224 can later update their htpasswd files.
225 """146 """
226 invalid_tokens = self._getInvalidTokens()147 invalid_tokens = self._getInvalidTokens()
227 return self.deactivateTokens(invalid_tokens, send_email=send_email)148 return self.deactivateTokens(invalid_tokens, send_email=send_email)
@@ -249,129 +170,13 @@ class HtaccessTokenGenerator(LaunchpadCronScript):
249 self.logger.info(170 self.logger.info(
250 "Expired subscriptions: %s" % ", ".join(subscription_names))171 "Expired subscriptions: %s" % ", ".join(subscription_names))
251172
252 def getTimeToSyncFrom(self):
253 """Return the time we'll synchronize from.
254
255 Any new PPAs or tokens created since this time will be used to
256 generate passwords.
257 """
258 # NTP is running on our servers and therefore we can assume
259 # only minimal skew, we include a fudge-factor of 1s so that
260 # even the minimal skew cannot demonstrate bug 627608.
261 last_activity = self.get_last_activity()
262 if not last_activity:
263 return
264 return last_activity.date_started - timedelta(seconds=1)
265
266 def getNewTokens(self, since=None):
267 """Return result set of new tokens created since the given time."""
268 store = IStore(ArchiveAuthToken)
269 extra_expr = []
270 if since:
271 extra_expr = [ArchiveAuthToken.date_created >= since]
272 new_ppa_tokens = store.find(
273 ArchiveAuthToken,
274 ArchiveAuthToken.date_deactivated == None,
275 *extra_expr)
276 return new_ppa_tokens
277
278 def getDeactivatedNamedTokens(self, since=None):
279 """Return result set of named tokens deactivated since given time."""
280 now = datetime.now(pytz.UTC)
281
282 store = IStore(ArchiveAuthToken)
283 extra_expr = []
284 if since:
285 extra_expr = [ArchiveAuthToken.date_deactivated >= since]
286 tokens = store.find(
287 ArchiveAuthToken,
288 ArchiveAuthToken.name != None,
289 ArchiveAuthToken.date_deactivated != None,
290 ArchiveAuthToken.date_deactivated <= now,
291 *extra_expr)
292 return tokens
293
294 def getNewPrivatePPAs(self, since=None):
295 """Return the recently created private PPAs."""
296 store = IStore(Archive)
297 extra_expr = []
298 if since:
299 extra_expr = [Archive.date_created >= since]
300 return store.find(
301 Archive, Archive._private == True, *extra_expr)
302
303 def main(self):173 def main(self):
304 """Script entry point."""174 """Script entry point."""
305 self.logger.info('Starting the PPA .htaccess generation')175 self.logger.info('Starting the PPA .htaccess generation')
306 self.expireSubscriptions()176 self.expireSubscriptions()
307 affected_ppas = self.deactivateInvalidTokens(send_email=True)177 affected_ppas = self.deactivateInvalidTokens(send_email=True)
308 current_ppa_count = len(affected_ppas)
309 self.logger.debug(
310 '%s PPAs with deactivated tokens' % current_ppa_count)
311
312 last_success = self.getTimeToSyncFrom()
313
314 # Include ppas with named tokens deactivated since last time we ran.
315 num_tokens = 0
316 for token in self.getDeactivatedNamedTokens(since=last_success):
317 affected_ppas.add(token.archive)
318 num_tokens += 1
319
320 new_ppa_count = len(affected_ppas)
321 self.logger.debug(
322 "%s deactivated named tokens since last run, %s PPAs affected"
323 % (num_tokens, new_ppa_count - current_ppa_count))
324 current_ppa_count = new_ppa_count
325
326 # In addition to the ppas that are affected by deactivated
327 # tokens, we also want to include any ppas that have tokens
328 # created since the last time we ran.
329 num_tokens = 0
330 for token in self.getNewTokens(since=last_success):
331 affected_ppas.add(token.archive)
332 num_tokens += 1
333
334 new_ppa_count = len(affected_ppas)
335 self.logger.debug(
336 "%s new tokens since last run, %s PPAs affected"
337 % (num_tokens, new_ppa_count - current_ppa_count))
338 current_ppa_count = new_ppa_count
339
340 affected_ppas.update(self.getNewPrivatePPAs(since=last_success))
341 new_ppa_count = len(affected_ppas)
342 self.logger.debug(178 self.logger.debug(
343 "%s new private PPAs since last run"179 '%s PPAs with deactivated tokens' % len(affected_ppas))
344 % (new_ppa_count - current_ppa_count))
345
346 self.logger.debug('%s PPAs require updating' % new_ppa_count)
347 for ppa in affected_ppas:
348 # If this PPA is blacklisted, do not touch its htaccess/pwd
349 # files.
350 blacklisted_ppa_names_for_owner = self.blacklist.get(
351 ppa.owner.name, [])
352 if ppa.name in blacklisted_ppa_names_for_owner:
353 self.logger.info(
354 "Skipping htaccess updates for blacklisted PPA "
355 " '%s' owned by %s.",
356 ppa.name,
357 ppa.owner.displayname)
358 continue
359 elif ppa.status == ArchiveStatus.DELETED or ppa.enabled is False:
360 self.logger.info(
361 "Skipping htaccess updates for deleted or disabled PPA "
362 " '%s' owned by %s.",
363 ppa.name,
364 ppa.owner.displayname)
365 continue
366
367 self.ensureHtaccess(ppa)
368 htpasswd_write_start = datetime.now()
369 temp_htpasswd = self.generateHtpasswd(ppa)
370 self.replaceUpdatedHtpasswd(ppa, temp_htpasswd)
371 htpasswd_write_duration = datetime.now() - htpasswd_write_start
372 self.logger.debug(
373 "Wrote htpasswd for '%s': %ss"
374 % (ppa.name, htpasswd_write_duration.total_seconds()))
375180
376 if self.options.no_deactivation or self.options.dryrun:181 if self.options.no_deactivation or self.options.dryrun:
377 self.logger.info('Dry run, so not committing transaction.')182 self.logger.info('Dry run, so not committing transaction.')
diff --git a/lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py b/lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py
index f11dba1..472b7bf 100644
--- a/lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py
+++ b/lib/lp/archivepublisher/tests/test_generate_ppa_htaccess.py
@@ -5,7 +5,6 @@
55
6from __future__ import absolute_import, print_function, unicode_literals6from __future__ import absolute_import, print_function, unicode_literals
77
8import crypt
9from datetime import (8from datetime import (
10 datetime,9 datetime,
11 timedelta,10 timedelta,
@@ -13,20 +12,10 @@ from datetime import (
13import os12import os
14import subprocess13import subprocess
15import sys14import sys
16import tempfile
1715
18import pytz16import pytz
19from testtools.matchers import (
20 AllMatch,
21 FileContains,
22 FileExists,
23 Not,
24 )
25import transaction
26from zope.component import getUtility17from zope.component import getUtility
27from zope.security.proxy import removeSecurityProxy
2818
29from lp.archivepublisher.config import getPubConfig
30from lp.archivepublisher.scripts.generate_ppa_htaccess import (19from lp.archivepublisher.scripts.generate_ppa_htaccess import (
31 HtaccessTokenGenerator,20 HtaccessTokenGenerator,
32 )21 )
@@ -36,16 +25,7 @@ from lp.registry.interfaces.teammembership import TeamMembershipStatus
36from lp.services.config import config25from lp.services.config import config
37from lp.services.features.testing import FeatureFixture26from lp.services.features.testing import FeatureFixture
38from lp.services.log.logger import BufferLogger27from lp.services.log.logger import BufferLogger
39from lp.services.osutils import (28from lp.soyuz.enums import ArchiveSubscriberStatus
40 ensure_directory_exists,
41 remove_if_exists,
42 write_file,
43 )
44from lp.services.scripts.interfaces.scriptactivity import IScriptActivitySet
45from lp.soyuz.enums import (
46 ArchiveStatus,
47 ArchiveSubscriberStatus,
48 )
49from lp.soyuz.interfaces.archive import NAMED_AUTH_TOKEN_FEATURE_FLAG29from lp.soyuz.interfaces.archive import NAMED_AUTH_TOKEN_FEATURE_FLAG
50from lp.testing import TestCaseWithFactory30from lp.testing import TestCaseWithFactory
51from lp.testing.dbuser import (31from lp.testing.dbuser import (
@@ -102,102 +82,6 @@ class TestPPAHtaccessTokenGeneration(TestCaseWithFactory):
102 stdout, stderr = process.communicate()82 stdout, stderr = process.communicate()
103 return process.returncode, stdout, stderr83 return process.returncode, stdout, stderr
10484
105 def testEnsureHtaccess(self):
106 """Ensure that the .htaccess file is generated correctly."""
107 # The publisher Config object does not have an interface, so we
108 # need to remove the security wrapper.
109 pub_config = getPubConfig(self.ppa)
110
111 filename = os.path.join(pub_config.archiveroot, ".htaccess")
112 remove_if_exists(filename)
113 script = self.getScript()
114 script.ensureHtaccess(self.ppa)
115 self.addCleanup(remove_if_exists, filename)
116
117 contents = [
118 "",
119 "AuthType Basic",
120 "AuthName \"Token Required\"",
121 "AuthUserFile %s/.htpasswd" % pub_config.archiveroot,
122 "Require valid-user",
123 "",
124 ]
125 self.assertThat(filename, FileContains('\n'.join(contents)))
126
127 def testGenerateHtpasswd(self):
128 """Given some `ArchiveAuthToken`s, test generating htpasswd."""
129 # Make some subscriptions and tokens.
130 tokens = []
131 for name in ['name12', 'name16']:
132 person = getUtility(IPersonSet).getByName(name)
133 self.ppa.newSubscription(person, self.ppa.owner)
134 tokens.append(self.ppa.newAuthToken(person))
135 token_usernames = [token.person.name for token in tokens]
136
137 # Generate the passwd file.
138 script = self.getScript()
139 filename = script.generateHtpasswd(self.ppa)
140 self.addCleanup(remove_if_exists, filename)
141
142 # It should be a temp file on the same filesystem as the target
143 # file, so os.rename() won't explode. temproot is relied on
144 # elsewhere for this same purpose, so it should be safe.
145 pub_config = getPubConfig(self.ppa)
146 self.assertEqual(pub_config.temproot, os.path.dirname(filename))
147
148 # Read it back in.
149 file_contents = [
150 line.strip().split(':', 1) for line in open(filename, 'r')]
151
152 # First entry is buildd secret, rest are from tokens.
153 usernames = list(list(zip(*file_contents))[0])
154 self.assertEqual(['buildd'] + token_usernames, usernames)
155
156 # We can re-encrypt the buildd_secret and it should match the
157 # one in the .htpasswd file.
158 password = file_contents[0][1]
159 encrypted_secret = crypt.crypt(self.ppa.buildd_secret, password)
160 self.assertEqual(encrypted_secret, password)
161
162 def testReplaceUpdatedHtpasswd(self):
163 """Test that the htpasswd file is only replaced if it changes."""
164 FILE_CONTENT = b"Kneel before Zod!"
165 # The publisher Config object does not have an interface, so we
166 # need to remove the security wrapper.
167 pub_config = getPubConfig(self.ppa)
168 filename = os.path.join(pub_config.archiveroot, ".htpasswd")
169
170 # Write out a dummy .htpasswd
171 ensure_directory_exists(pub_config.archiveroot)
172 write_file(filename, FILE_CONTENT)
173
174 # Write the same contents in a temp file.
175 def write_tempfile():
176 fd, temp_filename = tempfile.mkstemp(dir=pub_config.archiveroot)
177 file = os.fdopen(fd, "wb")
178 file.write(FILE_CONTENT)
179 file.close()
180 return temp_filename
181
182 # Replacement should not happen.
183 temp_filename = write_tempfile()
184 script = self.getScript()
185 self.assertTrue(os.path.exists(temp_filename))
186 self.assertFalse(
187 script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
188 self.assertFalse(os.path.exists(temp_filename))
189
190 # Writing a different .htpasswd should see it get replaced.
191 write_file(filename, b"Come to me, son of Jor-El!")
192
193 temp_filename = write_tempfile()
194 self.assertTrue(os.path.exists(temp_filename))
195 self.assertTrue(
196 script.replaceUpdatedHtpasswd(self.ppa, temp_filename))
197 self.assertFalse(os.path.exists(temp_filename))
198
199 os.remove(filename)
200
201 def assertDeactivated(self, token):85 def assertDeactivated(self, token):
202 """Helper function to test token deactivation state."""86 """Helper function to test token deactivation state."""
203 return self.assertNotEqual(token.date_deactivated, None)87 return self.assertNotEqual(token.date_deactivated, None)
@@ -341,15 +225,6 @@ class TestPPAHtaccessTokenGeneration(TestCaseWithFactory):
341 self.layer.txn.commit()225 self.layer.txn.commit()
342 return (sub1, sub2), (token1, token2, token3)226 return (sub1, sub2), (token1, token2, token3)
343227
344 def ensureNoFiles(self):
345 """Ensure the .ht* files don't already exist."""
346 pub_config = getPubConfig(self.ppa)
347 htaccess = os.path.join(pub_config.archiveroot, ".htaccess")
348 htpasswd = os.path.join(pub_config.archiveroot, ".htpasswd")
349 remove_if_exists(htaccess)
350 remove_if_exists(htpasswd)
351 return htaccess, htpasswd
352
353 def testSubscriptionExpiry(self):228 def testSubscriptionExpiry(self):
354 """Ensure subscriptions' statuses are set to EXPIRED properly."""229 """Ensure subscriptions' statuses are set to EXPIRED properly."""
355 subs, tokens = self.setupDummyTokens()230 subs, tokens = self.setupDummyTokens()
@@ -369,51 +244,6 @@ class TestPPAHtaccessTokenGeneration(TestCaseWithFactory):
369 self.assertEqual(subs[0].status, ArchiveSubscriberStatus.EXPIRED)244 self.assertEqual(subs[0].status, ArchiveSubscriberStatus.EXPIRED)
370 self.assertEqual(subs[1].status, ArchiveSubscriberStatus.CURRENT)245 self.assertEqual(subs[1].status, ArchiveSubscriberStatus.CURRENT)
371246
372 def testBasicOperation(self):
373 """Invoke the actual script and make sure it generates some files."""
374 self.setupDummyTokens()
375 htaccess, htpasswd = self.ensureNoFiles()
376
377 # Call the script and check that we have a .htaccess and a
378 # .htpasswd.
379 return_code, stdout, stderr = self.runScript()
380 self.assertEqual(
381 return_code, 0, "Got a bad return code of %s\nOutput:\n%s" %
382 (return_code, stderr))
383 self.assertThat([htaccess, htpasswd], AllMatch(FileExists()))
384 os.remove(htaccess)
385 os.remove(htpasswd)
386
387 def testBasicOperation_with_named_tokens(self):
388 """Invoke the actual script and make sure it generates some files."""
389 token1 = self.ppa.newNamedAuthToken("tokenname1")
390 token2 = self.ppa.newNamedAuthToken("tokenname2")
391 token3 = self.ppa.newNamedAuthToken("tokenname3")
392 token3.deactivate()
393
394 # Call the script and check that we have a .htaccess and a .htpasswd.
395 htaccess, htpasswd = self.ensureNoFiles()
396 script = self.getScript()
397 script.main()
398 self.assertThat([htaccess, htpasswd], AllMatch(FileExists()))
399 with open(htpasswd) as htpasswd_file:
400 contents = htpasswd_file.read()
401 self.assertIn('+' + token1.name, contents)
402 self.assertIn('+' + token2.name, contents)
403 self.assertNotIn('+' + token3.name, contents)
404
405 # Deactivate a named token and verify it is removed from .htpasswd.
406 token2.deactivate()
407 script.main()
408 self.assertThat([htaccess, htpasswd], AllMatch(FileExists()))
409 with open(htpasswd) as htpasswd_file:
410 contents = htpasswd_file.read()
411 self.assertIn('+' + token1.name, contents)
412 self.assertNotIn('+' + token2.name, contents)
413 self.assertNotIn('+' + token3.name, contents)
414 os.remove(htaccess)
415 os.remove(htpasswd)
416
417 def _setupOptionsData(self):247 def _setupOptionsData(self):
418 """Setup test data for option testing."""248 """Setup test data for option testing."""
419 subs, tokens = self.setupDummyTokens()249 subs, tokens = self.setupDummyTokens()
@@ -427,13 +257,9 @@ class TestPPAHtaccessTokenGeneration(TestCaseWithFactory):
427 """Test that the dryrun and no-deactivation option works."""257 """Test that the dryrun and no-deactivation option works."""
428 subs, tokens = self._setupOptionsData()258 subs, tokens = self._setupOptionsData()
429259
430 htaccess, htpasswd = self.ensureNoFiles()
431 script = self.getScript(test_args=["--dry-run"])260 script = self.getScript(test_args=["--dry-run"])
432 script.main()261 script.main()
433262
434 # Assert no files were written.
435 self.assertThat([htaccess, htpasswd], AllMatch(Not(FileExists())))
436
437 # Assert that the cancelled subscription did not cause the token263 # Assert that the cancelled subscription did not cause the token
438 # to get deactivated.264 # to get deactivated.
439 self.assertNotDeactivated(tokens[0])265 self.assertNotDeactivated(tokens[0])
@@ -448,65 +274,6 @@ class TestPPAHtaccessTokenGeneration(TestCaseWithFactory):
448 script.main()274 script.main()
449 self.assertDeactivated(tokens[0])275 self.assertDeactivated(tokens[0])
450276
451 def testBlacklistingPPAs(self):
452 """Test that the htaccess for blacklisted PPAs are not touched."""
453 subs, tokens = self.setupDummyTokens()
454 htaccess, htpasswd = self.ensureNoFiles()
455
456 # Setup the first subscription so that it is due to be expired.
457 now = datetime.now(pytz.UTC)
458 subs[0].date_expires = now - timedelta(minutes=3)
459 self.assertEqual(subs[0].status, ArchiveSubscriberStatus.CURRENT)
460
461 script = self.getScript()
462 script.blacklist = {'joe': ['my_other_ppa', 'myppa', 'and_another']}
463 script.main()
464
465 # The tokens will still be deactivated, and subscriptions expired.
466 self.assertDeactivated(tokens[0])
467 self.assertEqual(subs[0].status, ArchiveSubscriberStatus.EXPIRED)
468 # But the htaccess is not touched.
469 self.assertThat([htaccess, htpasswd], AllMatch(Not(FileExists())))
470
471 def testSkippingOfDisabledPPAs(self):
472 """Test that the htaccess for disabled PPAs are not touched."""
473 subs, tokens = self.setupDummyTokens()
474 htaccess, htpasswd = self.ensureNoFiles()
475
476 # Setup subscription so that htaccess/htpasswd is pending generation.
477 now = datetime.now(pytz.UTC)
478 subs[0].date_expires = now + timedelta(minutes=3)
479 self.assertEqual(subs[0].status, ArchiveSubscriberStatus.CURRENT)
480
481 # Set the PPA as disabled.
482 self.ppa.disable()
483 self.assertFalse(self.ppa.enabled)
484
485 script = self.getScript()
486 script.main()
487
488 # The htaccess and htpasswd files should not be generated.
489 self.assertThat([htaccess, htpasswd], AllMatch(Not(FileExists())))
490
491 def testSkippingOfDeletedPPAs(self):
492 """Test that the htaccess for deleted PPAs are not touched."""
493 subs, tokens = self.setupDummyTokens()
494 htaccess, htpasswd = self.ensureNoFiles()
495
496 # Setup subscription so that htaccess/htpasswd is pending generation.
497 now = datetime.now(pytz.UTC)
498 subs[0].date_expires = now + timedelta(minutes=3)
499 self.assertEqual(subs[0].status, ArchiveSubscriberStatus.CURRENT)
500
501 # Set the PPA as deleted.
502 self.ppa.status = ArchiveStatus.DELETED
503
504 script = self.getScript()
505 script.main()
506
507 # The htaccess and htpasswd files should not be generated.
508 self.assertThat([htaccess, htpasswd], AllMatch(Not(FileExists())))
509
510 def testSendingCancellationEmail(self):277 def testSendingCancellationEmail(self):
511 """Test that when a token is deactivated, its user gets an email.278 """Test that when a token is deactivated, its user gets an email.
512279
@@ -568,120 +335,3 @@ class TestPPAHtaccessTokenGeneration(TestCaseWithFactory):
568 script.sendCancellationEmail(token)335 script.sendCancellationEmail(token)
569336
570 self.assertEmailQueueLength(0)337 self.assertEmailQueueLength(0)
571
572 def test_getTimeToSyncFrom(self):
573 # Sync from 1s before previous start to catch anything made during the
574 # last script run, and to handle NTP clock skew.
575 now = datetime.now(pytz.UTC)
576 script_start_time = now - timedelta(seconds=2)
577 script_end_time = now
578
579 getUtility(IScriptActivitySet).recordSuccess(
580 self.SCRIPT_NAME, script_start_time, script_end_time)
581 script = self.getScript()
582 self.assertEqual(
583 script_start_time - timedelta(seconds=1),
584 script.getTimeToSyncFrom())
585
586 def test_getNewPrivatePPAs_no_previous_run(self):
587 # All private PPAs are returned if there was no previous run.
588 # This happens even if they have no tokens.
589
590 # Create a public PPA that should not be in the list.
591 self.factory.makeArchive(private=False)
592
593 script = self.getScript()
594 self.assertContentEqual([self.ppa], script.getNewPrivatePPAs())
595
596 def test_getNewPrivatePPAs_only_those_since_last_run(self):
597 # Only private PPAs created since the last run are returned.
598 # This happens even if they have no tokens.
599 last_start = datetime.now(pytz.UTC) - timedelta(seconds=90)
600 before_last_start = last_start - timedelta(seconds=30)
601 removeSecurityProxy(self.ppa).date_created = before_last_start
602
603 # Create a new PPA that should show up.
604 new_ppa = self.factory.makeArchive(private=True)
605
606 script = self.getScript()
607 new_ppas = script.getNewPrivatePPAs(since=last_start)
608 self.assertContentEqual([new_ppa], new_ppas)
609
610 def test_getNewTokens_no_previous_run(self):
611 """All valid tokens returned if there is no record of previous run."""
612 tokens = self.setupDummyTokens()[1]
613
614 # If there is no record of the script running previously, all
615 # valid tokens are returned.
616 script = self.getScript()
617 self.assertContentEqual(tokens, script.getNewTokens())
618
619 def test_getNewTokens_only_those_since_last_run(self):
620 """Only tokens created since the last run are returned."""
621 last_start = datetime.now(pytz.UTC) - timedelta(seconds=90)
622 before_last_start = last_start - timedelta(seconds=30)
623
624 tokens = self.setupDummyTokens()[1]
625 # This token will not be included.
626 removeSecurityProxy(tokens[0]).date_created = before_last_start
627
628 script = self.getScript()
629 new_tokens = script.getNewTokens(since=last_start)
630 self.assertContentEqual(tokens[1:], new_tokens)
631
632 def test_getNewTokens_only_active_tokens(self):
633 """Only active tokens are returned."""
634 tokens = self.setupDummyTokens()[1]
635 tokens[0].deactivate()
636
637 script = self.getScript()
638 self.assertContentEqual(tokens[1:], script.getNewTokens())
639
640 def test_getDeactivatedNamedTokens_no_previous_run(self):
641 """All deactivated named tokens returned if there is no record
642 of previous run."""
643 last_start = datetime.now(pytz.UTC) - timedelta(seconds=90)
644 before_last_start = last_start - timedelta(seconds=30)
645
646 self.ppa.newNamedAuthToken("tokenname1")
647 token2 = self.ppa.newNamedAuthToken("tokenname2")
648 token2.deactivate()
649 token3 = self.ppa.newNamedAuthToken("tokenname3")
650 token3.date_deactivated = before_last_start
651
652 script = self.getScript()
653 self.assertContentEqual(
654 [token2, token3], script.getDeactivatedNamedTokens())
655
656 def test_getDeactivatedNamedTokens_only_those_since_last_run(self):
657 """Only named tokens deactivated since last run are returned."""
658 last_start = datetime.now(pytz.UTC) - timedelta(seconds=90)
659 before_last_start = last_start - timedelta(seconds=30)
660 tomorrow = datetime.now(pytz.UTC) + timedelta(days=1)
661
662 self.ppa.newNamedAuthToken("tokenname1")
663 token2 = self.ppa.newNamedAuthToken("tokenname2")
664 token2.deactivate()
665 token3 = self.ppa.newNamedAuthToken("tokenname3")
666 token3.date_deactivated = before_last_start
667 token4 = self.ppa.newNamedAuthToken("tokenname4")
668 token4.date_deactivated = tomorrow
669
670 script = self.getScript()
671 self.assertContentEqual(
672 [token2], script.getDeactivatedNamedTokens(last_start))
673
674 def test_processes_PPAs_without_subscription(self):
675 # A .htaccess file is written for Private PPAs even if they don't have
676 # any subscriptions.
677 htaccess, htpasswd = self.ensureNoFiles()
678 transaction.commit()
679
680 # Call the script and check that we have a .htaccess and a .htpasswd.
681 return_code, stdout, stderr = self.runScript()
682 self.assertEqual(
683 return_code, 0, "Got a bad return code of %s\nOutput:\n%s" %
684 (return_code, stderr))
685 self.assertThat([htaccess, htpasswd], AllMatch(FileExists()))
686 os.remove(htaccess)
687 os.remove(htpasswd)
diff --git a/lib/lp/archivepublisher/tests/test_htaccess.py b/lib/lp/archivepublisher/tests/test_htaccess.py
688deleted file mode 100644338deleted file mode 100644
index d435a2d..0000000
--- a/lib/lp/archivepublisher/tests/test_htaccess.py
+++ /dev/null
@@ -1,139 +0,0 @@
1# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
2# GNU Affero General Public License version 3 (see the file LICENSE).
3
4"""Test htaccess/htpasswd file generation. """
5
6from __future__ import absolute_import, print_function, unicode_literals
7
8import os
9import tempfile
10
11from zope.component import getUtility
12
13from lp.archivepublisher.htaccess import (
14 htpasswd_credentials_for_archive,
15 write_htaccess,
16 write_htpasswd,
17 )
18from lp.registry.interfaces.distribution import IDistributionSet
19from lp.registry.interfaces.person import IPersonSet
20from lp.services.features.testing import FeatureFixture
21from lp.soyuz.interfaces.archive import NAMED_AUTH_TOKEN_FEATURE_FLAG
22from lp.testing import TestCaseWithFactory
23from lp.testing.layers import LaunchpadZopelessLayer
24
25
26class TestHtpasswdGeneration(TestCaseWithFactory):
27 """Test htpasswd generation."""
28
29 layer = LaunchpadZopelessLayer
30
31 def setUp(self):
32 super(TestHtpasswdGeneration, self).setUp()
33 self.owner = self.factory.makePerson(
34 name="joe", displayname="Joe Smith")
35 self.ppa = self.factory.makeArchive(
36 owner=self.owner, name="myppa", private=True)
37
38 # "Ubuntu" doesn't have a proper publisher config but Ubuntutest
39 # does, so override the PPA's distro here.
40 ubuntutest = getUtility(IDistributionSet)['ubuntutest']
41 self.ppa.distribution = ubuntutest
42
43 # Enable named auth tokens.
44 self.useFixture(FeatureFixture({NAMED_AUTH_TOKEN_FEATURE_FLAG: "on"}))
45
46 def test_write_htpasswd(self):
47 """Test that writing the .htpasswd file works properly."""
48 fd, filename = tempfile.mkstemp()
49 os.close(fd)
50
51 TEST_PASSWORD = "password"
52 TEST_PASSWORD2 = "passwor2"
53
54 # We provide a constant salt to the crypt function so that we
55 # can test the encrypted result.
56 SALT = "XX"
57
58 user1 = ("user", TEST_PASSWORD, SALT)
59 user2 = ("user2", TEST_PASSWORD2, SALT)
60 list_of_users = [user1]
61 list_of_users.append(user2)
62
63 write_htpasswd(filename, list_of_users)
64
65 expected_contents = [
66 "user:XXq2wKiyI43A2",
67 "user2:XXaQB8b5Gtwi.",
68 ]
69
70 file = open(filename, "r")
71 file_contents = file.read().splitlines()
72 file.close()
73 os.remove(filename)
74
75 self.assertEqual(expected_contents, file_contents)
76
77 def test_write_htaccess(self):
78 # write_access can write a correct htaccess file.
79 fd, filename = tempfile.mkstemp()
80 os.close(fd)
81
82 write_htaccess(filename, "/some/distroot")
83 self.assertTrue(
84 os.path.isfile(filename),
85 "%s is not present when it should be" % filename)
86 self.addCleanup(os.remove, filename)
87
88 contents = [
89 "",
90 "AuthType Basic",
91 "AuthName \"Token Required\"",
92 "AuthUserFile /some/distroot/.htpasswd",
93 "Require valid-user",
94 ]
95
96 file = open(filename, "r")
97 file_contents = file.read().splitlines()
98 file.close()
99
100 self.assertEqual(contents, file_contents)
101
102 def test_credentials_for_archive_empty(self):
103 # If there are no ArchiveAuthTokens for an archive just
104 # the buildd secret is returned.
105 self.ppa.buildd_secret = "sekr1t"
106 self.assertEqual(
107 [("buildd", "sekr1t", "bu")],
108 list(htpasswd_credentials_for_archive(self.ppa)))
109
110 def test_credentials_for_archive(self):
111 # ArchiveAuthTokens for an archive are returned by
112 # credentials_for_archive.
113 self.ppa.buildd_secret = "geheim"
114 name12 = getUtility(IPersonSet).getByName("name12")
115 name16 = getUtility(IPersonSet).getByName("name16")
116 hyphenated = self.factory.makePerson(name="a-b-c")
117 self.ppa.newSubscription(name12, self.ppa.owner)
118 self.ppa.newSubscription(name16, self.ppa.owner)
119 self.ppa.newSubscription(hyphenated, self.ppa.owner)
120 first_created_token = self.ppa.newAuthToken(name16)
121 second_created_token = self.ppa.newAuthToken(name12)
122 third_created_token = self.ppa.newAuthToken(hyphenated)
123 named_token_20 = self.ppa.newNamedAuthToken("name20", as_dict=False)
124 named_token_14 = self.ppa.newNamedAuthToken("name14", as_dict=False)
125 named_token_99 = self.ppa.newNamedAuthToken("name99", as_dict=False)
126 named_token_99.deactivate()
127
128 expected_credentials = [
129 ("buildd", "geheim", "bu"),
130 ("+name14", named_token_14.token, "bm"),
131 ("+name20", named_token_20.token, "bm"),
132 ("a-b-c", third_created_token.token, "YS"),
133 ("name12", second_created_token.token, "bm"),
134 ("name16", first_created_token.token, "bm"),
135 ]
136 credentials = list(htpasswd_credentials_for_archive(self.ppa))
137
138 # Use assertEqual instead of assertContentEqual to verify order.
139 self.assertEqual(expected_credentials, credentials)
diff --git a/lib/lp/archivepublisher/tests/test_publisher.py b/lib/lp/archivepublisher/tests/test_publisher.py
index 67ec904..0c8ff87 100644
--- a/lib/lp/archivepublisher/tests/test_publisher.py
+++ b/lib/lp/archivepublisher/tests/test_publisher.py
@@ -12,7 +12,6 @@ from collections import (
12 defaultdict,12 defaultdict,
13 OrderedDict,13 OrderedDict,
14 )14 )
15import crypt
16from datetime import (15from datetime import (
17 datetime,16 datetime,
18 timedelta,17 timedelta,
@@ -2328,44 +2327,6 @@ class TestPublisher(TestPublisherBase):
2328 hoary_pub.requestDeletion(self.ubuntutest.owner)2327 hoary_pub.requestDeletion(self.ubuntutest.owner)
2329 self._assertPublishesSeriesAlias(publisher, "breezy-autotest")2328 self._assertPublishesSeriesAlias(publisher, "breezy-autotest")
23302329
2331 def testHtaccessForPrivatePPA(self):
2332 # A htaccess file is created for new private PPA's.
2333
2334 ppa = self.factory.makeArchive(
2335 distribution=self.ubuntutest, private=True)
2336 ppa.buildd_secret = "geheim"
2337
2338 # Set up the publisher for it and publish its repository.
2339 # setupArchiveDirs is what actually configures the htaccess file.
2340 getPublisher(ppa, [], self.logger).setupArchiveDirs()
2341 pubconf = getPubConfig(ppa)
2342 htaccess_path = os.path.join(pubconf.archiveroot, ".htaccess")
2343 self.assertTrue(os.path.exists(htaccess_path))
2344 with open(htaccess_path, 'r') as htaccess_f:
2345 self.assertEqual(dedent("""
2346 AuthType Basic
2347 AuthName "Token Required"
2348 AuthUserFile %s/.htpasswd
2349 Require valid-user
2350 """) % pubconf.archiveroot,
2351 htaccess_f.read())
2352
2353 htpasswd_path = os.path.join(pubconf.archiveroot, ".htpasswd")
2354
2355 # Read it back in.
2356 with open(htpasswd_path, "r") as htpasswd_f:
2357 file_contents = htpasswd_f.readlines()
2358
2359 self.assertEqual(1, len(file_contents))
2360
2361 # The first line should be the buildd_secret.
2362 [user, password] = file_contents[0].strip().split(":", 1)
2363 self.assertEqual("buildd", user)
2364 # We can re-encrypt the buildd_secret and it should match the
2365 # one in the .htpasswd file.
2366 encrypted_secret = crypt.crypt(ppa.buildd_secret, password)
2367 self.assertEqual(encrypted_secret, password)
2368
2369 def testWriteSuiteI18n(self):2330 def testWriteSuiteI18n(self):
2370 """Test i18n/Index writing."""2331 """Test i18n/Index writing."""
2371 publisher = Publisher(2332 publisher = Publisher(
diff --git a/lib/lp/oci/model/ocirecipebuildjob.py b/lib/lp/oci/model/ocirecipebuildjob.py
index 8f55850..9ec2e91 100644
--- a/lib/lp/oci/model/ocirecipebuildjob.py
+++ b/lib/lp/oci/model/ocirecipebuildjob.py
@@ -45,10 +45,7 @@ from lp.oci.interfaces.ociregistryclient import (
45 )45 )
46from lp.services.config import config46from lp.services.config import config
47from lp.services.database.enumcol import DBEnum47from lp.services.database.enumcol import DBEnum
48from lp.services.database.interfaces import (48from lp.services.database.interfaces import IStore
49 IMasterStore,
50 IStore,
51 )
52from lp.services.database.locking import (49from lp.services.database.locking import (
53 AdvisoryLockHeld,50 AdvisoryLockHeld,
54 LockType,51 LockType,
@@ -189,6 +186,13 @@ class OCIRegistryUploadJob(OCIRecipeBuildJobDerived):
189186
190 class_job_type = OCIRecipeBuildJobType.REGISTRY_UPLOAD187 class_job_type = OCIRecipeBuildJobType.REGISTRY_UPLOAD
191188
189 # This is a known slow task that will exceed the timeouts for
190 # the normal job queue, so put it on a queue with longer timeouts
191 task_queue = 'launchpad_job_slow'
192
193 soft_time_limit = timedelta(minutes=60)
194 lease_duration = timedelta(minutes=60)
195
192 class ManifestListUploadError(Exception):196 class ManifestListUploadError(Exception):
193 pass197 pass
194198
diff --git a/lib/lp/oci/tests/test_ocirecipebuildjob.py b/lib/lp/oci/tests/test_ocirecipebuildjob.py
index 95718bb..9dfb785 100644
--- a/lib/lp/oci/tests/test_ocirecipebuildjob.py
+++ b/lib/lp/oci/tests/test_ocirecipebuildjob.py
@@ -53,10 +53,7 @@ from lp.services.database.locking import (
53from lp.services.features.testing import FeatureFixture53from lp.services.features.testing import FeatureFixture
54from lp.services.job.interfaces.job import JobStatus54from lp.services.job.interfaces.job import JobStatus
55from lp.services.job.runner import JobRunner55from lp.services.job.runner import JobRunner
56from lp.services.job.tests import (56from lp.services.job.tests import block_on_job
57 block_on_job,
58 pop_remote_notifications,
59 )
60from lp.services.statsd.tests import StatsMixin57from lp.services.statsd.tests import StatsMixin
61from lp.services.webapp import canonical_url58from lp.services.webapp import canonical_url
62from lp.services.webhooks.testing import LogsScheduledWebhooks59from lp.services.webhooks.testing import LogsScheduledWebhooks
@@ -71,7 +68,7 @@ from lp.testing.dbuser import (
71from lp.testing.fakemethod import FakeMethod68from lp.testing.fakemethod import FakeMethod
72from lp.testing.fixture import ZopeUtilityFixture69from lp.testing.fixture import ZopeUtilityFixture
73from lp.testing.layers import (70from lp.testing.layers import (
74 CeleryJobLayer,71 CelerySlowJobLayer,
75 DatabaseFunctionalLayer,72 DatabaseFunctionalLayer,
76 LaunchpadZopelessLayer,73 LaunchpadZopelessLayer,
77 )74 )
@@ -519,7 +516,6 @@ class TestOCIRegistryUploadJob(TestCaseWithFactory, MultiArchRecipeMixin,
519516
520 self.assertContentEqual([], ocibuild.registry_upload_jobs)517 self.assertContentEqual([], ocibuild.registry_upload_jobs)
521 job = OCIRegistryUploadJob.create(ocibuild)518 job = OCIRegistryUploadJob.create(ocibuild)
522 client = FakeRegistryClient()
523 switch_dbuser(config.IOCIRegistryUploadJobSource.dbuser)519 switch_dbuser(config.IOCIRegistryUploadJobSource.dbuser)
524 # Fork so that we can take an advisory lock from a different520 # Fork so that we can take an advisory lock from a different
525 # PostgreSQL session.521 # PostgreSQL session.
@@ -551,8 +547,6 @@ class TestOCIRegistryUploadJob(TestCaseWithFactory, MultiArchRecipeMixin,
551 os.kill(pid, signal.SIGINT)547 os.kill(pid, signal.SIGINT)
552548
553549
554
555
556class TestOCIRegistryUploadJobViaCelery(TestCaseWithFactory,550class TestOCIRegistryUploadJobViaCelery(TestCaseWithFactory,
557 MultiArchRecipeMixin):551 MultiArchRecipeMixin):
558 """Runs OCIRegistryUploadJob via Celery, to make sure the machinery552 """Runs OCIRegistryUploadJob via Celery, to make sure the machinery
@@ -563,7 +557,7 @@ class TestOCIRegistryUploadJobViaCelery(TestCaseWithFactory,
563 so we should make sure we are not breaking anything in the interaction557 so we should make sure we are not breaking anything in the interaction
564 with the job lifecycle via celery.558 with the job lifecycle via celery.
565 """559 """
566 layer = CeleryJobLayer560 layer = CelerySlowJobLayer
567561
568 def setUp(self):562 def setUp(self):
569 super(TestOCIRegistryUploadJobViaCelery, self).setUp()563 super(TestOCIRegistryUploadJobViaCelery, self).setUp()
@@ -583,4 +577,5 @@ class TestOCIRegistryUploadJobViaCelery(TestCaseWithFactory,
583 for build in builds:577 for build in builds:
584 OCIRegistryUploadJob.create(build)578 OCIRegistryUploadJob.create(build)
585 transaction.commit()579 transaction.commit()
586 self.assertEqual(0, len(pop_remote_notifications()))580 messages = [message.as_string() for message in pop_notifications()]
581 self.assertEqual(0, len(messages))
diff --git a/lib/lp/registry/browser/distribution.py b/lib/lp/registry/browser/distribution.py
index e5777ec..aa8a74c 100644
--- a/lib/lp/registry/browser/distribution.py
+++ b/lib/lp/registry/browser/distribution.py
@@ -82,9 +82,6 @@ from lp.bugs.browser.structuralsubscription import (
82 )82 )
83from lp.buildmaster.interfaces.processor import IProcessorSet83from lp.buildmaster.interfaces.processor import IProcessorSet
84from lp.code.browser.vcslisting import TargetDefaultVCSNavigationMixin84from lp.code.browser.vcslisting import TargetDefaultVCSNavigationMixin
85from lp.oci.interfaces.ociregistrycredentials import (
86 IOCIRegistryCredentialsSet,
87 )
88from lp.registry.browser import (85from lp.registry.browser import (
89 add_subscribe_link,86 add_subscribe_link,
90 RegistryEditFormView,87 RegistryEditFormView,
diff --git a/lib/lp/registry/interfaces/distribution.py b/lib/lp/registry/interfaces/distribution.py
index 5ec5271..8f43f58 100644
--- a/lib/lp/registry/interfaces/distribution.py
+++ b/lib/lp/registry/interfaces/distribution.py
@@ -14,15 +14,18 @@ __all__ = [
14 'IDistributionSet',14 'IDistributionSet',
15 'NoPartnerArchive',15 'NoPartnerArchive',
16 'NoSuchDistribution',16 'NoSuchDistribution',
17 'NoOCIAdminForDistribution',
17 ]18 ]
1819
19from lazr.lifecycle.snapshot import doNotSnapshot20from lazr.lifecycle.snapshot import doNotSnapshot
20from lazr.restful.declarations import (21from lazr.restful.declarations import (
21 call_with,22 call_with,
22 collection_default_content,23 collection_default_content,
24 error_status,
23 export_factory_operation,25 export_factory_operation,
24 export_operation_as,26 export_operation_as,
25 export_read_operation,27 export_read_operation,
28 export_write_operation,
26 exported,29 exported,
27 exported_as_webservice_collection,30 exported_as_webservice_collection,
28 exported_as_webservice_entry,31 exported_as_webservice_entry,
@@ -38,6 +41,7 @@ from lazr.restful.fields import (
38 Reference,41 Reference,
39 )42 )
40from lazr.restful.interface import copy_field43from lazr.restful.interface import copy_field
44from six.moves import http_client
41from zope.interface import (45from zope.interface import (
42 Attribute,46 Attribute,
43 Interface,47 Interface,
@@ -113,6 +117,15 @@ from lp.translations.interfaces.hastranslationimports import (
113from lp.translations.interfaces.translationpolicy import ITranslationPolicy117from lp.translations.interfaces.translationpolicy import ITranslationPolicy
114118
115119
120@error_status(http_client.BAD_REQUEST)
121class NoOCIAdminForDistribution(Exception):
122 """There is no OCI Project Admin for this distribution."""
123
124 def __init__(self):
125 super(NoOCIAdminForDistribution, self).__init__(
126 "There is no OCI Project Admin for this distribution.")
127
128
116class IDistributionMirrorMenuMarker(Interface):129class IDistributionMirrorMenuMarker(Interface):
117 """Marker interface for Mirror navigation."""130 """Marker interface for Mirror navigation."""
118131
@@ -129,6 +142,35 @@ class DistributionNameField(PillarNameField):
129class IDistributionEditRestricted(IOfficialBugTagTargetRestricted):142class IDistributionEditRestricted(IOfficialBugTagTargetRestricted):
130 """IDistribution properties requiring launchpad.Edit permission."""143 """IDistribution properties requiring launchpad.Edit permission."""
131144
145 @call_with(registrant=REQUEST_USER)
146 @operation_parameters(
147 registry_url=TextLine(
148 title=_("The registry url."),
149 description=_("The url of the OCI registry to use."),
150 required=True),
151 region=TextLine(
152 title=_("OCI registry region."),
153 description=_("The region of the OCI registry."),
154 required=False),
155 username=TextLine(
156 title=_("Username"),
157 description=_("The username for the OCI registry."),
158 required=False),
159 password=TextLine(
160 title=_("Password"),
161 description=_("The password for the OCI registry."),
162 required=False))
163 @export_write_operation()
164 @operation_for_version("devel")
165 def setOCICredentials(registrant, registry_url, region,
166 username, password):
167 """Set the credentials for the OCI registry for OCI projects."""
168
169 @export_write_operation()
170 @operation_for_version("devel")
171 def deleteOCICredentials():
172 """Delete any existing OCI credentials for the distribution."""
173
132174
133class IDistributionDriverRestricted(Interface):175class IDistributionDriverRestricted(Interface):
134 """IDistribution properties requiring launchpad.Driver permission."""176 """IDistribution properties requiring launchpad.Driver permission."""
@@ -727,7 +769,6 @@ class IDistributionPublic(
727 "images in this distribution to a registry."),769 "images in this distribution to a registry."),
728 required=False, readonly=False)770 required=False, readonly=False)
729771
730
731@exported_as_webservice_entry(as_of="beta")772@exported_as_webservice_entry(as_of="beta")
732class IDistribution(773class IDistribution(
733 IDistributionEditRestricted, IDistributionPublic, IHasBugSupervisor,774 IDistributionEditRestricted, IDistributionPublic, IHasBugSupervisor,
diff --git a/lib/lp/registry/model/distribution.py b/lib/lp/registry/model/distribution.py
index 0288c54..f76d28b 100644
--- a/lib/lp/registry/model/distribution.py
+++ b/lib/lp/registry/model/distribution.py
@@ -89,6 +89,7 @@ from lp.bugs.model.structuralsubscription import (
89from lp.code.interfaces.seriessourcepackagebranch import (89from lp.code.interfaces.seriessourcepackagebranch import (
90 IFindOfficialBranchLinks,90 IFindOfficialBranchLinks,
91 )91 )
92from lp.oci.interfaces.ociregistrycredentials import IOCIRegistryCredentialsSet
92from lp.registry.enums import (93from lp.registry.enums import (
93 BranchSharingPolicy,94 BranchSharingPolicy,
94 BugSharingPolicy,95 BugSharingPolicy,
@@ -101,6 +102,7 @@ from lp.registry.interfaces.accesspolicy import IAccessPolicySource
101from lp.registry.interfaces.distribution import (102from lp.registry.interfaces.distribution import (
102 IDistribution,103 IDistribution,
103 IDistributionSet,104 IDistributionSet,
105 NoOCIAdminForDistribution,
104 )106 )
105from lp.registry.interfaces.distributionmirror import (107from lp.registry.interfaces.distributionmirror import (
106 IDistributionMirror,108 IDistributionMirror,
@@ -1531,6 +1533,32 @@ class Distribution(SQLBase, BugTargetBase, MakesAnnouncements,
1531 pillar=self, registrant=registrant, name=name,1533 pillar=self, registrant=registrant, name=name,
1532 description=description)1534 description=description)
15331535
1536 def setOCICredentials(self, registrant, registry_url,
1537 region, username, password):
1538 """See `IDistribution`."""
1539 if not self.oci_project_admin:
1540 raise NoOCIAdminForDistribution()
1541 new_credentials = getUtility(IOCIRegistryCredentialsSet).getOrCreate(
1542 registrant,
1543 self.oci_project_admin,
1544 registry_url,
1545 {"username": username, "password": password, "region": region},
1546 override_owner=True)
1547 old_credentials = self.oci_registry_credentials
1548 if self.oci_registry_credentials != new_credentials:
1549 # Remove the old credentials as we're assigning new ones
1550 # or clearing them
1551 self.oci_registry_credentials = new_credentials
1552 if old_credentials:
1553 old_credentials.destroySelf()
1554
1555 def deleteOCICredentials(self):
1556 """See `IDistribution`."""
1557 old_credentials = self.oci_registry_credentials
1558 if old_credentials:
1559 self.oci_registry_credentials = None
1560 old_credentials.destroySelf()
1561
15341562
1535@implementer(IDistributionSet)1563@implementer(IDistributionSet)
1536class DistributionSet:1564class DistributionSet:
diff --git a/lib/lp/registry/scripts/closeaccount.py b/lib/lp/registry/scripts/closeaccount.py
index 27b2eb1..b4e505b 100644
--- a/lib/lp/registry/scripts/closeaccount.py
+++ b/lib/lp/registry/scripts/closeaccount.py
@@ -362,12 +362,9 @@ def close_account(username, log):
362 # the placeholder person row.362 # the placeholder person row.
363 skip.add(('sprintattendance', 'attendee'))363 skip.add(('sprintattendance', 'attendee'))
364364
365 # generate_ppa_htaccess currently relies on seeing active365 # PPA authorization is now handled dynamically and checks the
366 # ArchiveAuthToken rows so that it knows which ones to remove from366 # subscriber's account status, so this isn't strictly necessary, but
367 # .htpasswd files on disk in response to the cancellation of the367 # it's still nice to have the per-person audit trail.
368 # corresponding ArchiveSubscriber rows; but even once PPA authorisation
369 # is handled dynamically, we probably still want to have the per-person
370 # audit trail here.
371 archive_subscriber_ids = set(store.find(368 archive_subscriber_ids = set(store.find(
372 ArchiveSubscriber.id,369 ArchiveSubscriber.id,
373 ArchiveSubscriber.subscriber_id == person.id,370 ArchiveSubscriber.subscriber_id == person.id,
diff --git a/lib/lp/registry/tests/test_distribution.py b/lib/lp/registry/tests/test_distribution.py
index 0b9f712..005a7e6 100644
--- a/lib/lp/registry/tests/test_distribution.py
+++ b/lib/lp/registry/tests/test_distribution.py
@@ -28,6 +28,7 @@ from lp.app.enums import (
28 )28 )
29from lp.app.errors import NotFoundError29from lp.app.errors import NotFoundError
30from lp.app.interfaces.launchpad import ILaunchpadCelebrities30from lp.app.interfaces.launchpad import ILaunchpadCelebrities
31from lp.oci.tests.helpers import OCIConfigHelperMixin
31from lp.registry.enums import (32from lp.registry.enums import (
32 BranchSharingPolicy,33 BranchSharingPolicy,
33 BugSharingPolicy,34 BugSharingPolicy,
@@ -761,7 +762,7 @@ class DistributionOCIProjectAdminPermission(TestCaseWithFactory):
761 self.assertTrue(distro.canAdministerOCIProjects(admin))762 self.assertTrue(distro.canAdministerOCIProjects(admin))
762763
763764
764class TestDistributionWebservice(TestCaseWithFactory):765class TestDistributionWebservice(OCIConfigHelperMixin, TestCaseWithFactory):
765 """Test the IDistribution API.766 """Test the IDistribution API.
766767
767 Some tests already exist in xx-distribution.txt.768 Some tests already exist in xx-distribution.txt.
@@ -842,3 +843,92 @@ class TestDistributionWebservice(TestCaseWithFactory):
842 start_date=(now - day).isoformat(),843 start_date=(now - day).isoformat(),
843 end_date=now.isoformat())844 end_date=now.isoformat())
844 self.assertEqual([], empty_response.jsonBody())845 self.assertEqual([], empty_response.jsonBody())
846
847 def test_setOCICredentials(self):
848 # We can add OCI Credentials to the distribution
849 self.setConfig()
850 with person_logged_in(self.person):
851 distro = self.factory.makeDistribution(owner=self.person)
852 distro.oci_project_admin = self.person
853 distro_url = api_url(distro)
854
855 resp = self.webservice.named_post(
856 distro_url,
857 "setOCICredentials",
858 registry_url="http://registry.test",
859 username="test-username",
860 password="test-password",
861 region="test-region"
862 )
863
864 self.assertEqual(200, resp.status)
865 with person_logged_in(self.person):
866 self.assertEqual(
867 "http://registry.test",
868 distro.oci_registry_credentials.url
869 )
870 credentials = distro.oci_registry_credentials.getCredentials()
871 self.assertDictEqual({
872 "username": "test-username",
873 "password": "test-password",
874 "region": "test-region"},
875 credentials)
876
877 def test_setOCICredentials_no_oci_admin(self):
878 # If there's no oci_project_admin to own the credentials, error
879 self.setConfig()
880 with person_logged_in(self.person):
881 distro = self.factory.makeDistribution(owner=self.person)
882 distro_url = api_url(distro)
883
884 resp = self.webservice.named_post(
885 distro_url,
886 "setOCICredentials",
887 registry_url="http://registry.test",
888 )
889
890 self.assertEqual(400, resp.status)
891 self.assertIn(
892 b"no OCI Project Admin for this distribution",
893 resp.body)
894
895 def test_setOCICredentials_changes_credentials(self):
896 # if we have existing credentials, we should change them
897 self.setConfig()
898 with person_logged_in(self.person):
899 distro = self.factory.makeDistribution(owner=self.person)
900 distro.oci_project_admin = self.person
901 credentials = self.factory.makeOCIRegistryCredentials()
902 distro.oci_registry_credentials = credentials
903 distro_url = api_url(distro)
904
905 resp = self.webservice.named_post(
906 distro_url,
907 "setOCICredentials",
908 registry_url="http://registry.test",
909 )
910
911 self.assertEqual(200, resp.status)
912 with person_logged_in(self.person):
913 self.assertEqual(
914 "http://registry.test",
915 distro.oci_registry_credentials.url
916 )
917
918 def test_deleteOCICredentials(self):
919 # We can remove existing credentials
920 self.setConfig()
921 with person_logged_in(self.person):
922 distro = self.factory.makeDistribution(owner=self.person)
923 distro.oci_project_admin = self.person
924 credentials = self.factory.makeOCIRegistryCredentials()
925 distro.oci_registry_credentials = credentials
926 distro_url = api_url(distro)
927
928 resp = self.webservice.named_post(
929 distro_url,
930 "deleteOCICredentials")
931
932 self.assertEqual(200, resp.status)
933 with person_logged_in(self.person):
934 self.assertIsNone(distro.oci_registry_credentials)
diff --git a/lib/lp/registry/tests/test_personmerge.py b/lib/lp/registry/tests/test_personmerge.py
index a1c00be..5c75436 100644
--- a/lib/lp/registry/tests/test_personmerge.py
+++ b/lib/lp/registry/tests/test_personmerge.py
@@ -719,8 +719,8 @@ class TestMergePeople(TestCaseWithFactory, KarmaTestMixin):
719 self.useFixture(FeatureFixture({OCI_RECIPE_ALLOW_CREATE: 'on'}))719 self.useFixture(FeatureFixture({OCI_RECIPE_ALLOW_CREATE: 'on'}))
720 duplicate = self.factory.makePerson()720 duplicate = self.factory.makePerson()
721 mergee = self.factory.makePerson()721 mergee = self.factory.makePerson()
722 [ref] = self.factory.makeGitRefs(paths=['refs/heads/v1.0-20.04'])722 [ref] = self.factory.makeGitRefs(paths=[u'refs/heads/v1.0-20.04'])
723 [ref2] = self.factory.makeGitRefs(paths=['refs/heads/v1.0-20.04'])723 [ref2] = self.factory.makeGitRefs(paths=[u'refs/heads/v1.0-20.04'])
724 self.factory.makeOCIRecipe(724 self.factory.makeOCIRecipe(
725 registrant=duplicate, owner=duplicate, name=u'foo', git_ref=ref)725 registrant=duplicate, owner=duplicate, name=u'foo', git_ref=ref)
726 self.factory.makeOCIRecipe(726 self.factory.makeOCIRecipe(
diff --git a/lib/lp/services/scripts/base.py b/lib/lp/services/scripts/base.py
index bb4490b..66ebdbf 100644
--- a/lib/lp/services/scripts/base.py
+++ b/lib/lp/services/scripts/base.py
@@ -406,10 +406,6 @@ class LaunchpadCronScript(LaunchpadScript):
406 oops_hdlr = OopsHandler(self.name, logger=self.logger)406 oops_hdlr = OopsHandler(self.name, logger=self.logger)
407 logging.getLogger().addHandler(oops_hdlr)407 logging.getLogger().addHandler(oops_hdlr)
408408
409 def get_last_activity(self):
410 """Return the last activity, if any."""
411 return getUtility(IScriptActivitySet).getLastActivity(self.name)
412
413 @log_unhandled_exception_and_exit409 @log_unhandled_exception_and_exit
414 def record_activity(self, date_started, date_completed):410 def record_activity(self, date_started, date_completed):
415 """Record the successful completion of the script."""411 """Record the successful completion of the script."""
diff --git a/lib/lp/snappy/model/snap.py b/lib/lp/snappy/model/snap.py
index 58dc398..1a108b0 100644
--- a/lib/lp/snappy/model/snap.py
+++ b/lib/lp/snappy/model/snap.py
@@ -1187,13 +1187,6 @@ class Snap(Storm, WebhookTargetMixin):
1187 person.is_team and1187 person.is_team and
1188 person.anyone_can_join())1188 person.anyone_can_join())
11891189
1190 @property
1191 def subscribers(self):
1192 return Store.of(self).find(
1193 Person,
1194 SnapSubscription.person_id == Person.id,
1195 SnapSubscription.snap == self)
1196
1197 def subscribe(self, person, subscribed_by, ignore_permissions=False):1190 def subscribe(self, person, subscribed_by, ignore_permissions=False):
1198 """See `ISnap`."""1191 """See `ISnap`."""
1199 if not self.userCanBeSubscribed(person):1192 if not self.userCanBeSubscribed(person):
diff --git a/lib/lp/soyuz/scripts/expire_archive_files.py b/lib/lp/soyuz/scripts/expire_archive_files.py
index ade45d5..7ae54e8 100755
--- a/lib/lp/soyuz/scripts/expire_archive_files.py
+++ b/lib/lp/soyuz/scripts/expire_archive_files.py
@@ -49,6 +49,9 @@ netbook-remix-team
49netbook-team49netbook-team
50oem-solutions-group50oem-solutions-group
51payson51payson
52snappy-dev/edge
53snappy-dev/image
54snappy-dev/tools
52transyl55transyl
53ubuntu-cloud-archive56ubuntu-cloud-archive
54ubuntu-mobile57ubuntu-mobile
diff --git a/lib/lp/testing/layers.py b/lib/lp/testing/layers.py
index 1060fa9..541ef56 100644
--- a/lib/lp/testing/layers.py
+++ b/lib/lp/testing/layers.py
@@ -1899,6 +1899,24 @@ class CeleryJobLayer(AppServerLayer):
1899 cls.celery_worker = None1899 cls.celery_worker = None
19001900
19011901
1902class CelerySlowJobLayer(AppServerLayer):
1903 """Layer for tests that run jobs via Celery."""
1904
1905 celery_worker = None
1906
1907 @classmethod
1908 @profiled
1909 def setUp(cls):
1910 cls.celery_worker = celery_worker('launchpad_job_slow')
1911 cls.celery_worker.__enter__()
1912
1913 @classmethod
1914 @profiled
1915 def tearDown(cls):
1916 cls.celery_worker.__exit__(None, None, None)
1917 cls.celery_worker = None
1918
1919
1902class CeleryBzrsyncdJobLayer(AppServerLayer):1920class CeleryBzrsyncdJobLayer(AppServerLayer):
1903 """Layer for tests that run jobs that read from branches via Celery."""1921 """Layer for tests that run jobs that read from branches via Celery."""
19041922
diff --git a/utilities/manage-celery-workers.sh b/utilities/manage-celery-workers.sh
1905new file mode 1007551923new file mode 100755
index 0000000..f83b14f
--- /dev/null
+++ b/utilities/manage-celery-workers.sh
@@ -0,0 +1,58 @@
1#!/bin/sh
2
3# Used for dev and dogfood, do not use in a production like environment.
4
5start_worker() {
6 # Start a worker for a given queue
7 queue=$1
8 echo "Starting worker for $queue"
9 start-stop-daemon \
10 --start --oknodo --quiet --background \
11 --pidfile "/var/tmp/celeryd-$queue.pid" --make-pidfile \
12 --startas "$PWD/bin/celery" -- worker \
13 --queues="$queue"\
14 --config=lp.services.job.celeryconfig \
15 --hostname="$queue@%n" \
16 --loglevel=DEBUG \
17 --logfile="/var/tmp/celeryd-$queue.log"
18
19}
20
21stop_worker() {
22 queue=$1
23 echo "Stopping worker for $queue"
24 start-stop-daemon --oknodo --stop --pidfile "/var/tmp/celeryd-$queue.pid"
25}
26
27case "$1" in
28 start)
29 for queue in launchpad_job launchpad_job_slow bzrsyncd_job bzrsyncd_job_slow branch_write_job branch_write_job_slow celerybeat
30 do
31 start_worker $queue
32 done
33 ;;
34 stop)
35 for queue in launchpad_job launchpad_job_slow bzrsyncd_job bzrsyncd_job_slow branch_write_job branch_write_job_slow celerybeat
36 do
37 stop_worker $queue
38 done
39 ;;
40
41 restart|force-reload)
42 for queue in launchpad_job launchpad_job_slow bzrsyncd_job bzrsyncd_job_slow branch_write_job branch_write_job_slow celerybeat
43 do
44 stop_worker $queue
45 done
46 sleep 1
47 for queue in launchpad_job launchpad_job_slow bzrsyncd_job bzrsyncd_job_slow branch_write_job branch_write_job_slow celerybeat
48 do
49 start_worker $queue
50 done
51 echo "$NAME."
52 ;;
53 *)
54 N=/etc/init.d/$NAME
55 echo "Usage: $N {start|stop|restart|force-reload}" >&2
56 exit 1
57 ;;
58esac

Subscribers

People subscribed via source and target branches

to status/vote changes: