Merge lp:~cjwatson/launchpad/loggerhead-gunicorn into lp:launchpad

Proposed by Colin Watson
Status: Merged
Merged at revision: 18685
Proposed branch: lp:~cjwatson/launchpad/loggerhead-gunicorn
Merge into: lp:launchpad
Diff against target: 841 lines (+334/-336)
12 files modified
Makefile (+2/-2)
constraints.txt (+2/-0)
lib/launchpad_loggerhead/debug.py (+0/-120)
lib/launchpad_loggerhead/testing.py (+72/-0)
lib/launchpad_loggerhead/tests.py (+70/-8)
lib/launchpad_loggerhead/wsgi.py (+159/-0)
lib/lp/services/osutils.py (+0/-1)
lib/lp/services/scripts/tests/__init__.py (+7/-2)
lib/lp/testing/__init__.py (+2/-1)
scripts/start-loggerhead.py (+8/-186)
scripts/stop-loggerhead.py (+11/-16)
setup.py (+1/-0)
To merge this branch: bzr merge lp:~cjwatson/launchpad/loggerhead-gunicorn
Reviewer Review Type Date Requested Status
William Grant code Approve
Review via email: mp+347387@code.launchpad.net

Commit message

Port the loggerhead integration to gunicorn.

Description of the change

Split out from https://code.launchpad.net/~cjwatson/launchpad/private-loggerhead/+merge/345680 without the private port stuff, as requested by William. See that MP for rationale and comments.

To post a comment you must log in.
Revision history for this message
William Grant (wgrant) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'Makefile'
2--- Makefile 2018-05-21 20:30:16 +0000
3+++ Makefile 2018-06-06 08:58:34 +0000
4@@ -306,10 +306,10 @@
5 memcached,rabbitmq,txlongpoll -i $(LPCONFIG)
6
7 run_codebrowse: compile
8- BZR_PLUGIN_PATH=bzrplugins $(PY) scripts/start-loggerhead.py -f
9+ BZR_PLUGIN_PATH=bzrplugins $(PY) scripts/start-loggerhead.py
10
11 start_codebrowse: compile
12- BZR_PLUGIN_PATH=$(shell pwd)/bzrplugins $(PY) scripts/start-loggerhead.py
13+ BZR_PLUGIN_PATH=$(shell pwd)/bzrplugins $(PY) scripts/start-loggerhead.py --daemon
14
15 stop_codebrowse:
16 $(PY) scripts/stop-loggerhead.py
17
18=== modified file 'constraints.txt'
19--- constraints.txt 2018-05-31 13:12:39 +0000
20+++ constraints.txt 2018-06-06 08:58:34 +0000
21@@ -257,7 +257,9 @@
22 feedvalidator==0.0.0DEV-r1049
23 fixtures==3.0.0
24 FormEncode==1.2.4
25+futures==3.2.0
26 grokcore.component==1.6
27+gunicorn==19.8.1
28 html5browser==0.0.9
29 httplib2==0.8
30 hyperlink==18.0.0
31
32=== removed file 'lib/launchpad_loggerhead/debug.py'
33--- lib/launchpad_loggerhead/debug.py 2010-04-27 01:35:56 +0000
34+++ lib/launchpad_loggerhead/debug.py 1970-01-01 00:00:00 +0000
35@@ -1,120 +0,0 @@
36-# Copyright 2009 Canonical Ltd. This software is licensed under the
37-# GNU Affero General Public License version 3 (see the file LICENSE).
38-
39-import thread
40-import time
41-
42-from paste.request import construct_url
43-
44-
45-def tabulate(cells):
46- """Format a list of lists of strings in a table.
47-
48- The 'cells' are centered.
49-
50- >>> print ''.join(tabulate(
51- ... [['title 1', 'title 2'],
52- ... ['short', 'rather longer']]))
53- title 1 title 2
54- short rather longer
55- """
56- widths = {}
57- for row in cells:
58- for col_index, cell in enumerate(row):
59- widths[col_index] = max(len(cell), widths.get(col_index, 0))
60- result = []
61- for row in cells:
62- result_row = ''
63- for col_index, cell in enumerate(row):
64- result_row += cell.center(widths[col_index] + 2)
65- result.append(result_row.rstrip() + '\n')
66- return result
67-
68-
69-def threadpool_debug(app):
70- """Wrap `app` to provide debugging information about the threadpool state.
71-
72- The returned application will serve debugging information about the state
73- of the threadpool at '/thread-debug' -- but only when accessed directly,
74- not when accessed through Apache.
75- """
76- def wrapped(environ, start_response):
77- if ('HTTP_X_FORWARDED_SERVER' in environ
78- or environ['PATH_INFO'] != '/thread-debug'):
79- environ['lp.timestarted'] = time.time()
80- return app(environ, start_response)
81- threadpool = environ['paste.httpserver.thread_pool']
82- start_response("200 Ok", [])
83- output = [("url", "time running", "time since last activity")]
84- now = time.time()
85- # Because we're accessing mutable structures without locks here,
86- # we're a bit cautious about things looking like we expect -- if a
87- # worker doesn't seem fully set up, we just ignore it.
88- for worker in threadpool.workers:
89- if not hasattr(worker, 'thread_id'):
90- continue
91- time_started, info = threadpool.worker_tracker.get(
92- worker.thread_id, (None, None))
93- if time_started is not None and info is not None:
94- real_time_started = info.get(
95- 'lp.timestarted', time_started)
96- output.append(
97- map(str,
98- (construct_url(info),
99- now - real_time_started,
100- now - time_started,)))
101- return tabulate(output)
102- return wrapped
103-
104-
105-def change_kill_thread_criteria(application):
106- """Interfere with threadpool so that threads are killed for inactivity.
107-
108- The usual rules with paste's threadpool is that a thread that takes longer
109- than 'hung_thread_limit' seconds to process a request is considered hung
110- and more than 'kill_thread_limit' seconds is killed.
111-
112- Because loggerhead streams its output, how long the entire request takes
113- to process depends on things like how fast the users internet connection
114- is. What we'd like to do is kill threads that don't _start_ to produce
115- output for 'kill_thread_limit' seconds.
116-
117- What this class actually does is arrange things so that threads that
118- produce no output for 'kill_thread_limit' are killed, because that's the
119- rule Apache uses when interpreting ProxyTimeout.
120- """
121- def wrapped_application(environ, start_response):
122- threadpool = environ['paste.httpserver.thread_pool']
123- def reset_timer():
124- """Make this thread safe for another 'kill_thread_limit' seconds.
125-
126- We do this by hacking the threadpool's record of when this thread
127- started to pretend that it started right now. Hacky, but it's
128- enough to fool paste.httpserver.ThreadPool.kill_hung_threads and
129- that's what matters.
130- """
131- threadpool.worker_tracker[thread.get_ident()][0] = time.time()
132- def response_hook(status, response_headers, exc_info=None):
133- # We reset the timer when the HTTP headers are sent...
134- reset_timer()
135- writer = start_response(status, response_headers, exc_info)
136- def wrapped_writer(arg):
137- # ... and whenever more output has been generated.
138- reset_timer()
139- return writer(arg)
140- return wrapped_writer
141- result = application(environ, response_hook)
142- # WSGI allows the application to return an iterable, which could be a
143- # generator that does significant processing between successive items,
144- # so we should reset the timer between each item.
145- #
146- # This isn't really necessary as loggerhead doesn't return any
147- # non-trivial iterables to the WSGI server. But it's probably better
148- # to cope with this case to avoid nasty suprises if loggerhead
149- # changes.
150- def reset_timer_between_items(iterable):
151- for item in iterable:
152- reset_timer()
153- yield item
154- return reset_timer_between_items(result)
155- return wrapped_application
156
157=== added file 'lib/launchpad_loggerhead/testing.py'
158--- lib/launchpad_loggerhead/testing.py 1970-01-01 00:00:00 +0000
159+++ lib/launchpad_loggerhead/testing.py 2018-06-06 08:58:34 +0000
160@@ -0,0 +1,72 @@
161+# Copyright 2018 Canonical Ltd. This software is licensed under the
162+# GNU Affero General Public License version 3 (see the file LICENSE).
163+
164+from __future__ import absolute_import, print_function, unicode_literals
165+
166+__metaclass__ = type
167+__all__ = [
168+ 'LoggerheadFixture',
169+ ]
170+
171+import os.path
172+import time
173+import warnings
174+
175+from fixtures import Fixture
176+
177+from lp.services.config import config
178+from lp.services.osutils import (
179+ get_pid_from_file,
180+ kill_by_pidfile,
181+ remove_if_exists,
182+ )
183+from lp.services.pidfile import pidfile_path
184+from lp.services.scripts.tests import run_script
185+from lp.testing.layers import (
186+ BaseLayer,
187+ LayerProcessController,
188+ )
189+
190+
191+class LoggerheadFixtureException(Exception):
192+ pass
193+
194+
195+class LoggerheadFixture(Fixture):
196+ """Start loggerhead as a fixture."""
197+
198+ def _setUp(self):
199+ pidfile = pidfile_path(
200+ "codebrowse", use_config=LayerProcessController.appserver_config)
201+ pid = get_pid_from_file(pidfile)
202+ if pid is not None:
203+ warnings.warn(
204+ "Attempt to start LoggerheadFixture with an existing "
205+ "instance (%d) running in %s." % (pid, pidfile))
206+ kill_by_pidfile(pidfile)
207+ self.logfile = os.path.join(config.codebrowse.log_folder, "debug.log")
208+ remove_if_exists(self.logfile)
209+ self.addCleanup(kill_by_pidfile, pidfile)
210+ run_script(
211+ os.path.join("scripts", "start-loggerhead.py"), ["--daemon"],
212+ # The testrunner-appserver config provides the correct
213+ # openid_provider_root URL.
214+ extra_env={"LPCONFIG": BaseLayer.appserver_config_name})
215+ self._waitForStartup()
216+
217+ def _hasStarted(self):
218+ if os.path.exists(self.logfile):
219+ with open(self.logfile) as logfile:
220+ return "Listening at:" in logfile.read()
221+ else:
222+ return False
223+
224+ def _waitForStartup(self):
225+ now = time.time()
226+ deadline = now + 20
227+ while now < deadline and not self._hasStarted():
228+ time.sleep(0.1)
229+ now = time.time()
230+
231+ if now >= deadline:
232+ raise LoggerheadFixtureException("Unable to start loggerhead.")
233
234=== modified file 'lib/launchpad_loggerhead/tests.py'
235--- lib/launchpad_loggerhead/tests.py 2018-01-19 17:21:44 +0000
236+++ lib/launchpad_loggerhead/tests.py 2018-06-06 08:58:34 +0000
237@@ -1,29 +1,44 @@
238 # Copyright 2010-2018 Canonical Ltd. This software is licensed under the
239 # GNU Affero General Public License version 3 (see the file LICENSE).
240
241-import urllib
242-
243 import lazr.uri
244 from paste.httpexceptions import HTTPExceptionHandler
245+import requests
246+from six.moves.urllib_parse import (
247+ urlencode,
248+ urlsplit,
249+ )
250+import soupmatchers
251+from testtools.content import Content
252+from testtools.content_type import UTF8_TEXT
253 import wsgi_intercept
254 from wsgi_intercept.urllib2_intercept import (
255 install_opener,
256 uninstall_opener,
257 )
258 import wsgi_intercept.zope_testbrowser
259+from zope.security.proxy import removeSecurityProxy
260
261 from launchpad_loggerhead.app import RootApp
262 from launchpad_loggerhead.session import SessionHandler
263+from launchpad_loggerhead.testing import LoggerheadFixture
264+from lp.app.enums import InformationType
265 from lp.services.config import config
266 from lp.services.webapp.vhosts import allvhosts
267-from lp.testing import TestCase
268-from lp.testing.layers import DatabaseFunctionalLayer
269+from lp.testing import (
270+ TestCase,
271+ TestCaseWithFactory,
272+ )
273+from lp.testing.layers import (
274+ AppServerLayer,
275+ DatabaseFunctionalLayer,
276+ )
277
278
279 SESSION_VAR = 'lh.session'
280
281-# See sourcecode/launchpad-loggerhead/start-loggerhead.py for the production
282-# mechanism for getting the secret.
283+# See lib/launchpad_loggerhead/wsgi.py for the production mechanism for
284+# getting the secret.
285 SECRET = 'secret'
286
287
288@@ -132,8 +147,7 @@
289 self.intercept(dummy_root, dummy_destination)
290 self.browser.open(
291 config.codehosting.secure_codebrowse_root +
292- '+logout?' +
293- urllib.urlencode(dict(next_to=dummy_root + '+logout')))
294+ '+logout?' + urlencode(dict(next_to=dummy_root + '+logout')))
295
296 # We are logged out, as before.
297 self.assertEqual(self.session, {})
298@@ -142,3 +156,51 @@
299 self.assertEqual(self.browser.url, dummy_root + '+logout')
300 self.assertEqual(self.browser.contents,
301 'This is a dummy destination.\n')
302+
303+
304+class TestWSGI(TestCaseWithFactory):
305+ """Smoke tests for Launchpad's loggerhead WSGI server."""
306+
307+ layer = AppServerLayer
308+
309+ def setUp(self):
310+ super(TestWSGI, self).setUp()
311+ self.useBzrBranches()
312+ loggerhead_fixture = self.useFixture(LoggerheadFixture())
313+
314+ def get_debug_log_bytes():
315+ try:
316+ with open(loggerhead_fixture.logfile, "rb") as logfile:
317+ return [logfile.read()]
318+ except IOError:
319+ return [b""]
320+
321+ self.addDetail(
322+ "loggerhead-debug", Content(UTF8_TEXT, get_debug_log_bytes))
323+
324+ def test_public_port_public_branch(self):
325+ # Requests for public branches on the public port are allowed.
326+ db_branch, _ = self.create_branch_and_tree()
327+ branch_url = "http://127.0.0.1:%d/%s" % (
328+ config.codebrowse.port, db_branch.unique_name)
329+ response = requests.get(branch_url)
330+ self.assertEqual(200, response.status_code)
331+ title_tag = soupmatchers.Tag(
332+ "page title", "title", text="%s : changes" % db_branch.unique_name)
333+ self.assertThat(response.text, soupmatchers.HTMLContains(title_tag))
334+
335+ def test_public_port_private_branch(self):
336+ # Requests for private branches on the public port send the user
337+ # through the login workflow.
338+ db_branch, _ = self.create_branch_and_tree(
339+ information_type=InformationType.USERDATA)
340+ naked_branch = removeSecurityProxy(db_branch)
341+ branch_url = "http://127.0.0.1:%d/%s" % (
342+ config.codebrowse.port, naked_branch.unique_name)
343+ response = requests.get(
344+ branch_url, headers={"X-Forwarded-Scheme": "https"},
345+ allow_redirects=False)
346+ self.assertEqual(301, response.status_code)
347+ self.assertEqual(
348+ "testopenid.dev:8085",
349+ urlsplit(response.headers["Location"]).netloc)
350
351=== added file 'lib/launchpad_loggerhead/wsgi.py'
352--- lib/launchpad_loggerhead/wsgi.py 1970-01-01 00:00:00 +0000
353+++ lib/launchpad_loggerhead/wsgi.py 2018-06-06 08:58:34 +0000
354@@ -0,0 +1,159 @@
355+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
356+# GNU Affero General Public License version 3 (see the file LICENSE).
357+
358+from __future__ import absolute_import, print_function, unicode_literals
359+
360+__metaclass__ = type
361+__all__ = [
362+ 'LoggerheadApplication',
363+ ]
364+
365+import logging
366+from optparse import OptionParser
367+import os.path
368+import time
369+import traceback
370+
371+from gunicorn.app.base import Application
372+from gunicorn.glogging import Logger
373+from openid import oidutil
374+from paste.deploy.config import PrefixMiddleware
375+from paste.httpexceptions import HTTPExceptionHandler
376+from paste.request import construct_url
377+from paste.wsgilib import catch_errors
378+
379+from launchpad_loggerhead.app import (
380+ oops_middleware,
381+ RootApp,
382+ )
383+from launchpad_loggerhead.session import SessionHandler
384+import lp.codehosting
385+from lp.services.config import config
386+from lp.services.pidfile import pidfile_path
387+from lp.services.scripts import (
388+ logger,
389+ logger_options,
390+ )
391+from lp.services.scripts.logger import LaunchpadFormatter
392+
393+
394+log = logging.getLogger("loggerhead")
395+
396+
397+SESSION_VAR = "lh.session"
398+
399+
400+def log_request_start_and_stop(app):
401+ def wrapped(environ, start_response):
402+ url = construct_url(environ)
403+ log.info("Starting to process %s", url)
404+ start_time = time.time()
405+
406+ def request_done_ok():
407+ log.info(
408+ "Processed ok %s [%0.3f seconds]",
409+ url, time.time() - start_time)
410+
411+ def request_done_err(exc_info):
412+ log.info(
413+ "Processed err %s [%0.3f seconds]: %s",
414+ url, time.time() - start_time,
415+ traceback.format_exception_only(*exc_info[:2]))
416+
417+ return catch_errors(
418+ app, environ, start_response, request_done_err, request_done_ok)
419+
420+ return wrapped
421+
422+
423+class LoggerheadLogger(Logger):
424+
425+ def setup(self, cfg):
426+ super(LoggerheadLogger, self).setup(cfg)
427+ formatter = LaunchpadFormatter(datefmt=None)
428+ for handler in self.error_log.handlers:
429+ handler.setFormatter(formatter)
430+
431+ # Force Launchpad's logging machinery to set up the root logger the
432+ # way we want it.
433+ parser = OptionParser()
434+ logger_options(parser)
435+ log_options, _ = parser.parse_args(
436+ ['-q', '--ms', '--log-file=DEBUG:%s' % cfg.errorlog])
437+ logger(log_options)
438+
439+ # Make the OpenID library use proper logging rather than writing to
440+ # stderr.
441+ oidutil.log = lambda message, level=0: log.debug(message)
442+
443+
444+class LoggerheadApplication(Application):
445+
446+ def __init__(self, **kwargs):
447+ self.options = kwargs
448+ super(LoggerheadApplication, self).__init__()
449+
450+ def init(self, parser, opts, args):
451+ top = os.path.abspath(os.path.join(
452+ os.path.dirname(__file__), os.pardir, os.pardir))
453+ listen_host = config.codebrowse.listen_host
454+ log_folder = config.codebrowse.log_folder or os.path.join(top, "logs")
455+ if not os.path.exists(log_folder):
456+ os.makedirs(log_folder)
457+
458+ cfg = {
459+ "accesslog": os.path.join(log_folder, "access.log"),
460+ "bind": [
461+ "%s:%s" % (listen_host, config.codebrowse.port),
462+ ],
463+ "errorlog": os.path.join(log_folder, "debug.log"),
464+ # Trust that firewalls only permit sending requests to
465+ # loggerhead via a frontend.
466+ "forwarded_allow_ips": "*",
467+ "logger_class": "launchpad_loggerhead.wsgi.LoggerheadLogger",
468+ "loglevel": "debug",
469+ "pidfile": pidfile_path("codebrowse"),
470+ "preload_app": True,
471+ # XXX cjwatson 2018-05-15: These are gunicorn defaults plus
472+ # X-Forwarded-Scheme: https, which we use in staging/production.
473+ # We should switch the staging/production configuration to
474+ # something that gunicorn understands natively and then drop
475+ # this.
476+ "secure_scheme_headers": {
477+ "X-FORWARDED-PROTOCOL": "ssl",
478+ "X-FORWARDED-PROTO": "https",
479+ "X-FORWARDED-SCHEME": "https",
480+ "X-FORWARDED-SSL": "on",
481+ },
482+ # Kill threads after 300 seconds of inactivity. This is
483+ # insanely high, but loggerhead is often pretty slow.
484+ "timeout": 300,
485+ "threads": 10,
486+ "worker_class": "gthread",
487+ }
488+ cfg.update(self.options)
489+ return cfg
490+
491+ def _load_bzr_plugins(self):
492+ from bzrlib.plugin import load_plugins
493+ load_plugins()
494+
495+ import bzrlib.plugins
496+ if getattr(bzrlib.plugins, "loom", None) is None:
497+ log.error("Loom plugin loading failed.")
498+
499+ def load(self):
500+ self._load_bzr_plugins()
501+
502+ with open(os.path.join(
503+ config.root, config.codebrowse.secret_path)) as secret_file:
504+ secret = secret_file.read()
505+
506+ app = RootApp(SESSION_VAR)
507+ app = HTTPExceptionHandler(app)
508+ app = SessionHandler(app, SESSION_VAR, secret)
509+ app = log_request_start_and_stop(app)
510+ app = PrefixMiddleware(app)
511+ app = oops_middleware(app)
512+
513+ return app
514
515=== modified file 'lib/lp/services/osutils.py'
516--- lib/lp/services/osutils.py 2018-03-27 17:43:27 +0000
517+++ lib/lp/services/osutils.py 2018-06-06 08:58:34 +0000
518@@ -9,7 +9,6 @@
519 'find_on_path',
520 'get_pid_from_file',
521 'kill_by_pidfile',
522- 'get_pid_from_file',
523 'open_for_writing',
524 'override_environ',
525 'process_exists',
526
527=== modified file 'lib/lp/services/scripts/tests/__init__.py'
528--- lib/lp/services/scripts/tests/__init__.py 2018-05-06 08:52:34 +0000
529+++ lib/lp/services/scripts/tests/__init__.py 2018-06-06 08:58:34 +0000
530@@ -44,7 +44,7 @@
531 return sorted(scripts)
532
533
534-def run_script(script_relpath, args, expect_returncode=0):
535+def run_script(script_relpath, args, expect_returncode=0, extra_env=None):
536 """Run a script for testing purposes.
537
538 :param script_relpath: The relative path to the script, from the tree
539@@ -52,11 +52,16 @@
540 :param args: Arguments to provide to the script.
541 :param expect_returncode: The return code expected. If a different value
542 is returned, and exception will be raised.
543+ :param extra_env: A dictionary of extra environment variables to provide
544+ to the script, or None.
545 """
546 script = os.path.join(config.root, script_relpath)
547 args = [script] + args
548+ env = dict(os.environ)
549+ if extra_env is not None:
550+ env.update(extra_env)
551 process = subprocess.Popen(
552- args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
553+ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
554 stdout, stderr = process.communicate()
555 if process.returncode != expect_returncode:
556 raise AssertionError('Failed:\n%s\n%s' % (stdout, stderr))
557
558=== modified file 'lib/lp/testing/__init__.py'
559--- lib/lp/testing/__init__.py 2018-05-09 16:55:39 +0000
560+++ lib/lp/testing/__init__.py 2018-06-06 08:58:34 +0000
561@@ -893,7 +893,8 @@
562 db_branch = self.factory.makeAnyBranch(**kwargs)
563 else:
564 db_branch = self.factory.makeProductBranch(product, **kwargs)
565- branch_url = 'lp-internal:///' + db_branch.unique_name
566+ branch_url = (
567+ 'lp-internal:///' + removeSecurityProxy(db_branch).unique_name)
568 if not self.direct_database_server:
569 transaction.commit()
570 bzr_branch = self.createBranchAtURL(branch_url, format=format)
571
572=== modified file 'scripts/start-loggerhead.py'
573--- scripts/start-loggerhead.py 2012-06-27 13:57:04 +0000
574+++ scripts/start-loggerhead.py 2018-06-06 08:58:34 +0000
575@@ -1,192 +1,14 @@
576 #!/usr/bin/python -S
577 #
578-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
579+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
580 # GNU Affero General Public License version 3 (see the file LICENSE).
581
582+from __future__ import absolute_import, print_function, unicode_literals
583+
584 import _pythonpath
585
586-import logging
587-from optparse import OptionParser
588-import os
589-import sys
590-import time
591-import traceback
592-
593-from paste import httpserver
594-from paste.deploy.config import PrefixMiddleware
595-from paste.httpexceptions import HTTPExceptionHandler
596-from paste.request import construct_url
597-from paste.translogger import TransLogger
598-from paste.wsgilib import catch_errors
599-
600-import lp.codehosting
601-from lp.services.config import config
602-
603-
604-LISTEN_HOST = config.codebrowse.listen_host
605-LISTEN_PORT = config.codebrowse.port
606-THREADPOOL_WORKERS = 10
607-
608-
609-class NoLockingFileHandler(logging.FileHandler):
610- """A version of logging.FileHandler that doesn't do it's own locking.
611-
612- We experienced occasional hangs in production where gdb-ery on the server
613- revealed that we sometimes end up with many threads blocking on the RLock
614- held by the logging file handler, and log reading finds that an exception
615- managed to kill a thread in an unsafe window for RLock's.
616-
617- Luckily, there's no real reason for us to take a lock during logging as
618- each log message translates to one call to .write on a file object, which
619- translates to one fwrite call, and it seems that this does enough locking
620- itself for our purposes.
621-
622- So this handler just doesn't lock in log message handling.
623- """
624-
625- def acquire(self):
626- pass
627-
628- def release(self):
629- pass
630-
631-
632-def setup_logging(home, foreground):
633- # i hate that stupid logging config format, so just set up logging here.
634-
635- log_folder = config.codebrowse.log_folder
636- if not log_folder:
637- log_folder = os.path.join(home, 'logs')
638- if not os.path.exists(log_folder):
639- os.makedirs(log_folder)
640-
641- f = logging.Formatter(
642- '%(levelname)-.3s [%(asctime)s.%(msecs)03d] [%(thread)d] %(name)s: %(message)s',
643- '%Y%m%d-%H:%M:%S')
644- debug_log = NoLockingFileHandler(os.path.join(log_folder, 'debug.log'))
645- debug_log.setLevel(logging.DEBUG)
646- debug_log.setFormatter(f)
647- if foreground:
648- stdout_log = logging.StreamHandler(sys.stdout)
649- stdout_log.setLevel(logging.DEBUG)
650- stdout_log.setFormatter(f)
651- f = logging.Formatter('[%(asctime)s.%(msecs)03d] %(message)s',
652- '%Y%m%d-%H:%M:%S')
653- access_log = NoLockingFileHandler(os.path.join(log_folder, 'access.log'))
654- access_log.setLevel(logging.INFO)
655- access_log.setFormatter(f)
656-
657- logging.getLogger('').setLevel(logging.DEBUG)
658- logging.getLogger('').addHandler(debug_log)
659- logging.getLogger('wsgi').addHandler(access_log)
660-
661- if foreground:
662- logging.getLogger('').addHandler(stdout_log)
663- else:
664- class S(object):
665- def write(self, str):
666- logging.getLogger().error(str.rstrip('\n'))
667- def flush(self):
668- pass
669- sys.stderr = S()
670-
671-
672-parser = OptionParser(description="Start loggerhead.")
673-parser.add_option(
674- "-f", "--foreground", default=False, action="store_true",
675- help="Run loggerhead in the foreground.")
676-options, _ = parser.parse_args()
677-
678-home = os.path.realpath(os.path.dirname(__file__))
679-pidfile = os.path.join(home, 'loggerhead.pid')
680-
681-if not options.foreground:
682- sys.stderr.write('\n')
683- sys.stderr.write('Launching loggerhead into the background.\n')
684- sys.stderr.write('PID file: %s\n' % (pidfile,))
685- sys.stderr.write('\n')
686-
687- from loggerhead.daemon import daemonize
688- daemonize(pidfile, home)
689-
690-setup_logging(home, foreground=options.foreground)
691-
692-log = logging.getLogger('loggerhead')
693-log.info('Starting up...')
694-
695-log.info('Loading the bzr plugins...')
696-from bzrlib.plugin import load_plugins
697-load_plugins()
698-
699-import bzrlib.plugins
700-if getattr(bzrlib.plugins, 'loom', None) is None:
701- log.error('Loom plugin loading failed.')
702-
703-from launchpad_loggerhead.debug import (
704- change_kill_thread_criteria, threadpool_debug)
705-from launchpad_loggerhead.app import RootApp, oops_middleware
706-from launchpad_loggerhead.session import SessionHandler
707-
708-SESSION_VAR = 'lh.session'
709-
710-secret = open(os.path.join(config.root, config.codebrowse.secret_path)).read()
711-
712-app = RootApp(SESSION_VAR)
713-app = HTTPExceptionHandler(app)
714-app = SessionHandler(app, SESSION_VAR, secret)
715-def log_request_start_and_stop(app):
716- def wrapped(environ, start_response):
717- log = logging.getLogger('loggerhead')
718- url = construct_url(environ)
719- log.info("Starting to process %s", url)
720- start_time = time.time()
721- def request_done_ok():
722- log.info("Processed ok %s [%0.3f seconds]", url, time.time() -
723- start_time)
724- def request_done_err(exc_info):
725- log.info("Processed err %s [%0.3f seconds]: %s", url, time.time() -
726- start_time, traceback.format_exception_only(*exc_info[:2]))
727- return catch_errors(app, environ, start_response, request_done_err,
728- request_done_ok)
729- return wrapped
730-app = log_request_start_and_stop(app)
731-app = PrefixMiddleware(app)
732-app = TransLogger(app)
733-app = threadpool_debug(app)
734-
735-def set_scheme(app):
736- """Set wsgi.url_scheme in the environment correctly.
737-
738- We serve requests that originated from both http and https, and
739- distinguish between them by adding a header in the https Apache config.
740- """
741- def wrapped(environ, start_response):
742- environ['wsgi.url_scheme'] = environ.pop(
743- 'HTTP_X_FORWARDED_SCHEME', 'http')
744- return app(environ, start_response)
745- return wrapped
746-app = set_scheme(app)
747-app = change_kill_thread_criteria(app)
748-app = oops_middleware(app)
749-
750-try:
751- httpserver.serve(
752- app, host=LISTEN_HOST, port=LISTEN_PORT,
753- threadpool_workers=THREADPOOL_WORKERS,
754- threadpool_options={
755- # Kill threads after 300 seconds. This is insanely high, but
756- # lower enough than the default (1800 seconds!) that evidence
757- # suggests it will be hit occasionally, and there's very little
758- # chance of it having negative consequences.
759- 'kill_thread_limit': 300,
760- # Check for threads that should be killed every 10 requests. The
761- # default is every 100, which is easily long enough for things to
762- # gum up completely in between checks.
763- 'hung_check_period': 10,
764- })
765-finally:
766- log.info('Shutdown.')
767- try:
768- os.remove(pidfile)
769- except OSError:
770- pass
771+from launchpad_loggerhead.wsgi import LoggerheadApplication
772+
773+
774+if __name__ == "__main__":
775+ LoggerheadApplication().run()
776
777=== modified file 'scripts/stop-loggerhead.py'
778--- scripts/stop-loggerhead.py 2012-06-29 08:40:05 +0000
779+++ scripts/stop-loggerhead.py 2018-06-06 08:58:34 +0000
780@@ -1,38 +1,33 @@
781 #!/usr/bin/python -S
782 #
783-# Copyright 2009-2012 Canonical Ltd. This software is licensed under the
784+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
785 # GNU Affero General Public License version 3 (see the file LICENSE).
786
787+from __future__ import absolute_import, print_function, unicode_literals
788+
789 import _pythonpath
790
791 from optparse import OptionParser
792 import os
793+import signal
794 import sys
795
796+from lp.services.pidfile import get_pid
797+
798
799 parser = OptionParser(description="Stop loggerhead.")
800 parser.parse_args()
801
802-home = os.path.realpath(os.path.dirname(__file__))
803-pidfile = os.path.join(home, 'loggerhead.pid')
804-
805-try:
806- f = open(pidfile, 'r')
807-except IOError as e:
808- print 'No pid file found.'
809- sys.exit(1)
810-
811-pid = int(f.readline())
812+pid = get_pid("codebrowse")
813
814 try:
815 os.kill(pid, 0)
816 except OSError as e:
817- print 'Stale pid file; server is not running.'
818+ print('Stale pid file; server is not running.')
819 sys.exit(1)
820
821-print
822-print 'Shutting down previous server @ pid %d.' % (pid,)
823-print
824+print()
825+print('Shutting down previous server @ pid %d.' % (pid,))
826+print()
827
828-import signal
829 os.kill(pid, signal.SIGTERM)
830
831=== modified file 'setup.py'
832--- setup.py 2018-05-31 13:12:39 +0000
833+++ setup.py 2018-06-06 08:58:34 +0000
834@@ -159,6 +159,7 @@
835 'FeedParser',
836 'feedvalidator',
837 'fixtures',
838+ 'gunicorn[gthread]',
839 'html5browser',
840 'importlib-resources',
841 'ipython',