Talisker 0.9.2, deployed on trusty and python 3.4.3 has a broken /_status/error endpoint.l Hitting that endpoint with curl produces the following output on the terminal where talisker is running:
2017-04-12 02:03:08.718Z INFO gunicorn.access "GET /_status/error" method=GET path=None qs= status=500 ip=127.0.0.1 proto=HTTP/1.1 length=0 referrer=None ua=None duration=2.164 request_id=371502e1-c5bc-4fed-ac94-44983
63065de
--- Logging error ---
Traceback (most recent call last):
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 135, in handle
self.handle_request(listener, req, client, addr)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 176, in handle_request
respiter = self.wsgi(environ, resp.start_response)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/raven/middleware.py", line 98, in __call__
iterable = self.application(environ, start_response)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/context.py", line 46, in middleware
return ClosingIterator(app(environ, start_response), clear)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/request_id.py", line 106, in __call__
return self.app(environ, add_id_header)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/wsgi.py", line 44, in middleware
return app(environ, start_response)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/endpoints.py", line 139, in __call__
response = func(request)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/endpoints.py", line 81, in wrapper
return f(self, request)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/endpoints.py", line 192, in error
raise TestException('this is a test, ignore')
talisker.endpoints.TestException: this is a test, ignore
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/arbiter.py", line 578, in spawn_worker
worker.init_process()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/base.py", line 131, in init_process
self.run()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 124, in run
self.run_for_one(timeout)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 68, in run_for_one
self.accept(listener)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 30, in accept
self.handle(listener, client, addr)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 156, in handle
self.handle_error(req, client, addr, e)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/base.py", line 248, in handle_error
self.log.access(resp, req, environ, request_time)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/gunicorn.py", line 133, in access
"gunicorn.request.status.{}".format(resp.status_code), 1)
AttributeError: 'Response' object has no attribute 'status_code'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.4/logging/__init__.py", line 978, in emit
msg = self.format(record)
File "/usr/lib/python3.4/logging/__init__.py", line 828, in format
return fmt.format(record)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/logs.py", line 378, in format
return super(ColoredFormatter, self).format(record)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/logs.py", line 280, in format
record.message = self.escape_quotes(record.getMessage())
File "/usr/lib/python3.4/logging/__init__.py", line 328, in getMessage
msg = msg % self.args
TypeError: not all arguments converted during string formatting
Call stack:
File "/home/ubuntu/code/canonical/snap-delta-service/env/bin/talisker", line 11, in <module>
sys.exit(run())
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/gunicorn.py", line 268, in run
return app.run()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/app/base.py", line 203, in run
super(Application, self).run()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/app/base.py", line 72, in run
Arbiter(self).run()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/arbiter.py", line 211, in run
self.manage_workers()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/arbiter.py", line 544, in manage_workers
self.spawn_workers()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/arbiter.py", line 611, in spawn_workers
self.spawn_worker()
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/arbiter.py", line 589, in spawn_worker
self.log.exception("Exception in worker process"),
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/gunicorn.py", line 110, in exception
super().exception(self, msg, *args, **kwargs)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/glogging.py", line 253, in exception
self.error_log.exception(msg, *args, **kwargs)
File "/usr/lib/python3.4/logging/__init__.py", line 1310, in exception
self.error(msg, *args, **kwargs)
File "/usr/lib/python3.4/logging/__init__.py", line 1303, in error
self._log(ERROR, msg, args, **kwargs)
File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/logs.py", line 250, in _log
super()._log(level, msg, *args, **kwargs)
Message: <talisker.gunicorn.GunicornLogger object at 0x2acf62b99e48>
Arguments: ('Exception in worker process',)
Top level Sentry exception caught - failed creating log record
b'<talisker.gunicorn.GunicornLogger object at 0x2acf62b99e48>'
b'Traceback (most recent call last):\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 135, in handle\n self.handle_request(listener, req, clien
t, addr)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 176, in handle_request\n respiter = self.wsgi(environ, resp.start_response)\n File "
/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/raven/middleware.py", line 98, in __call__\n iterable = self.application(environ, start_response)\n File "/home/ubuntu/code/canonical/
snap-delta-service/env/lib/python3.4/site-packages/talisker/context.py", line 46, in middleware\n return ClosingIterator(app(environ, start_response), clear)\n File "/home/ubuntu/code/canonical/snap-delta-service/
env/lib/python3.4/site-packages/talisker/request_id.py", line 106, in __call__\n return self.app(environ, add_id_header)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talis
ker/wsgi.py", line 44, in middleware\n return app(environ, start_response)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/endpoints.py", line 139, in __call__\n
response = func(request)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/endpoints.py", line 81, in wrapper\n return f(self, request)\n File "/home/ubuntu/code/cano
nical/snap-delta-service/env/lib/python3.4/site-packages/talisker/endpoints.py", line 192, in error\n raise TestException(\'this is a test, ignore\')\ntalisker.endpoints.TestException: this is a test, ignore\n\nDur
ing handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/arbiter.py", line
578, in spawn_worker\n worker.init_process()\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/base.py", line 131, in init_process\n self.run()\n File "/h
ome/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 124, in run\n self.run_for_one(timeout)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/p
ython3.4/site-packages/gunicorn/workers/sync.py", line 68, in run_for_one\n self.accept(listener)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", l
ine 30, in accept\n self.handle(listener, client, addr)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/sync.py", line 156, in handle\n self.handle_error(
req, client, addr, e)\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/gunicorn/workers/base.py", line 248, in handle_error\n self.log.access(resp, req, environ, request_time)
\n File "/home/ubuntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/gunicorn.py", line 133, in access\n "gunicorn.request.status.{}".format(resp.status_code), 1)\nAttributeError: \'Res
ponse\' object has no attribute \'status_code\'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File "/home/ubuntu/code/canonical/snap-delta-service/env/
lib/python3.4/site-packages/raven/handlers/logging.py", line 83, in emit\n self.format(record)\n File "/usr/lib/python3.4/logging/__init__.py", line 828, in format\n return fmt.format(record)\n File "/home/ubu
ntu/code/canonical/snap-delta-service/env/lib/python3.4/site-packages/talisker/logs.py", line 280, in format\n record.message = self.escape_quotes(record.getMessage())\n File "/usr/lib/python3.4/logging/__init__.p
y", line 328, in getMessage\n msg = msg % self.args\nTypeError: not all arguments converted during string formatting\n'
I'm not sure exactly what causes this issue, but can confirm that the same thing happens with python 3.5.2 on yakkety.