"""
Gunicorn config file based on https://gist.github.com/hynek/ba655c8756924a5febc5285c712a7946
"""
import gc
import logging
import os
import sys
log = logging.getLogger(__name__)
[docs]
def is_preload_app():
return "--preload" in os.environ.get("GUNICORN_CMD_ARGS", "") or "--preload" in sys.argv
[docs]
def on_starting(server):
"""
Attach a set of IDs that can be temporarily re-used.
Used on reloads when each worker exists twice.
"""
server._worker_id_overload = set()
[docs]
def nworkers_changed(server, new_value, old_value):
"""
Gets called on startup too.
Set the current number of workers. Required if we raise the worker count
temporarily using TTIN because server.cfg.workers won't be updated and if
one of those workers dies, we wouldn't know the ids go that far.
"""
server._worker_id_current_workers = new_value
def _next_worker_id(server):
"""
If there are IDs open for re-use, take one. Else look for a free one.
"""
if server._worker_id_overload:
return server._worker_id_overload.pop()
in_use = {w._worker_id for w in tuple(server.WORKERS.values()) if w.alive}
free = set(range(1, server._worker_id_current_workers + 1)) - in_use
return free.pop()
[docs]
def on_reload(server):
"""
Add a full set of ids into overload so it can be re-used once.
"""
server._worker_id_overload = set(range(1, server.cfg.workers + 1))
[docs]
def when_ready(server):
# freeze objects after preloading app
if is_preload_app():
gc.freeze()
print("Objects frozen in perm gen: ", gc.get_freeze_count())
[docs]
def pre_fork(server, worker):
"""
Attach the next free worker_id before forking off.
"""
worker._worker_id = _next_worker_id(server)
[docs]
def post_fork(server, worker):
"""
Put worker_id and listeners into an env variable for further use within the app.
"""
os.environ["GUNICORN_WORKER_ID"] = str(worker._worker_id)
os.environ["GUNICORN_LISTENERS"] = ",".join(str(bind) for bind in server.LISTENERS)
if is_preload_app():
gc.enable()
from galaxy.web_stack import GunicornApplicationStack
GunicornApplicationStack.late_postfork_event.set()
[docs]
def worker_abort(worker):
"""
Log what the worker was serving when the arbiter aborted it.
Reached when a worker misses ``timeout``, usually because a graceful drain never
finished. The summary is logged at ERROR so that it reaches Sentry (Galaxy's
Sentry client turns ERROR log records into events) with the stack dump alongside
it as a breadcrumb.
"""
summarize = getattr(worker, "in_flight_summary", None)
if summarize is None:
return
try:
log.warning("Worker aborted by arbiter, stacks follow\n%s", worker.in_flight_stacks())
log.error("Worker aborted by arbiter (worker timeout)\n%s", summarize())
except Exception:
log.exception("Failed to report in-flight requests for aborted worker")