Warning
This document is for an in-development version of Galaxy. You can alternatively view this page in the latest release if it exists or view the top of the latest release's documentation.
Source code for galaxy.app
from __future__ import absolute_import
import logging
import signal
import sys
import time
import galaxy.model
import galaxy.model.security
import galaxy.queues
import galaxy.quota
import galaxy.security
from galaxy import config, job_metrics, jobs
from galaxy.config_watchers import ConfigWatchers
from galaxy.containers import build_container_interfaces
from galaxy.managers.collections import DatasetCollectionManager
from galaxy.managers.folders import FolderManager
from galaxy.managers.hdas import HDAManager
from galaxy.managers.histories import HistoryManager
from galaxy.managers.interactivetool import InteractiveToolManager
from galaxy.managers.libraries import LibraryManager
from galaxy.managers.tools import DynamicToolManager
from galaxy.managers.users import UserManager
from galaxy.managers.workflows import WorkflowsManager
from galaxy.model.database_heartbeat import DatabaseHeartbeat
from galaxy.model.tags import GalaxyTagHandler
from galaxy.queue_worker import GalaxyQueueWorker
from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager
from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager
from galaxy.tool_util.deps.views import DependencyResolversView
from galaxy.tool_util.verify import test_data
from galaxy.tools.cache import (
ToolCache,
ToolShedRepositoryCache
)
from galaxy.tools.data_manager.manager import DataManagers
from galaxy.tools.error_reports import ErrorReports
from galaxy.tools.special_tools import load_lib_tools
from galaxy.tours import ToursRegistry
from galaxy.util import (
ExecutionTimer,
heartbeat,
StructuredExecutionTimer,
)
from galaxy.visualization.data_providers.registry import DataProviderRegistry
from galaxy.visualization.genomes import Genomes
from galaxy.visualization.plugins.registry import VisualizationsRegistry
from galaxy.web import url_for
from galaxy.web.proxy import ProxyManager
from galaxy.web_stack import application_stack_instance
from galaxy.webhooks import WebhooksRegistry
log = logging.getLogger(__name__)
app = None
[docs]class UniverseApplication(config.ConfiguresGalaxyMixin):
"""Encapsulates the state of a Universe application"""
[docs] def __init__(self, **kwargs):
if not log.handlers:
# Paste didn't handle it, so we need a temporary basic log
# configured. The handler added here gets dumped and replaced with
# an appropriately configured logger in configure_logging below.
logging.basicConfig(level=logging.DEBUG)
log.debug("python path is: %s", ", ".join(sys.path))
self.name = 'galaxy'
self.startup_timer = ExecutionTimer()
self.new_installation = False
# Read config file and check for errors
self.config = config.Configuration(**kwargs)
self.config.check()
config.configure_logging(self.config)
self.execution_timer_factory = ExecutionTimerFactory(self.config)
self.configure_fluent_log()
# A lot of postfork initialization depends on the server name, ensure it is set immediately after forking before other postfork functions
self.application_stack = application_stack_instance(app=self)
self.application_stack.register_postfork_function(self.application_stack.set_postfork_server_name, self)
self.config.reload_sanitize_whitelist(explicit='sanitize_whitelist_file' in kwargs)
self.amqp_internal_connection_obj = galaxy.queues.connection_from_config(self.config)
# queue_worker *can* be initialized with a queue, but here we don't
# want to and we'll allow postfork to bind and start it.
self.queue_worker = GalaxyQueueWorker(self)
self._configure_tool_shed_registry()
self._configure_object_store(fsmon=True)
# Setup the database engine and ORM
config_file = kwargs.get('global_conf', {}).get('__file__', None)
if config_file:
log.debug('Using "galaxy.ini" config file: %s', config_file)
check_migrate_tools = self.config.check_migrate_tools
self._configure_models(check_migrate_databases=self.config.check_migrate_databases, check_migrate_tools=check_migrate_tools, config_file=config_file)
# Security helper
self._configure_security()
# Tag handler
self.tag_handler = GalaxyTagHandler(self.model.context)
self.dataset_collections_service = DatasetCollectionManager(self)
self.history_manager = HistoryManager(self)
self.hda_manager = HDAManager(self)
self.workflow_manager = WorkflowsManager(self)
self.dependency_resolvers_view = DependencyResolversView(self)
self.test_data_resolver = test_data.TestDataResolver(file_dirs=self.config.tool_test_data_directories)
self.library_folder_manager = FolderManager()
self.library_manager = LibraryManager()
self.dynamic_tool_manager = DynamicToolManager(self)
# Tool Data Tables
self._configure_tool_data_tables(from_shed_config=False)
# Load dbkey / genome build manager
self._configure_genome_builds(data_table_name="__dbkeys__", load_old_style=True)
# Genomes
self.genomes = Genomes(self)
# Data providers registry.
self.data_provider_registry = DataProviderRegistry()
# Initialize job metrics manager, needs to be in place before
# config so per-destination modifications can be made.
self.job_metrics = job_metrics.JobMetrics(self.config.job_metrics_config_file, app=self)
# Initialize error report plugins.
self.error_reports = ErrorReports(self.config.error_report_file, app=self)
# Initialize the job management configuration
self.job_config = jobs.JobConfiguration(self)
# Setup a Tool Cache
self.tool_cache = ToolCache()
self.tool_shed_repository_cache = ToolShedRepositoryCache(self)
# Watch various config files for immediate reload
self.watchers = ConfigWatchers(self)
self._configure_tool_config_files()
self.installed_repository_manager = InstalledRepositoryManager(self)
self._configure_datatypes_registry(self.installed_repository_manager)
galaxy.model.set_datatypes_registry(self.datatypes_registry)
self._configure_toolbox()
# Load Data Manager
self.data_managers = DataManagers(self)
# Load the update repository manager.
self.update_repository_manager = UpdateRepositoryManager(self)
# Load proprietary datatype converters and display applications.
self.installed_repository_manager.load_proprietary_converters_and_display_applications()
# Load datatype display applications defined in local datatypes_conf.xml
self.datatypes_registry.load_display_applications(self)
# Load datatype converters defined in local datatypes_conf.xml
self.datatypes_registry.load_datatype_converters(self.toolbox)
# Load external metadata tool
self.datatypes_registry.load_external_metadata_tool(self.toolbox)
# Load history import/export tools.
load_lib_tools(self.toolbox)
# visualizations registry: associates resources with visualizations, controls how to render
self.visualizations_registry = VisualizationsRegistry(
self,
directories_setting=self.config.visualization_plugins_directory,
template_cache_dir=self.config.template_cache_path)
# Tours registry
self.tour_registry = ToursRegistry(self.config.tour_config_dir)
# Webhooks registry
self.webhooks_registry = WebhooksRegistry(self.config.webhooks_dir)
# Load security policy.
self.security_agent = self.model.security_agent
self.host_security_agent = galaxy.model.security.HostAgent(
model=self.security_agent.model,
permitted_actions=self.security_agent.permitted_actions)
# Load quota management.
if self.config.enable_quotas:
self.quota_agent = galaxy.quota.QuotaAgent(self.model)
else:
self.quota_agent = galaxy.quota.NoQuotaAgent(self.model)
# Heartbeat for thread profiling
self.heartbeat = None
from galaxy import auth
self.auth_manager = auth.AuthManager(self)
self.user_manager = UserManager(self)
# Start the heartbeat process if configured and available (wait until
# postfork if using uWSGI)
if self.config.use_heartbeat:
if heartbeat.Heartbeat:
self.heartbeat = heartbeat.Heartbeat(
self.config,
period=self.config.heartbeat_interval,
fname=self.config.heartbeat_log
)
self.heartbeat.daemon = True
self.application_stack.register_postfork_function(self.heartbeat.start)
self.authnz_manager = None
if self.config.enable_oidc:
from galaxy.authnz import managers
self.authnz_manager = managers.AuthnzManager(self,
self.config.oidc_config,
self.config.oidc_backends_config)
self.sentry_client = None
if self.config.sentry_dsn:
def postfork_sentry_client():
import raven
self.sentry_client = raven.Client(self.config.sentry_dsn, transport=raven.transport.HTTPTransport)
self.application_stack.register_postfork_function(postfork_sentry_client)
# Transfer manager client
if self.config.get_bool('enable_beta_job_managers', False):
from galaxy.jobs import transfer_manager
self.transfer_manager = transfer_manager.TransferManager(self)
# Start the job manager
from galaxy.jobs import manager
self.job_manager = manager.JobManager(self)
self.application_stack.register_postfork_function(self.job_manager.start)
self.proxy_manager = ProxyManager(self.config)
from galaxy.workflow import scheduling_manager
# Must be initialized after job_config.
self.workflow_scheduling_manager = scheduling_manager.WorkflowSchedulingManager(self)
# Must be initialized after any component that might make use of stack messaging is configured. Alternatively if
# it becomes more commonly needed we could create a prefork function registration method like we do with
# postfork functions.
self.application_stack.init_late_prefork()
self.containers = {}
if self.config.enable_beta_containers_interface:
self.containers = build_container_interfaces(
self.config.containers_config_file,
containers_conf=self.config.containers_conf
)
self.interactivetool_manager = InteractiveToolManager(self)
# Configure handling of signals
handlers = {}
if self.heartbeat:
handlers[signal.SIGUSR1] = self.heartbeat.dump_signal_handler
self._configure_signal_handlers(handlers)
self.database_heartbeat = DatabaseHeartbeat(
application_stack=self.application_stack
)
self.database_heartbeat.add_change_callback(self.watchers.change_state)
self.application_stack.register_postfork_function(self.database_heartbeat.start)
# Start web stack message handling
self.application_stack.register_postfork_function(self.application_stack.start)
self.model.engine.dispose()
# Inject url_for for components to more easily optionally depend
# on url_for.
self.url_for = url_for
self.server_starttime = int(time.time()) # used for cachebusting
log.info("Galaxy app startup finished %s" % self.startup_timer)
[docs] def shutdown(self):
log.debug('Shutting down')
exception = None
try:
self.queue_worker.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown control worker cleanly")
try:
self.watchers.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown configuration watchers cleanly")
try:
self.database_heartbeat.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown database heartbeat cleanly")
try:
self.workflow_scheduling_manager.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown workflow scheduling manager cleanly")
try:
self.job_manager.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown job manager cleanly")
try:
self.object_store.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown object store cleanly")
try:
if self.heartbeat:
self.heartbeat.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown heartbeat cleanly")
try:
self.update_repository_manager.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown update repository manager cleanly")
try:
self.model.engine.dispose()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown SA database engine cleanly")
try:
self.application_stack.shutdown()
except Exception as e:
exception = exception or e
log.exception("Failed to shutdown application stack interface cleanly")
if exception:
raise exception
else:
log.debug('Finished shutting down')
[docs] def configure_fluent_log(self):
if self.config.fluent_log:
from galaxy.util.custom_logging.fluent_log import FluentTraceLogger
self.trace_logger = FluentTraceLogger('galaxy', self.config.fluent_host, self.config.fluent_port)
else:
self.trace_logger = None
@property
def is_job_handler(self):
return (self.config.track_jobs_in_database and self.job_config.is_handler) or not self.config.track_jobs_in_database
[docs]class StatsdStructuredExecutionTimer(StructuredExecutionTimer):
[docs] def __init__(self, galaxy_statsd_client, *args, **kwds):
self.galaxy_statsd_client = galaxy_statsd_client
super(StatsdStructuredExecutionTimer, self).__init__(*args, **kwds)
[docs] def to_str(self, **kwd):
self.galaxy_statsd_client.timing(self.timer_id, self.elapsed * 1000., kwd)
return super(StatsdStructuredExecutionTimer, self).to_str()
[docs]class ExecutionTimerFactory(object):
[docs] def __init__(self, config):
statsd_host = getattr(config, "statsd_host", None)
if statsd_host:
from galaxy.web.framework.middleware.statsd import GalaxyStatsdClient
self.galaxy_statsd_client = GalaxyStatsdClient(
statsd_host,
getattr(config, 'statsd_port', 8125),
getattr(config, 'statsd_prefix', 'galaxy'),
getattr(config, 'statsd_influxdb', False),
)
else:
self.galaxy_statsd_client = None
[docs] def get_timer(self, *args, **kwd):
if self.galaxy_statsd_client:
return StatsdStructuredExecutionTimer(self.galaxy_statsd_client, *args, **kwd)
else:
return StructuredExecutionTimer(*args, **kwd)