Warning

This document is for an in-development version of Galaxy. You can alternatively view this page in the latest release if it exists or view the top of the latest release's documentation.

galaxy.job_execution package

Subpackages

• galaxy.job_execution.actions package
  • Submodules
  • galaxy.job_execution.actions.post module
    • DefaultJobAction
      • DefaultJobAction.name
      • DefaultJobAction.verbose_name
      • DefaultJobAction.execute()
      • DefaultJobAction.execute_on_mapped_over()
      • DefaultJobAction.get_short_str()
    • EmailAction
      • EmailAction.name
      • EmailAction.verbose_name
      • EmailAction.execute()
      • EmailAction.get_short_str()
    • ValidateOutputsAction
      • ValidateOutputsAction.name
      • ValidateOutputsAction.verbose_name
      • ValidateOutputsAction.execute()
      • ValidateOutputsAction.get_short_str()
    • ChangeDatatypeAction
      • ChangeDatatypeAction.name
      • ChangeDatatypeAction.verbose_name
      • ChangeDatatypeAction.execute()
      • ChangeDatatypeAction.get_short_str()
    • RenameDatasetAction
      • RenameDatasetAction.name
      • RenameDatasetAction.verbose_name
      • RenameDatasetAction.execute_on_mapped_over()
      • RenameDatasetAction.execute()
      • RenameDatasetAction.get_short_str()
    • HideDatasetAction
      • HideDatasetAction.name
      • HideDatasetAction.verbose_name
      • HideDatasetAction.execute()
      • HideDatasetAction.execute_on_mapped_over()
      • HideDatasetAction.get_short_str()
    • DeleteDatasetAction
      • DeleteDatasetAction.name
      • DeleteDatasetAction.verbose_name
      • DeleteDatasetAction.execute()
      • DeleteDatasetAction.execute_on_mapped_over()
      • DeleteDatasetAction.get_short_str()
    • ColumnSetAction
      • ColumnSetAction.name
      • ColumnSetAction.verbose_name
      • ColumnSetAction.execute()
      • ColumnSetAction.get_short_str()
    • SetMetadataAction
      • SetMetadataAction.name
      • SetMetadataAction.execute()
    • DeleteIntermediatesAction
      • DeleteIntermediatesAction.name
      • DeleteIntermediatesAction.verbose_name
      • DeleteIntermediatesAction.execute()
      • DeleteIntermediatesAction.get_short_str()
    • TagDatasetAction
      • TagDatasetAction.name
      • TagDatasetAction.verbose_name
      • TagDatasetAction.action
      • TagDatasetAction.direction
      • TagDatasetAction.execute_on_mapped_over()
      • TagDatasetAction.execute()
      • TagDatasetAction.get_short_str()
    • RemoveTagDatasetAction
      • RemoveTagDatasetAction.name
      • RemoveTagDatasetAction.verbose_name
      • RemoveTagDatasetAction.action
      • RemoveTagDatasetAction.direction
    • ActionBox
      • ActionBox.actions
      • ActionBox.public_actions
      • ActionBox.immediate_actions
      • ActionBox.mapped_over_output_actions
      • ActionBox.get_short_str()
      • ActionBox.handle_incoming()
      • ActionBox.execute_on_mapped_over()
      • ActionBox.execute()
• galaxy.job_execution.ports package
  • JobPortsView
    • JobPortsView.__init__()
    • JobPortsView.register_container_information()
  • Submodules
  • galaxy.job_execution.ports.view module
    • JobPortsView
      • JobPortsView.__init__()
      • JobPortsView.register_container_information()

Submodules

galaxy.job_execution.compute_environment module

class galaxy.job_execution.compute_environment.ComputeEnvironment

Bases: object

Definition of the job as it will be run on the (potentially) remote compute server.

abstract output_names()

Output unqualified filenames defined by job.

abstract input_path_rewrite(dataset)

Input path for specified dataset.

abstract output_path_rewrite(dataset)

Output path for specified dataset.

abstract input_extra_files_rewrite(dataset)

Input extra files path rewrite for specified dataset.

abstract output_extra_files_rewrite(dataset)

Output extra files path rewrite for specified dataset.

abstract input_metadata_rewrite(dataset, metadata_value)

Input metadata path rewrite for specified dataset.

abstract unstructured_path_rewrite(path)

Rewrite loc file paths, etc..

abstract working_directory()

Job working directory (potentially remote)

abstract config_directory()

Directory containing config files (potentially remote)

abstract env_config_directory()

Working directory (possibly as environment variable evaluation).

abstract sep()

os.path.sep for the platform this job will execute in.

abstract new_file_path()

Absolute path to dump new files for this job on compute server.

abstract tool_directory()

Absolute path to tool files for this job on compute server.

abstract version_path()

Location of the version file for the underlying tool.

abstract home_directory()

Home directory of target job - none if HOME should not be set.

abstract tmp_directory()

Temp directory of target job - none if HOME should not be set.

abstract galaxy_url()

URL to access Galaxy API from for this compute environment.

abstract get_file_sources_dict() → Dict[str, Any]

Return file sources dict for current user.

class galaxy.job_execution.compute_environment.SimpleComputeEnvironment

Bases: object

config_directory()

sep()

class galaxy.job_execution.compute_environment.SharedComputeEnvironment(job_io: JobIO, job: Job)

Bases: SimpleComputeEnvironment, ComputeEnvironment

Default ComputeEnvironment for job and task wrapper to pass to ToolEvaluator - valid when Galaxy and compute share all the relevant file systems.

job_id: JobIO

__init__(job_io: JobIO, job: Job)

job: Job

get_file_sources_dict() → Dict[str, Any]

Return file sources dict for current user.

output_names()

Output unqualified filenames defined by job.

output_paths()

input_path_rewrite(dataset)

Input path for specified dataset.

output_path_rewrite(dataset)

Output path for specified dataset.

input_extra_files_rewrite(dataset)

Input extra files path rewrite for specified dataset.

output_extra_files_rewrite(dataset)

Output extra files path rewrite for specified dataset.

input_metadata_rewrite(dataset, metadata_value)

Input metadata path rewrite for specified dataset.

unstructured_path_rewrite(path)

Rewrite loc file paths, etc..

working_directory()

Job working directory (potentially remote)

env_config_directory()

Working directory (possibly as environment variable evaluation).

new_file_path()

Absolute path to dump new files for this job on compute server.

version_path()

Location of the version file for the underlying tool.

tool_directory()

Absolute path to tool files for this job on compute server.

home_directory()

Home directory of target job - none if HOME should not be set.

tmp_directory()

Temp directory of target job - none if HOME should not be set.

galaxy_url()

URL to access Galaxy API from for this compute environment.

galaxy.job_execution.datasets module

Utility classes allowing Job interface to reason about datasets.

galaxy.job_execution.datasets.dataset_path_rewrites(dataset_paths)

class galaxy.job_execution.datasets.DatasetPath(dataset_id, real_path, false_path=None, false_extra_files_path=None, false_metadata_path=None, mutable=True, dataset_uuid=None, object_store_id=None)

Bases: object

__init__(dataset_id, real_path, false_path=None, false_extra_files_path=None, false_metadata_path=None, mutable=True, dataset_uuid=None, object_store_id=None)

with_path_for_job(false_path, false_extra_files_path=None, false_metadata_path=None)

Clone the dataset path but with a new false_path.

class galaxy.job_execution.datasets.DatasetPathRewriter

Bases: object

Used by runner to rewrite paths.

abstract rewrite_dataset_path(dataset, dataset_type)

Dataset type is ‘input’ or ‘output’. Return None to indicate not to rewrite this path.

class galaxy.job_execution.datasets.NullDatasetPathRewriter

Bases: DatasetPathRewriter

Used by default for jobwrapper, do not rewrite anything.

rewrite_dataset_path(dataset, dataset_type)

Keep path the same.

class galaxy.job_execution.datasets.OutputsToWorkingDirectoryPathRewriter(working_directory, outputs_directory_name)

Bases: DatasetPathRewriter

Rewrites all paths to place them in the specified working directory for normal jobs when Galaxy is configured with app.config.outputs_to_working_directory. Job runner base class is responsible for copying these out after job is complete.

__init__(working_directory, outputs_directory_name)

rewrite_dataset_path(dataset, dataset_type)

Keep path the same.

class galaxy.job_execution.datasets.TaskPathRewriter(working_directory, job_dataset_path_rewriter)

Bases: DatasetPathRewriter

Rewrites all paths to place them in the specified working directory for TaskWrapper. TaskWrapper is responsible for putting them there and pulling them out.

__init__(working_directory, job_dataset_path_rewriter)

rewrite_dataset_path(dataset, dataset_type)

galaxy.job_execution.datasets.get_path_rewriter(outputs_to_working_directory, working_directory, outputs_directory, is_task) → DatasetPathRewriter

galaxy.job_execution.output_collect module

Code allowing tools to define extra files associated with an output datset.

class galaxy.job_execution.output_collect.PermissionProvider(inp_data, security_agent, job)

Bases: PermissionProvider

__init__(inp_data, security_agent, job)

property permissions

set_default_hda_permissions(primary_data)

copy_dataset_permissions(init_from, primary_data)

Copy dataset permissions from supplied input dataset.

class galaxy.job_execution.output_collect.MetadataSourceProvider(inp_data)

Bases: MetadataSourceProvider

__init__(inp_data)

get_metadata_source(input_name)

Get metadata for supplied input_name.

galaxy.job_execution.output_collect.collect_dynamic_outputs(job_context, output_collections)

class galaxy.job_execution.output_collect.BaseJobContext

Bases: ModelPersistenceContext

tool_provided_metadata: BaseToolProvidedMetadata

job_working_directory: str

add_dataset_collection(collection)

find_files(output_name, collection, dataset_collectors) → list

get_job_id()

class galaxy.job_execution.output_collect.JobContext(tool, tool_provided_metadata: BaseToolProvidedMetadata, job, job_working_directory, permission_provider, metadata_source_provider, input_dbkey, object_store, final_job_state, max_discovered_files: Optional[int], flush_per_n_datasets=None)

Bases: BaseJobContext

__init__(tool, tool_provided_metadata: BaseToolProvidedMetadata, job, job_working_directory, permission_provider, metadata_source_provider, input_dbkey, object_store, final_job_state, max_discovered_files: Optional[int], flush_per_n_datasets=None)

job_working_directory: str

tool_provided_metadata: BaseToolProvidedMetadata

discovered_file_count: int

property tag_handler

Return a galaxy.model.tags.TagHandler-like object for persisting tags.

property work_context

property sa_session: scoped_session

If bound to a database, return the SQL Alchemy session.

Return None otherwise.

property permission_provider: PermissionProvider

If bound to a database, return the SQL Alchemy session.

Return None otherwise.

property metadata_source_provider: MetadataSourceProvider

Return associated MetadataSourceProvider object.

property job: Job

Return associated job object if bound to a job finish context connected to a database.

property flush_per_n_datasets: Optional[int]

property input_dbkey: str

property object_store: ObjectStore

Return object store to use for populating discovered dataset contents.

property user

If bound to a database, return the user the datasets should be created for.

Return None otherwise.

persist_object(obj)

Add the target to the persistence layer.

flush()

If database bound, flush the persisted objects to ensure IDs.

get_library_folder(destination)

get_hdca(object_id)

create_library_folder(parent_folder, name, description)

Create a library folder ready from supplied attributes for supplied parent.

create_hdca(name, structure)

add_output_dataset_association(name, dataset)

If discovering outputs for a job, persist output dataset association.

add_library_dataset_to_folder(library_folder, ld)

Add library dataset to persisted library folder.

add_datasets_to_history(datasets, for_output_dataset=None)

Add datasets to the history this context points at.

output_collection_def(name)

output_def(name)

job_id()

get_job_id()

get_implicit_collection_jobs_association_id()

No-op, no job context.

class galaxy.job_execution.output_collect.SessionlessJobContext(metadata_params, tool_provided_metadata: BaseToolProvidedMetadata, object_store, export_store, import_store, working_directory, final_job_state, max_discovered_files: Optional[int])

Bases: SessionlessModelPersistenceContext, BaseJobContext

__init__(metadata_params, tool_provided_metadata: BaseToolProvidedMetadata, object_store, export_store, import_store, working_directory, final_job_state, max_discovered_files: Optional[int])

tool_provided_metadata: BaseToolProvidedMetadata

discovered_file_count: int

output_collection_def(name)

output_def(name)

job_id()

get_hdca(object_id)

add_dataset_collection(collection)

add_output_dataset_association(name, dataset_instance)

No-op, no job context to persist this association for.

get_job_id()

get_implicit_collection_jobs_association_id()

No-op, no job context.

job_working_directory: str

galaxy.job_execution.output_collect.collect_primary_datasets(job_context: Union[JobContext, SessionlessJobContext], output, input_ext)

galaxy.job_execution.output_collect.discover_files(output_name, tool_provided_metadata, extra_file_collectors, job_working_directory, matchable)

galaxy.job_execution.output_collect.walk_over_file_collectors(extra_file_collectors, job_working_directory, matchable)

galaxy.job_execution.output_collect.walk_over_extra_files(target_dir, extra_file_collector, job_working_directory, matchable, parent_paths=None)

Walks through all files in a given directory, and returns all files that match the given collector’s match criteria. If the collector has the recurse flag enabled, will also recursively descend into child folders.

galaxy.job_execution.output_collect.dataset_collector(dataset_collection_description)

class galaxy.job_execution.output_collect.ToolMetadataDatasetCollector(dataset_collection_description)

Bases: object

__init__(dataset_collection_description)

class galaxy.job_execution.output_collect.DatasetCollector(dataset_collection_description)

Bases: object

__init__(dataset_collection_description)

match(dataset_instance, filename, path=None, parent_paths=None)

sort(matches)

galaxy.job_execution.output_collect.read_exit_code_from(exit_code_file, id_tag)

Read exit code reported for a Galaxy job.

galaxy.job_execution.output_collect.default_exit_code_file(files_dir, id_tag)

galaxy.job_execution.output_collect.collect_extra_files(object_store, dataset, job_working_directory)

galaxy.job_execution.output_collect.collect_shrinked_content_from_path(path)

galaxy.job_execution.setup module

Utilities to help job and tool code setup jobs.

class galaxy.job_execution.setup.JobIO(sa_session, job: Job, working_directory: str, outputs_directory: str, outputs_to_working_directory: bool, galaxy_url: str, version_path: str, tool_directory: str, home_directory: str, tmp_directory: str, tool_data_path: str, galaxy_data_manager_data_path: str, new_file_path: str, len_file_path: str, builds_file_path: str, check_job_script_integrity: bool, check_job_script_integrity_count: int, check_job_script_integrity_sleep: float, file_sources_dict: Dict[str, Any], user_context: Union[ProvidesUserFileSourcesUserContext, Dict[str, Any]], tool_source: Optional[str] = None, tool_source_class: Optional[str] = 'XmlToolSource', tool_dir: Optional[str] = None, is_task: bool = False)

Bases: Dictifiable

dict_collection_visible_keys = ('job_id', 'working_directory', 'outputs_directory', 'outputs_to_working_directory', 'galaxy_url', 'version_path', 'tool_directory', 'home_directory', 'tmp_directory', 'tool_data_path', 'galaxy_data_manager_data_path', 'new_file_path', 'len_file_path', 'builds_file_path', 'file_sources_dict', 'check_job_script_integrity', 'check_job_script_integrity_count', 'check_job_script_integrity_sleep', 'tool_source', 'tool_source_class', 'tool_dir', 'is_task')

__init__(sa_session, job: Job, working_directory: str, outputs_directory: str, outputs_to_working_directory: bool, galaxy_url: str, version_path: str, tool_directory: str, home_directory: str, tmp_directory: str, tool_data_path: str, galaxy_data_manager_data_path: str, new_file_path: str, len_file_path: str, builds_file_path: str, check_job_script_integrity: bool, check_job_script_integrity_count: int, check_job_script_integrity_sleep: float, file_sources_dict: Dict[str, Any], user_context: Union[ProvidesUserFileSourcesUserContext, Dict[str, Any]], tool_source: Optional[str] = None, tool_source_class: Optional[str] = 'XmlToolSource', tool_dir: Optional[str] = None, is_task: bool = False)

classmethod from_json(path, sa_session)

classmethod from_dict(io_dict, sa_session)

to_dict()

Return item dictionary.

to_json(path)

property file_sources: ConfiguredFileSources

property dataset_path_rewriter: DatasetPathRewriter

property output_paths: List[DatasetPath]

property output_hdas_and_paths: Dict[str, Tuple[DatasetInstance, DatasetPath]]

get_input_dataset_fnames(ds: DatasetInstance) → List[str]

get_input_fnames() → List[str]

get_input_paths() → List[DatasetPath]

get_input_path(dataset: DatasetInstance) → DatasetPath

get_output_basenames() → List[str]

get_output_fnames() → List[DatasetPath]

get_output_path(dataset)

get_mutable_output_fnames()

get_output_hdas_and_fnames() → Dict[str, Tuple[DatasetInstance, DatasetPath]]

compute_outputs() → None

get_output_file_id(file: str) → Optional[int]

galaxy.job_execution.setup.ensure_configs_directory(work_dir: str) → str

galaxy.job_execution.setup.create_working_directory_for_job(object_store, job) → str