Warning
This document is for an in-development version of Galaxy. You can alternatively view this page in the latest release if it exists or view the top of the latest release's documentation.
Source code for galaxy.jobs.runners.util.pykube_util
"""Interface layer for pykube library shared between Galaxy and Pulsar."""
import logging
import os
import re
from pathlib import PurePath
try:
from pykube.config import KubeConfig
from pykube.exceptions import HTTPError
from pykube.http import HTTPClient
from pykube.objects import (
Ingress,
Job,
Pod,
Service,
)
except ImportError as exc:
KubeConfig = None
Ingress = None
Job = None
Pod = None
Service = None
HTTPError = None
K8S_IMPORT_MESSAGE = (
"The Python pykube package is required to use "
"this feature, please install it or correct the "
"following error:\nImportError %s" % str(exc)
)
log = logging.getLogger(__name__)
DEFAULT_JOB_API_VERSION = "batch/v1"
DEFAULT_SERVICE_API_VERSION = "v1"
DEFAULT_INGRESS_API_VERSION = "networking.k8s.io/v1"
DEFAULT_NAMESPACE = "default"
INSTANCE_ID_INVALID_MESSAGE = (
"Galaxy instance [%s] is either too long "
"(>20 characters) or it includes non DNS "
"acceptable characters, ignoring it."
)
[docs]def pykube_client_from_dict(params):
if "k8s_use_service_account" in params and params["k8s_use_service_account"]:
pykube_client = HTTPClient(KubeConfig.from_service_account())
else:
config_path = params.get("k8s_config_path")
if config_path is None:
config_path = os.environ.get("KUBECONFIG", None)
if config_path is None:
config_path = "~/.kube/config"
pykube_client = HTTPClient(KubeConfig.from_file(config_path))
return pykube_client
[docs]def produce_k8s_job_prefix(app_prefix=None, instance_id=None):
job_name_elems = [app_prefix or "", instance_id or ""]
return "-".join(elem for elem in job_name_elems if elem)
[docs]def pull_policy(params):
# If this doesn't validate it returns None, that seems odd?
if "k8s_pull_policy" in params:
if params["k8s_pull_policy"] in ["Always", "IfNotPresent", "Never"]:
return params["k8s_pull_policy"]
return None
[docs]def find_service_object_by_name(pykube_api, service_name, namespace=None):
if not service_name:
raise ValueError("service name must not be empty")
return Service.objects(pykube_api).filter(field_selector={"metadata.name": service_name}, namespace=namespace)
[docs]def find_ingress_object_by_name(pykube_api, ingress_name, namespace=None):
if not ingress_name:
raise ValueError("ingress name must not be empty")
return Ingress.objects(pykube_api).filter(field_selector={"metadata.name": ingress_name}, namespace=namespace)
[docs]def find_job_object_by_name(pykube_api, job_name, namespace=None):
if not job_name:
raise ValueError("job name must not be empty")
return Job.objects(pykube_api).filter(field_selector={"metadata.name": job_name}, namespace=namespace)
[docs]def find_pod_object_by_name(pykube_api, job_name, namespace=None):
return Pod.objects(pykube_api).filter(selector=f"job-name={job_name}", namespace=namespace)
[docs]def is_pod_unschedulable(pykube_api, pod, namespace=None):
is_unschedulable = any(c.get("reason") == "Unschedulable" for c in pod.obj["status"].get("conditions", []))
if pod.obj["status"].get("phase") == "Pending" and is_unschedulable:
return True
return False
[docs]def delete_job(job, cleanup="always"):
job_failed = job.obj["status"]["failed"] > 0 if "failed" in job.obj["status"] else False
# Scale down the job just in case even if cleanup is never
job.scale(replicas=0)
api_delete = cleanup == "always"
if not api_delete and cleanup == "onsuccess" and not job_failed:
api_delete = True
if api_delete:
delete_options = {"apiVersion": "v1", "kind": "DeleteOptions", "propagationPolicy": "Background"}
r = job.api.delete(json=delete_options, **job.api_kwargs())
job.api.raise_for_status(r)
[docs]def delete_ingress(ingress, cleanup="always", job_failed=False):
api_delete = cleanup == "always"
if not api_delete and cleanup == "onsuccess" and not job_failed:
api_delete = True
if api_delete:
delete_options = {"apiVersion": "v1", "kind": "DeleteOptions", "propagationPolicy": "Background"}
r = ingress.api.delete(json=delete_options, **ingress.api_kwargs())
ingress.api.raise_for_status(r)
[docs]def delete_service(service, cleanup="always", job_failed=False):
api_delete = cleanup == "always"
if not api_delete and cleanup == "onsuccess" and not job_failed:
api_delete = True
if api_delete:
delete_options = {"apiVersion": "v1", "kind": "DeleteOptions", "propagationPolicy": "Background"}
r = service.api.delete(json=delete_options, **service.api_kwargs())
service.api.raise_for_status(r)
[docs]def job_object_dict(params, job_prefix, spec):
k8s_job_obj = {
"apiVersion": params.get("k8s_job_api_version", DEFAULT_JOB_API_VERSION),
"kind": "Job",
"metadata": {
"generateName": f"{job_prefix}-",
"namespace": params.get("k8s_namespace", DEFAULT_NAMESPACE),
},
"spec": spec,
}
return k8s_job_obj
[docs]def service_object_dict(params, service_name, spec):
k8s_service_obj = {
"apiVersion": params.get("k8s_service_api_version", DEFAULT_SERVICE_API_VERSION),
"kind": "Service",
"metadata": {
"name": service_name,
"namespace": params.get("k8s_namespace", DEFAULT_NAMESPACE),
},
}
k8s_service_obj["metadata"].update(spec.pop("metadata", {}))
k8s_service_obj.update(spec)
return k8s_service_obj
[docs]def ingress_object_dict(params, ingress_name, spec):
k8s_ingress_obj = {
"apiVersion": params.get("k8s_ingress_api_version", DEFAULT_INGRESS_API_VERSION),
"kind": "Ingress",
"metadata": {
"name": ingress_name,
"namespace": params.get("k8s_namespace", DEFAULT_NAMESPACE),
# TODO: Add default annotations
},
}
k8s_ingress_obj["metadata"].update(spec.pop("metadata", {}))
k8s_ingress_obj.update(spec)
return k8s_ingress_obj
[docs]def parse_pvc_param_line(pvc_param):
"""
:type pvc_param: str
:param pvc_param: the pvc mount param in the format ``pvc-name/subpath/desired:/mountpath/desired[:r]``
:rtype: dict
:return: a dict
like::
{"name": "pvc-name",
"subPath": "subpath/desired",
"mountPath": "/mountpath/desired",
"readOnly": False}
"""
claim, _, rest = pvc_param.partition(":")
mount_path, _, mode = rest.partition(":")
read_only = mode == "r"
claim_name, _, subpath = claim.partition("/")
return {
"name": claim_name.strip(),
"subPath": subpath.strip(),
"mountPath": mount_path.strip(),
"readOnly": read_only,
}
def generate_relative_mounts(pvc_param, files):
"""
Maps a list of files as mounts, relative to the base volume mount.
For example, given the pvc mount:
{
'name': 'my_pvc',
'mountPath': '/galaxy/database/jobs',
'subPath': 'data',
'readOnly': False
}
and files: ['/galaxy/database/jobs/01/input.txt', '/galaxy/database/jobs/01/working']
returns each file as a relative mount as follows:
[
{
'name': 'my_pvc',
'mountPath': '/galaxy/database/jobs/01/input.txt',
'subPath': 'data/01/input.txt',
'readOnly': False
},
{
'name': 'my_pvc',
'mountPath': '/galaxy/database/jobs/01/working',
'subPath': 'data/01/working',
'readOnly': False
}
]
:param pvc_param: the pvc claim dict
:param files: a list of file or folder names
:return: A list of volume mounts
"""
if not pvc_param:
return
param_claim = parse_pvc_param_line(pvc_param)
claim_name = param_claim["name"]
base_subpath = PurePath(param_claim.get("subPath", ""))
base_mount = PurePath(param_claim["mountPath"])
read_only = param_claim["readOnly"]
volume_mounts = []
for f in files:
file_path = PurePath(str(f))
if base_mount not in file_path.parents:
# force relative directory, needed for the job working directory in particular
file_path = base_mount.joinpath(file_path.relative_to("/") if file_path.is_absolute() else file_path)
relpath = file_path.relative_to(base_mount)
subpath = base_subpath.joinpath(relpath)
volume_mounts.append(
{"name": claim_name, "mountPath": str(file_path), "subPath": str(subpath), "readOnly": read_only}
)
return volume_mounts
def deduplicate_entries(obj_list):
# remove duplicate entries in a list of dictionaries
# based on: https://stackoverflow.com/a/9428041
return [i for n, i in enumerate(obj_list) if i not in obj_list[n + 1 :]]
[docs]def get_volume_mounts_for_job(job_wrapper, data_claim=None, working_claim=None):
volume_mounts = []
if data_claim:
volume_mounts.extend(generate_relative_mounts(data_claim, job_wrapper.job_io.get_input_fnames()))
# for individual output files, mount the parent folder of each output as there could be wildcard outputs
output_folders = deduplicate_entries(
[str(PurePath(str(f)).parent) for f in job_wrapper.job_io.get_output_fnames()]
)
volume_mounts.extend(generate_relative_mounts(data_claim, output_folders))
if working_claim:
volume_mounts.extend(generate_relative_mounts(working_claim, [job_wrapper.working_directory]))
return deduplicate_entries(volume_mounts)
[docs]def galaxy_instance_id(params):
"""Parse and validate the id of the Galaxy instance from supplied dict.
This will be added to Jobs and Pods names, so it needs to be DNS friendly,
this means: `The Internet standards (Requests for Comments) for protocols mandate that component hostname labels
may contain only the ASCII letters 'a' through 'z' (in a case-insensitive manner), the digits '0' through '9',
and the minus sign ('-').`
It looks for the value set on params['k8s_galaxy_instance_id'], which might or not be set. The
idea behind this is to allow the Galaxy instance to trust (or not) existing k8s Jobs and Pods that match the
setup of a Job that is being recovered or restarted after a downtime/reboot.
"""
if "k8s_galaxy_instance_id" in params:
raw_value = params["k8s_galaxy_instance_id"]
if re.match(r"(?!-)[a-z\d-]{1,20}(?<!-)$", raw_value):
return raw_value
else:
log.error(INSTANCE_ID_INVALID_MESSAGE % raw_value)
return None
__all__ = (
"DEFAULT_JOB_API_VERSION",
"DEFAULT_SERVICE_API_VERSION",
"DEFAULT_INGRESS_API_VERSION",
"ensure_pykube",
"find_service_object_by_name",
"find_ingress_object_by_name",
"find_job_object_by_name",
"find_pod_object_by_name",
"galaxy_instance_id",
"HTTPError",
"is_pod_unschedulable",
"Job",
"Service",
"Ingress",
"job_object_dict",
"service_object_dict",
"ingress_object_dict",
"Pod",
"produce_k8s_job_prefix",
"pull_policy",
"pykube_client_from_dict",
"delete_job",
"delete_service",
"delete_ingress",
"get_volume_mounts_for_job",
"parse_pvc_param_line",
)