Warning
This document is for an in-development version of Galaxy. You can alternatively view this page in the latest release if it exists or view the top of the latest release's documentation.
Source code for galaxy.tool_util.deps.conda_util
import functools
import hashlib
import json
import logging
import os
import re
import shutil
import sys
import tempfile
import packaging.version
import six
from six.moves import shlex_quote
from galaxy.util import (
commands,
smart_str,
unicodify
)
from . import installable
log = logging.getLogger(__name__)
# Not sure there are security concerns, lets just fail fast if we are going
# break shell commands we are building.
SHELL_UNSAFE_PATTERN = re.compile(r"[\s\"']")
IS_OS_X = sys.platform == "darwin"
# BSD 3-clause
CONDA_LICENSE = "http://docs.continuum.io/anaconda/eula"
VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)")
UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_")
USE_PATH_EXEC_DEFAULT = False
CONDA_VERSION = "4.6.14"
CONDA_BUILD_VERSION = "3.17.8"
USE_LOCAL_DEFAULT = False
def conda_link():
if IS_OS_X:
url = "https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh"
else:
if sys.maxsize > 2**32:
url = "https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh"
else:
url = "https://repo.anaconda.com/miniconda/Miniconda3-4.5.12-Linux-x86.sh"
return url
def find_conda_prefix(conda_prefix=None):
""" If supplied conda_prefix is not set, default to the default location
for Miniconda installs.
"""
if conda_prefix is None:
home = os.path.expanduser("~")
miniconda_2_dest = os.path.join(home, "miniconda2")
miniconda_3_dest = os.path.join(home, "miniconda3")
anaconda_2_dest = os.path.join(home, "anaconda2")
anaconda_3_dest = os.path.join(home, "anaconda3")
# Prefer miniconda3 install if both available
if os.path.exists(miniconda_3_dest):
return miniconda_3_dest
elif os.path.exists(miniconda_2_dest):
return miniconda_2_dest
elif os.path.exists(anaconda_3_dest):
return anaconda_3_dest
elif os.path.exists(anaconda_2_dest):
return anaconda_2_dest
else:
return miniconda_3_dest
return conda_prefix
[docs]class CondaContext(installable.InstallableContext):
installable_description = "Conda"
[docs] def __init__(self, conda_prefix=None, conda_exec=None,
shell_exec=None, debug=False, ensure_channels='',
condarc_override=None, use_path_exec=USE_PATH_EXEC_DEFAULT,
copy_dependencies=False, use_local=USE_LOCAL_DEFAULT):
self.condarc_override = condarc_override
if not conda_exec and use_path_exec:
conda_exec = commands.which("conda")
if conda_exec:
conda_exec = os.path.normpath(conda_exec)
self.conda_exec = conda_exec
self.debug = debug
self.shell_exec = shell_exec or commands.shell
self.copy_dependencies = copy_dependencies
if conda_prefix is None:
info = self.conda_info()
if info and "default_prefix" in info:
conda_prefix = info["default_prefix"]
if conda_prefix is None:
conda_prefix = find_conda_prefix(conda_prefix)
self.conda_prefix = conda_prefix
if conda_exec is None:
self.conda_exec = self._bin("conda")
if ensure_channels:
if not isinstance(ensure_channels, list):
ensure_channels = [c for c in ensure_channels.split(",") if c]
else:
ensure_channels = None
self.ensure_channels = ensure_channels
self._conda_version = None
self._miniconda_version = None
self._conda_build_available = None
self.use_local = use_local
@property
def conda_version(self):
if self._conda_version is None:
self._guess_conda_properties()
return self._conda_version
@property
def conda_build_available(self):
if self._conda_build_available is None:
self._guess_conda_properties()
return self._conda_build_available
def _guess_conda_properties(self):
conda_meta_path = self._conda_meta_path
# Perhaps we should call "conda info --json" and parse it but for now we are going
# to assume the default.
conda_version = packaging.version.parse(CONDA_VERSION)
conda_build_available = False
miniconda_version = "3"
if os.path.exists(conda_meta_path):
for package in os.listdir(conda_meta_path):
package_parts = package.split("-")
if len(package_parts) < 3:
continue
package = '-'.join(package_parts[:-2])
version = package_parts[-2]
# build = package_parts[-1]
if package == "conda":
conda_version = packaging.version.parse(version)
if package == "python" and version.startswith("2"):
miniconda_version = "2"
if package == "conda-build":
conda_build_available = True
self._conda_version = conda_version
self._miniconda_version = miniconda_version
self._conda_build_available = conda_build_available
@property
def _conda_meta_path(self):
return os.path.join(self.conda_prefix, "conda-meta")
@property
def _override_channels_args(self):
override_channels_args = []
if self.ensure_channels:
override_channels_args.append("--override-channels")
for channel in self.ensure_channels:
override_channels_args.extend(["--channel", channel])
return override_channels_args
[docs] def ensure_conda_build_installed_if_needed(self):
if self.use_local and not self.conda_build_available:
conda_targets = [CondaTarget("conda-build", version=CONDA_BUILD_VERSION)]
# Cannot use --use-local during installation of conda-build.
return install_conda_targets(conda_targets, conda_context=self, env_name=None, allow_local=False)
else:
return 0
[docs] def conda_info(self):
if self.conda_exec is not None:
info_out = commands.execute([self.conda_exec, "info", "--json"])
info_out = unicodify(info_out)
info = json.loads(info_out)
return info
else:
return None
[docs] def is_conda_installed(self):
"""
Check if conda_exec exists
"""
if os.path.exists(self.conda_exec):
return True
else:
return False
[docs] def can_install_conda(self):
"""
If conda_exec is set to a path outside of conda_prefix,
there is no use installing conda into conda_prefix, since it can't be used by galaxy.
If conda_exec equals conda_prefix/bin/conda, we can install conda if either conda_prefix
does not exist or is empty.
"""
conda_exec = os.path.abspath(self.conda_exec)
conda_prefix_plus_exec = os.path.abspath(os.path.join(self.conda_prefix, 'bin/conda'))
if conda_exec == conda_prefix_plus_exec:
if not os.path.exists(self.conda_prefix):
return True
elif os.listdir(self.conda_prefix) == []:
os.rmdir(self.conda_prefix) # Conda's install script fails if path exists (even if empty).
return True
else:
log.warning("Cannot install Conda because conda_prefix '%s' exists and is not empty.",
self.conda_prefix)
return False
else:
log.warning("Skipping installation of Conda into conda_prefix '%s', "
"since conda_exec '%s' is set to a path outside of conda_prefix.",
self.conda_prefix, self.conda_exec)
return False
[docs] def exec_command(self, operation, args, stdout_path=None):
"""
Execute the requested command.
Return the process exit code (i.e. 0 in case of success).
"""
cmd = [self.conda_exec]
cmd.extend(operation.split())
if self.debug:
cmd.append("--debug")
cmd.extend(args)
env = {}
if self.condarc_override:
env["CONDARC"] = self.condarc_override
cmd_string = ' '.join(map(shlex_quote, cmd))
kwds = dict()
try:
if stdout_path:
kwds['stdout'] = open(stdout_path, 'w')
cmd_string += " > '%s'" % stdout_path
conda_exec_home = env['HOME'] = tempfile.mkdtemp(prefix='conda_exec_home_') # We don't want to pollute ~/.conda, which may not even be writable
log.debug("Executing command: %s", cmd_string)
return self.shell_exec(cmd, env=env, **kwds)
except Exception:
log.exception("Failed to execute command: %s", cmd_string)
return 1
finally:
if kwds.get('stdout'):
kwds['stdout'].close()
if conda_exec_home:
shutil.rmtree(conda_exec_home, ignore_errors=True)
[docs] def exec_create(self, args, allow_local=True, stdout_path=None):
"""
Return the process exit code (i.e. 0 in case of success).
"""
create_args = [
"-y",
"--quiet"
]
if allow_local and self.use_local:
create_args.extend(["--use-local"])
create_args.extend(self._override_channels_args)
create_args.extend(args)
return self.exec_command("create", create_args, stdout_path=stdout_path)
[docs] def exec_remove(self, args):
"""
Remove a conda environment using conda env remove -y --name `args`.
Return the process exit code (i.e. 0 in case of success).
"""
remove_args = [
"-y",
"--name"
]
remove_args.extend(args)
return self.exec_command("env remove", remove_args)
[docs] def exec_install(self, args, allow_local=True, stdout_path=None):
"""
Return the process exit code (i.e. 0 in case of success).
"""
install_args = [
"-y"
]
if allow_local and self.use_local:
install_args.append("--use-local")
install_args.extend(self._override_channels_args)
install_args.extend(args)
return self.exec_command("install", install_args, stdout_path=stdout_path)
[docs] def exec_clean(self, args=None, quiet=False):
"""
Clean up after conda installation.
Return the process exit code (i.e. 0 in case of success).
"""
clean_args = [
"--tarballs",
"-y"
]
if args:
clean_args.extend(args)
stdout_path = None
if quiet:
stdout_path = "/dev/null"
return self.exec_command("clean", clean_args, stdout_path=stdout_path)
[docs] def export_list(self, name, path):
"""
Return the process exit code (i.e. 0 in case of success).
"""
return self.exec_command("list", [
"--name", name,
"--export"
], stdout_path=path)
@property
def envs_path(self):
return os.path.join(self.conda_prefix, "envs")
[docs] def has_env(self, env_name):
env_path = self.env_path(env_name)
return os.path.isdir(env_path)
@property
def deactivate(self):
return self._bin("deactivate")
@property
def activate(self):
return self._bin("activate")
@property
def parent_path(self):
return os.path.dirname(os.path.abspath(self.conda_prefix))
def _bin(self, name):
return os.path.join(self.conda_prefix, "bin", name)
def installed_conda_targets(conda_context):
envs_path = conda_context.envs_path
dir_contents = os.listdir(envs_path) if os.path.exists(envs_path) else []
for name in dir_contents:
versioned_match = VERSIONED_ENV_DIR_NAME.match(name)
if versioned_match:
yield CondaTarget(versioned_match.group(1), versioned_match.group(2))
unversioned_match = UNVERSIONED_ENV_DIR_NAME.match(name)
if unversioned_match:
yield CondaTarget(unversioned_match.group(1))
[docs]@six.python_2_unicode_compatible
class CondaTarget(object):
[docs] def __init__(self, package, version=None, channel=None):
if SHELL_UNSAFE_PATTERN.search(package) is not None:
raise ValueError("Invalid package [%s] encountered." % package)
self.package = package
if version and SHELL_UNSAFE_PATTERN.search(version) is not None:
raise ValueError("Invalid version [%s] encountered." % version)
self.version = version
if channel and SHELL_UNSAFE_PATTERN.search(channel) is not None:
raise ValueError("Invalid version [%s] encountered." % channel)
self.channel = channel
def __str__(self):
attributes = "package=%s" % self.package
if self.version is not None:
attributes = "%s,version=%s" % (self.package, self.version)
else:
attributes = "%s,unversioned" % self.package
if self.channel:
attributes = "%s,channel=%s" % self.channel
return "CondaTarget[%s]" % attributes
__repr__ = __str__
@property
def package_specifier(self):
""" Return a package specifier as consumed by conda install/create.
"""
if self.version:
return "%s=%s" % (self.package, self.version)
else:
return self.package
@property
def install_environment(self):
""" The dependency resolution and installation frameworks will
expect each target to be installed it its own environment with
a fixed and predictable name given package and version.
"""
if self.version:
return "__%s@%s" % (self.package, self.version)
else:
return "__%s@_uv_" % (self.package)
def __hash__(self):
return hash((self.package, self.version, self.channel))
def __eq__(self, other):
if isinstance(other, self.__class__):
return (self.package, self.version, self.channel) == (other.package, other.version, other.channel)
return False
def __ne__(self, other):
return not(self == other)
def hash_conda_packages(conda_packages, conda_target=None):
""" Produce a unique hash on supplied packages.
TODO: Ideally we would do this in such a way that preserved environments.
"""
h = hashlib.new('sha256')
for conda_package in conda_packages:
h.update(smart_str(conda_package.install_environment))
return h.hexdigest()
# shell makes sense for planemo, in Galaxy this should just execute
# these commands as Python
[docs]def install_conda(conda_context, force_conda_build=False):
f, script_path = tempfile.mkstemp(suffix=".sh", prefix="conda_install")
os.close(f)
download_cmd = commands.download_command(conda_link(), to=script_path)
install_cmd = ['bash', script_path, '-b', '-p', conda_context.conda_prefix]
package_targets = [
"conda=%s" % CONDA_VERSION,
]
if force_conda_build or conda_context.use_local:
package_targets.append("conda-build=%s" % CONDA_BUILD_VERSION)
log.info("Installing conda, this may take several minutes.")
try:
exit_code = conda_context.shell_exec(download_cmd)
if exit_code:
return exit_code
exit_code = conda_context.shell_exec(install_cmd)
except Exception:
log.exception('Failed to install conda')
return 1
finally:
if os.path.exists(script_path):
os.remove(script_path)
if exit_code:
return exit_code
return conda_context.exec_install(package_targets, allow_local=False)
def install_conda_targets(conda_targets, conda_context, env_name=None, allow_local=True):
"""
Return the process exit code (i.e. 0 in case of success).
"""
if env_name is not None:
create_args = [
"--name", env_name, # environment for package
]
for conda_target in conda_targets:
create_args.append(conda_target.package_specifier)
return conda_context.exec_create(create_args, allow_local=allow_local)
else:
return conda_context.exec_install([t.package_specifier for t in conda_targets], allow_local=allow_local)
[docs]def install_conda_target(conda_target, conda_context, skip_environment=False):
"""
Install specified target into a its own environment.
Return the process exit code (i.e. 0 in case of success).
"""
if not skip_environment:
create_args = [
"--name", conda_target.install_environment, # environment for package
conda_target.package_specifier,
]
return conda_context.exec_create(create_args)
else:
return conda_context.exec_install([conda_target.package_specifier])
def cleanup_failed_install_of_environment(env, conda_context):
if conda_context.has_env(env):
conda_context.exec_remove([env])
def cleanup_failed_install(conda_target, conda_context=None):
cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
def best_search_result(conda_target, conda_context, channels_override=None, offline=False, platform=None):
"""Find best "conda search" result for specified target.
Return ``None`` if no results match.
"""
search_cmd = []
conda_exec = conda_context.conda_exec
if isinstance(conda_exec, list):
# for CondaInDockerContext
search_cmd.extend(conda_exec)
else:
search_cmd.append(conda_exec)
search_cmd.extend(["search", "--full-name", "--json"])
if offline:
search_cmd.append("--offline")
if platform:
search_cmd.extend(['--platform', platform])
if channels_override:
search_cmd.append("--override-channels")
for channel in channels_override:
search_cmd.extend(["--channel", channel])
else:
search_cmd.extend(conda_context._override_channels_args)
search_cmd.append(conda_target.package)
try:
res = commands.execute(search_cmd)
res = unicodify(res)
# Use python's stable list sorting to sort by date,
# then build_number, then version. The top of the list
# then is the newest version with the newest build and
# the latest update time.
hits = json.loads(res).get(conda_target.package, [])[::-1]
hits = sorted(hits, key=lambda hit: hit['build_number'], reverse=True)
hits = sorted(hits, key=lambda hit: packaging.version.parse(hit['version']), reverse=True)
except commands.CommandLineException:
log.error("Could not execute: '%s'", search_cmd)
hits = []
if len(hits) == 0:
return (None, None)
best_result = (hits[0], False)
for hit in hits:
if is_search_hit_exact(conda_target, hit):
best_result = (hit, True)
break
return best_result
def is_search_hit_exact(conda_target, search_hit):
target_version = conda_target.version
# It'd be nice to make request verson of 1.0 match available
# version of 1.0.3 or something like that.
return not target_version or search_hit['version'] == target_version
def is_conda_target_installed(conda_target, conda_context):
# fail by default
if conda_context.has_env(conda_target.install_environment):
return True
else:
return False
def filter_installed_targets(conda_targets, conda_context):
installed = functools.partial(is_conda_target_installed,
conda_context=conda_context)
return list(filter(installed, conda_targets))
def build_isolated_environment(
conda_packages,
conda_context,
path=None,
copy=False,
quiet=False,
):
""" Build a new environment (or reuse an existing one from hashes)
for specified conda packages.
"""
if not isinstance(conda_packages, list):
conda_packages = [conda_packages]
# Lots we could do in here, hashing, checking revisions, etc...
tempdir = None
try:
hash = hash_conda_packages(conda_packages)
tempdir = tempfile.mkdtemp(prefix="jobdeps", suffix=hash)
tempdir_name = os.path.basename(tempdir)
export_paths = []
for conda_package in conda_packages:
name = conda_package.install_environment
export_path = os.path.join(tempdir, name)
conda_context.export_list(
name,
export_path
)
export_paths.append(export_path)
create_args = ["--unknown"]
# Works in 3.19, 4.0 - 4.2 - not in 4.3.
# Adjust fix if they fix Conda - xref
# - https://github.com/galaxyproject/galaxy/issues/3635
# - https://github.com/conda/conda/issues/2035
offline_works = (conda_context.conda_version < packaging.version.parse("4.3")) or \
(conda_context.conda_version >= packaging.version.parse("4.4"))
if offline_works:
create_args.extend(["--offline"])
else:
create_args.extend(["--use-index-cache"])
if path is None:
create_args.extend(["--name", tempdir_name])
else:
create_args.extend(["--prefix", path])
if copy:
create_args.append("--copy")
for export_path in export_paths:
create_args.extend([
"--file", export_path
])
stdout_path = None
if quiet:
stdout_path = "/dev/null"
if path is not None and os.path.exists(path):
exit_code = conda_context.exec_install(create_args, stdout_path=stdout_path)
else:
exit_code = conda_context.exec_create(create_args, stdout_path=stdout_path)
return (path or tempdir_name, exit_code)
finally:
conda_context.exec_clean(quiet=quiet)
if tempdir is not None:
shutil.rmtree(tempdir)
def requirement_to_conda_targets(requirement):
conda_target = None
if requirement.type == "package":
conda_target = CondaTarget(requirement.name,
version=requirement.version)
return conda_target
[docs]def requirements_to_conda_targets(requirements):
conda_targets = (requirement_to_conda_targets(_) for _ in requirements)
return [c for c in conda_targets if c is not None]
__all__ = (
'CondaContext',
'CondaTarget',
'install_conda',
'install_conda_target',
'requirements_to_conda_targets',
)