import functools
import hashlib
import json
import logging
import os
import re
import shutil
import sys
import tempfile
from typing import (
List,
Optional,
)
import packaging.version
from galaxy.util import (
commands,
listify,
shlex_join,
smart_str,
which,
)
from . import installable
log = logging.getLogger(__name__)
# Not sure there are security concerns, lets just fail fast if we are going
# break shell commands we are building.
SHELL_UNSAFE_PATTERN = re.compile(r"[\s\"']")
IS_OS_X = sys.platform == "darwin"
VERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@(.*)")
UNVERSIONED_ENV_DIR_NAME = re.compile(r"__(.*)@_uv_")
USE_PATH_EXEC_DEFAULT = False
CONDA_PACKAGE_SPECS = ("conda=4.6.14", "'pyopenssl>=22.1.0'")
CONDA_BUILD_VERSION = "3.17.8"
USE_LOCAL_DEFAULT = False
def conda_link():
if IS_OS_X:
url = "https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-MacOSX-x86_64.sh"
else:
if sys.maxsize > 2**32:
url = "https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh"
else:
url = "https://repo.anaconda.com/miniconda/Miniconda3-4.5.12-Linux-x86.sh"
return url
def find_conda_prefix(conda_prefix=None):
"""If supplied conda_prefix is not set, default to the default location
for Miniconda installs.
"""
if conda_prefix is None:
home = os.path.expanduser("~")
miniconda_2_dest = os.path.join(home, "miniconda2")
miniconda_3_dest = os.path.join(home, "miniconda3")
anaconda_2_dest = os.path.join(home, "anaconda2")
anaconda_3_dest = os.path.join(home, "anaconda3")
# Prefer miniconda3 install if both available
if os.path.exists(miniconda_3_dest):
return miniconda_3_dest
elif os.path.exists(miniconda_2_dest):
return miniconda_2_dest
elif os.path.exists(anaconda_3_dest):
return anaconda_3_dest
elif os.path.exists(anaconda_2_dest):
return anaconda_2_dest
else:
return miniconda_3_dest
return conda_prefix
[docs]class CondaContext(installable.InstallableContext):
installable_description = "Conda"
[docs] def __init__(
self,
conda_prefix=None,
conda_exec=None,
shell_exec=None,
debug=False,
ensure_channels="",
condarc_override=None,
use_path_exec=USE_PATH_EXEC_DEFAULT,
copy_dependencies=False,
use_local=USE_LOCAL_DEFAULT,
):
self.condarc_override = condarc_override
if not conda_exec and use_path_exec:
conda_exec = which("conda")
if conda_exec and isinstance(conda_exec, str):
conda_exec = os.path.normpath(conda_exec)
self.conda_exec = conda_exec
self.debug = debug
self.shell_exec = shell_exec or commands.shell
self.copy_dependencies = copy_dependencies
if conda_prefix is None:
info = self.conda_info()
if info and "default_prefix" in info:
conda_prefix = info["default_prefix"]
if conda_prefix is None:
conda_prefix = find_conda_prefix(conda_prefix)
self.conda_prefix = conda_prefix
if conda_exec is None:
self.conda_exec = self._bin("conda")
self.ensure_channels: List[str] = listify(ensure_channels)
self.use_local = use_local
self._reset_conda_properties()
def _reset_conda_properties(self):
self._conda_version = None
self._conda_build_available = None
@property
def conda_version(self):
if self._conda_version is None:
self._guess_conda_properties()
return self._conda_version
@property
def conda_build_available(self):
if self._conda_build_available is None:
self._guess_conda_properties()
return self._conda_build_available
def _guess_conda_properties(self):
info = self.conda_info()
self._conda_version = packaging.version.parse(info["conda_version"])
self._conda_build_available = False
conda_build_version = info.get("conda_build_version")
if conda_build_version and conda_build_version != "not installed":
try:
packaging.version.parse(conda_build_version)
self._conda_build_available = True
except Exception:
pass
@property
def _override_channels_args(self):
override_channels_args = []
if self.ensure_channels:
override_channels_args.append("--override-channels")
for channel in self.ensure_channels:
override_channels_args.extend(["--channel", channel])
return override_channels_args
[docs] def ensure_conda_build_installed_if_needed(self):
if self.use_local and not self.conda_build_available:
conda_targets = [CondaTarget("conda-build", version=CONDA_BUILD_VERSION)]
# Cannot use --use-local during installation of conda-build.
return install_conda_targets(conda_targets, conda_context=self, env_name=None, allow_local=False)
else:
return 0
[docs] def conda_info(self):
if self.conda_exec is not None:
cmd = listify(self.conda_exec) + ["info", "--json"]
info_out = commands.execute(cmd)
info = json.loads(info_out)
return info
else:
return None
[docs] def is_conda_installed(self):
"""
Check if conda_exec exists
"""
if os.path.exists(self.conda_exec):
return True
else:
return False
[docs] def can_install_conda(self):
"""
If conda_exec is set to a path outside of conda_prefix,
there is no use installing conda into conda_prefix, since it can't be used by galaxy.
If conda_exec equals conda_prefix/bin/conda, we can install conda if either conda_prefix
does not exist or is empty.
"""
conda_exec = os.path.abspath(self.conda_exec)
conda_prefix_plus_exec = os.path.abspath(os.path.join(self.conda_prefix, "bin/conda"))
if conda_exec == conda_prefix_plus_exec:
if not os.path.exists(self.conda_prefix):
return True
elif os.listdir(self.conda_prefix) == []:
os.rmdir(self.conda_prefix) # Conda's install script fails if path exists (even if empty).
return True
else:
log.warning(
"Cannot install Conda because conda_prefix '%s' exists and is not empty.", self.conda_prefix
)
return False
else:
log.warning(
"Skipping installation of Conda into conda_prefix '%s', "
"since conda_exec '%s' is set to a path outside of conda_prefix.",
self.conda_prefix,
self.conda_exec,
)
return False
[docs] def exec_command(self, operation, args, stdout_path=None):
"""
Execute the requested command.
Return the process exit code (i.e. 0 in case of success).
"""
cmd = listify(self.conda_exec) + operation.split()
if self.debug:
cmd.append("--debug")
cmd.extend(args)
env = {}
if self.condarc_override:
env["CONDARC"] = self.condarc_override
cmd_string = shlex_join(cmd)
kwds = dict()
try:
if stdout_path:
kwds["stdout"] = open(stdout_path, "w")
cmd_string += f" > '{stdout_path}'"
conda_exec_home = env["HOME"] = tempfile.mkdtemp(
prefix="conda_exec_home_"
) # We don't want to pollute ~/.conda, which may not even be writable
log.debug("Executing command: %s", cmd_string)
return self.shell_exec(cmd, env=env, **kwds)
except Exception:
log.exception("Failed to execute command: %s", cmd_string)
return 1
finally:
if kwds.get("stdout"):
kwds["stdout"].close()
if conda_exec_home:
shutil.rmtree(conda_exec_home, ignore_errors=True)
[docs] def exec_create(self, args, allow_local=True, stdout_path=None):
"""
Return the process exit code (i.e. 0 in case of success).
"""
for try_strict in [True, False]:
create_args = ["-y", "--quiet"]
if try_strict:
if self.conda_version >= packaging.version.parse("4.7.5"):
create_args.append("--strict-channel-priority")
else:
continue
if allow_local and self.use_local:
create_args.append("--use-local")
create_args.extend(self._override_channels_args)
create_args.extend(args)
ret = self.exec_command("create", create_args, stdout_path=stdout_path)
if ret == 0:
break
return ret
[docs] def exec_remove(self, args):
"""
Remove a conda environment using conda env remove -y --name `args`.
Return the process exit code (i.e. 0 in case of success).
"""
remove_args = ["-y", "--name"]
remove_args.extend(args)
return self.exec_command("env remove", remove_args)
[docs] def exec_install(self, args, allow_local=True, stdout_path=None):
"""
Return the process exit code (i.e. 0 in case of success).
"""
for try_strict in [True, False]:
install_args = ["-y"]
if try_strict:
if self.conda_version >= packaging.version.parse("4.7.5"):
install_args.append("--strict-channel-priority")
else:
continue
if allow_local and self.use_local:
install_args.append("--use-local")
install_args.extend(self._override_channels_args)
install_args.extend(args)
ret = self.exec_command("install", install_args, stdout_path=stdout_path)
if ret == 0:
break
if ret == 0:
self._reset_conda_properties()
return ret
[docs] def exec_clean(self, args=None, quiet=False):
"""
Clean up after conda installation.
Return the process exit code (i.e. 0 in case of success).
"""
clean_args = ["--tarballs", "-y"]
if args:
clean_args.extend(args)
stdout_path = None
if quiet:
stdout_path = "/dev/null"
return self.exec_command("clean", clean_args, stdout_path=stdout_path)
[docs] def exec_search(self, args: List[str], json: bool = False, offline: bool = False, platform: Optional[str] = None):
"""
Search conda channels for a package
Return the standard output of the conda process.
"""
cmd = listify(self.conda_exec)[:]
cmd.append("search")
cmd.extend(self._override_channels_args)
if json:
cmd.append("--json")
if offline:
cmd.append("--offline")
if platform:
cmd.extend(["--platform", platform])
cmd.extend(args)
return commands.execute(cmd)
[docs] def export_list(self, name, path):
"""
Return the process exit code (i.e. 0 in case of success).
"""
return self.exec_command("list", ["--name", name, "--export"], stdout_path=path)
[docs] def env_path(self, env_name):
return os.path.join(self.envs_path, env_name)
@property
def envs_path(self):
return os.path.join(self.conda_prefix, "envs")
[docs] def has_env(self, env_name):
env_path = self.env_path(env_name)
return os.path.isdir(env_path)
@property
def deactivate(self):
return self._bin("deactivate")
@property
def activate(self):
return self._bin("activate")
[docs] def is_installed(self):
return self.is_conda_installed()
[docs] def can_install(self):
return self.can_install_conda()
@property
def parent_path(self):
return os.path.dirname(os.path.abspath(self.conda_prefix))
def _bin(self, name):
return os.path.join(self.conda_prefix, "bin", name)
def installed_conda_targets(conda_context):
envs_path = conda_context.envs_path
dir_contents = os.listdir(envs_path) if os.path.exists(envs_path) else []
for name in dir_contents:
versioned_match = VERSIONED_ENV_DIR_NAME.match(name)
if versioned_match:
yield CondaTarget(versioned_match.group(1), versioned_match.group(2))
unversioned_match = UNVERSIONED_ENV_DIR_NAME.match(name)
if unversioned_match:
yield CondaTarget(unversioned_match.group(1))
[docs]class CondaTarget:
[docs] def __init__(self, package, version=None, channel=None):
if SHELL_UNSAFE_PATTERN.search(package) is not None:
raise ValueError(f"Invalid package [{package}] encountered.")
self.package = package
if version and SHELL_UNSAFE_PATTERN.search(version) is not None:
raise ValueError(f"Invalid version [{version}] encountered.")
self.version = version
if channel and SHELL_UNSAFE_PATTERN.search(channel) is not None:
raise ValueError(f"Invalid version [{channel}] encountered.")
self.channel = channel
def __str__(self):
attributes = f"package={self.package}"
if self.version is not None:
attributes = f"{self.package},version={self.version}"
else:
attributes = f"{self.package},unversioned"
if self.channel:
attributes = "%s,channel=%s" % self.channel
return f"CondaTarget[{attributes}]"
__repr__ = __str__
@property
def package_specifier(self):
"""Return a package specifier as consumed by conda install/create."""
if self.version:
return f"{self.package}={self.version}"
else:
return self.package
@property
def install_environment(self):
"""The dependency resolution and installation frameworks will
expect each target to be installed it its own environment with
a fixed and predictable name given package and version.
"""
if self.version:
return f"__{self.package}@{self.version}"
else:
return f"__{self.package}@_uv_"
def __hash__(self):
return hash((self.package, self.version, self.channel))
def __eq__(self, other):
if isinstance(other, self.__class__):
return (self.package, self.version, self.channel) == (other.package, other.version, other.channel)
return False
def __ne__(self, other):
return not (self == other)
def hash_conda_packages(conda_packages, conda_target=None):
"""Produce a unique hash on supplied packages.
TODO: Ideally we would do this in such a way that preserved environments.
"""
h = hashlib.new("sha256")
for conda_package in conda_packages:
h.update(smart_str(conda_package.install_environment))
return h.hexdigest()
# shell makes sense for planemo, in Galaxy this should just execute
# these commands as Python
[docs]def install_conda(conda_context, force_conda_build=False):
with tempfile.NamedTemporaryFile(suffix=".sh", prefix="conda_install", delete=False) as temp:
script_path = temp.name
download_cmd = commands.download_command(conda_link(), to=script_path)
install_cmd = ["bash", script_path, "-b", "-p", conda_context.conda_prefix]
package_targets = list(CONDA_PACKAGE_SPECS)
if force_conda_build or conda_context.use_local:
package_targets.append(f"conda-build={CONDA_BUILD_VERSION}")
log.info("Installing conda, this may take several minutes.")
try:
exit_code = conda_context.shell_exec(download_cmd)
if exit_code:
return exit_code
exit_code = conda_context.shell_exec(install_cmd)
except Exception:
log.exception("Failed to install conda")
return 1
finally:
if os.path.exists(script_path):
os.remove(script_path)
if exit_code:
return exit_code
return conda_context.exec_install(package_targets, allow_local=False)
def install_conda_targets(conda_targets, conda_context, env_name=None, allow_local=True):
"""
Return the process exit code (i.e. 0 in case of success).
"""
if env_name is not None:
create_args = [
"--name",
env_name, # environment for package
]
for conda_target in conda_targets:
create_args.append(conda_target.package_specifier)
return conda_context.exec_create(create_args, allow_local=allow_local)
else:
return conda_context.exec_install([t.package_specifier for t in conda_targets], allow_local=allow_local)
[docs]def install_conda_target(conda_target, conda_context, skip_environment=False):
"""
Install specified target into a its own environment.
Return the process exit code (i.e. 0 in case of success).
"""
if not skip_environment:
create_args = [
"--name",
conda_target.install_environment, # environment for package
conda_target.package_specifier,
]
return conda_context.exec_create(create_args)
else:
return conda_context.exec_install([conda_target.package_specifier])
def cleanup_failed_install_of_environment(env, conda_context):
if conda_context.has_env(env):
conda_context.exec_remove([env])
def cleanup_failed_install(conda_target, conda_context=None):
cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
def best_search_result(
conda_target, conda_context: CondaContext, offline: bool = False, platform: Optional[str] = None
):
"""Find best "conda search" result for specified target.
Return ``None`` if no results match.
"""
search_args = [conda_target.package]
try:
res = conda_context.exec_search(search_args, json=True, offline=offline, platform=platform)
# Use python's stable list sorting to sort by date,
# then build_number, then version. The top of the list
# then is the newest version with the newest build and
# the latest update time.
hits = json.loads(res).get(conda_target.package, [])[::-1]
hits = sorted(hits, key=lambda hit: hit["build_number"], reverse=True)
hits = sorted(hits, key=lambda hit: packaging.version.parse(hit["version"]), reverse=True)
except commands.CommandLineException as e:
log.error(f"Could not execute: '{e.command}'\n{e}")
hits = []
if len(hits) == 0:
return (None, None)
best_result = (hits[0], False)
for hit in hits:
if is_search_hit_exact(conda_target, hit):
best_result = (hit, True)
break
return best_result
def is_search_hit_exact(conda_target, search_hit):
target_version = conda_target.version
# It'd be nice to make request verson of 1.0 match available
# version of 1.0.3 or something like that.
return not target_version or search_hit["version"] == target_version
def is_conda_target_installed(conda_target, conda_context):
# fail by default
if conda_context.has_env(conda_target.install_environment):
return True
else:
return False
def filter_installed_targets(conda_targets, conda_context):
installed = functools.partial(is_conda_target_installed, conda_context=conda_context)
return list(filter(installed, conda_targets))
def build_isolated_environment(
conda_packages,
conda_context,
path=None,
copy=False,
quiet=False,
):
"""Build a new environment (or reuse an existing one from hashes)
for specified conda packages.
"""
if not isinstance(conda_packages, list):
conda_packages = [conda_packages]
# Lots we could do in here, hashing, checking revisions, etc...
tempdir = None
try:
hash = hash_conda_packages(conda_packages)
tempdir = tempfile.mkdtemp(prefix="jobdeps", suffix=hash)
tempdir_name = os.path.basename(tempdir)
export_paths = []
for conda_package in conda_packages:
name = conda_package.install_environment
export_path = os.path.join(tempdir, name)
conda_context.export_list(name, export_path)
export_paths.append(export_path)
create_args = ["--unknown"]
# Works in 3.19, 4.0 - 4.2 - not in 4.3.
# Adjust fix if they fix Conda - xref
# - https://github.com/galaxyproject/galaxy/issues/3635
# - https://github.com/conda/conda/issues/2035
offline_works = (conda_context.conda_version < packaging.version.parse("4.3")) or (
conda_context.conda_version >= packaging.version.parse("4.4")
)
if offline_works:
create_args.extend(["--offline"])
else:
create_args.extend(["--use-index-cache"])
if path is None:
create_args.extend(["--name", tempdir_name])
else:
create_args.extend(["--prefix", path])
if copy:
create_args.append("--copy")
for export_path in export_paths:
create_args.extend(["--file", export_path])
stdout_path = None
if quiet:
stdout_path = "/dev/null"
if path is not None and os.path.exists(path):
exit_code = conda_context.exec_install(create_args, stdout_path=stdout_path)
else:
exit_code = conda_context.exec_create(create_args, stdout_path=stdout_path)
return (path or tempdir_name, exit_code)
finally:
conda_context.exec_clean(quiet=quiet)
if tempdir is not None:
shutil.rmtree(tempdir)
def requirement_to_conda_targets(requirement):
conda_target = None
if requirement.type == "package":
conda_target = CondaTarget(requirement.name, version=requirement.version)
return conda_target
[docs]def requirements_to_conda_targets(requirements):
conda_targets = (requirement_to_conda_targets(_) for _ in requirements)
return [c for c in conda_targets if c is not None]
__all__ = (
"CondaContext",
"CondaTarget",
"install_conda",
"install_conda_target",
"requirements_to_conda_targets",
)