#!/usr/bin/env python
"""Build a mulled image for specified conda targets.
Examples
Build a mulled image with:
mulled-build build 'samtools=1.3.1--4,bedtools=2.22'
"""
import json
import logging
import os
import shutil
import stat
import string
import subprocess
import sys
from sys import platform as _platform
from typing import (
Any,
Dict,
Iterable,
List,
)
import yaml
from galaxy.tool_util.deps import installable
from galaxy.tool_util.deps.conda_util import (
best_search_result,
CondaContext,
CondaTarget,
)
from galaxy.tool_util.deps.docker_util import command_list as docker_command_list
from galaxy.util import (
commands,
download_to_file,
safe_makedirs,
shlex_join,
unicodify,
)
from ._cli import arg_parser
from .util import (
build_target,
conda_build_target_str,
create_repository,
default_mulled_conda_channels_from_env,
get_files_from_conda_package,
PrintProgress,
quay_repository,
v1_image_name,
v2_image_name,
)
from ..conda_compat import MetaData
log = logging.getLogger(__name__)
INVFILE = os.environ.get("INVFILE", os.path.join(os.path.dirname(__file__), "invfile.lua"))
DEFAULT_BASE_IMAGE = os.environ.get("DEFAULT_BASE_IMAGE", "quay.io/bioconda/base-glibc-busybox-bash:latest")
DEFAULT_EXTENDED_BASE_IMAGE = os.environ.get(
"DEFAULT_EXTENDED_BASE_IMAGE", "quay.io/bioconda/base-glibc-debian-bash:latest"
)
DEFAULT_CHANNELS = default_mulled_conda_channels_from_env() or ["conda-forge", "bioconda"]
DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}"
DEFAULT_BINDS = ["build/dist:/usr/local/"]
DEFAULT_WORKING_DIR = "/source/"
IS_OS_X = _platform == "darwin"
INVOLUCRO_VERSION = "1.1.2"
DEST_BASE_IMAGE = os.environ.get("DEST_BASE_IMAGE", None)
CONDA_IMAGE = os.environ.get("CONDA_IMAGE", None)
SINGULARITY_TEMPLATE = """Bootstrap: docker
From: %(base_image)s
%%setup
echo "Copying conda environment"
mkdir -p /tmp/conda
cp -r /data/dist/* /tmp/conda/
%%post
rm -R /usr/local || true
mkdir -p /usr/local
cp -R /tmp/conda/* /usr/local/
%%test
%(container_test)s
"""
def involucro_link():
if IS_OS_X:
url = f"https://github.com/mvdbeek/involucro/releases/download/v{INVOLUCRO_VERSION}/involucro.darwin"
else:
url = f"https://github.com/involucro/involucro/releases/download/v{INVOLUCRO_VERSION}/involucro"
return url
def get_tests(args, pkg_path):
"""Extract test cases given a recipe's meta.yaml file."""
recipes_dir = args.recipes_dir
tests = []
input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
recipe_meta = MetaData(input_dir)
tests_commands = recipe_meta.get_value("test/commands")
tests_imports = recipe_meta.get_value("test/imports")
requirements = recipe_meta.get_value("requirements/run")
if tests_imports or tests_commands:
if tests_commands:
tests.append(" && ".join(tests_commands))
if tests_imports and "python" in requirements:
tests.append(" && ".join(f'python -c "import {imp}"' for imp in tests_imports))
elif tests_imports and ("perl" in requirements or "perl-threaded" in requirements):
tests.append(" && ".join(f'''perl -e "use {imp};\"''' for imp in tests_imports))
tests = " && ".join(tests)
tests = tests.replace("$R ", "Rscript ")
return tests
def get_pkg_name(args, pkg_path):
"""Extract the package name from a given meta.yaml file."""
recipes_dir = args.recipes_dir
input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
recipe_meta = MetaData(input_dir)
return recipe_meta.get_value("package/name")
def get_affected_packages(args):
"""Return a list of all meta.yaml file that where modified/created recently.
Length of time to check for indicated by the ``hours`` parameter.
"""
recipes_dir = args.recipes_dir
hours = args.diff_hours
cmd = ["git", "log", "--diff-filter=ACMRTUXB", "--name-only", '--pretty=""', f'--since="{hours} hours ago"']
changed_files = unicodify(subprocess.check_output(cmd, cwd=recipes_dir)).splitlines()
pkg_list = {x for x in changed_files if x.startswith("recipes/") and x.endswith("meta.yaml")}
for pkg in pkg_list:
if pkg and os.path.exists(os.path.join(recipes_dir, pkg)):
yield (get_pkg_name(args, pkg), get_tests(args, pkg))
def conda_versions(pkg_name, file_name):
"""Return all conda version strings for a specified package name."""
j = json.load(open(file_name))
ret = []
for pkg in j["packages"].values():
if pkg["name"] == pkg_name:
ret.append(f"{pkg['version']}--{pkg['build']}")
return ret
def get_conda_hits_for_targets(targets: Iterable[CondaTarget], conda_context: CondaContext) -> List[Dict[str, Any]]:
search_results = (best_search_result(t, conda_context, platform="linux-64")[0] for t in targets)
return [r for r in search_results if r]
def base_image_for_targets(targets: Iterable[CondaTarget], conda_context: CondaContext) -> str:
"""
determine base image (DEFAULT_BASE_IMAGE/DEFAULT_EXTENDED_BASE_IMAGE) for a
list of targets by inspecting the conda package (i.e. if the use of an
extended image is indicated in info/about.json or info/recipe/meta.yaml
"""
hits = get_conda_hits_for_targets(targets, conda_context)
for hit in hits:
try:
content_dict = get_files_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])
if "info/about.json" in content_dict and json.loads(unicodify(content_dict["info/about.json"])).get(
"extra", {}
).get("container", {}).get("extended-base", False):
return DEFAULT_EXTENDED_BASE_IMAGE
elif "info/recipe/meta.yaml" in content_dict and (
yaml.safe_load(unicodify(content_dict["info/recipe/meta.yaml"]))
.get("extra", {})
.get("container", {})
.get("extended-base", False)
):
return DEFAULT_EXTENDED_BASE_IMAGE
except Exception:
log.warning(
"Could not load metadata.yaml for '%s', version '%s'", hit["name"], hit["version"], exc_info=True
)
return DEFAULT_BASE_IMAGE
class BuildExistsException(Exception):
"""Exception indicating mull_targets is skipping an existing build.
If mull_targets is called with rebuild=False and the target built is already published
an instance of this exception is thrown.
"""
def mull_targets(
targets: List[CondaTarget],
involucro_context=None,
command="build",
channels=DEFAULT_CHANNELS,
namespace="biocontainers",
test="true",
test_files=None,
image_build=None,
name_override=None,
repository_template=DEFAULT_REPOSITORY_TEMPLATE,
dry_run=False,
conda_version=None,
mamba_version=None,
use_mamba=False,
verbose=False,
binds=DEFAULT_BINDS,
rebuild=True,
oauth_token=None,
hash_func="v2",
singularity=False,
singularity_image_dir="singularity_import",
base_image=None,
determine_base_image=True,
invfile=INVFILE,
):
if involucro_context is None:
involucro_context = InvolucroContext()
image_function = v1_image_name if hash_func == "v1" else v2_image_name
if len(targets) > 1 and image_build is None:
# Force an image build in this case - this seems hacky probably
# shouldn't work this way but single case broken else wise.
image_build = "0"
repo_template_kwds = {
"namespace": namespace,
"image": image_function(targets, image_build=image_build, name_override=name_override),
}
repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
if not rebuild or "push" in command:
repo_name = repo_template_kwds["image"].split(":", 1)[0]
repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
if not rebuild:
tags = repo_data.get("tags", [])
target_tag = None
if ":" in repo_template_kwds["image"]:
image_name_parts = repo_template_kwds["image"].split(":")
assert len(image_name_parts) == 2, f": not allowed in image name [{repo_template_kwds['image']}]"
target_tag = image_name_parts[1]
if tags and (target_tag is None or target_tag in tags):
raise BuildExistsException()
if "push" in command and "error_type" in repo_data and oauth_token:
# Explicitly create the repository so it can be built as public.
create_repository(repo_template_kwds["namespace"], repo_name, oauth_token)
for channel in channels:
if channel.startswith("file://"):
bind_path = channel[7:]
binds.append(f"{bind_path}:{bind_path}")
channels_str = ",".join(channels)
target_str = ",".join(map(conda_build_target_str, targets))
bind_str = ",".join(binds)
involucro_args = [
"-f",
invfile,
"-set",
f"CHANNELS={channels_str}",
"-set",
f"TARGETS={target_str}",
"-set",
f"REPO={repo}",
"-set",
f"BINDS={bind_str}",
]
dest_base_image = None
if base_image:
dest_base_image = base_image
elif DEST_BASE_IMAGE:
dest_base_image = DEST_BASE_IMAGE
elif determine_base_image:
conda_context = CondaInDockerContext(ensure_channels=channels)
dest_base_image = base_image_for_targets(targets, conda_context)
if dest_base_image:
involucro_args.extend(["-set", f"DEST_BASE_IMAGE={dest_base_image}"])
if CONDA_IMAGE:
involucro_args.extend(["-set", f"CONDA_IMAGE={CONDA_IMAGE}"])
if verbose:
involucro_args.extend(["-set", "VERBOSE=1"])
if singularity:
singularity_image_name = repo_template_kwds["image"]
involucro_args.extend(["-set", "SINGULARITY=1"])
involucro_args.extend(["-set", f"SINGULARITY_IMAGE_NAME={singularity_image_name}"])
involucro_args.extend(["-set", f"SINGULARITY_IMAGE_DIR={singularity_image_dir}"])
involucro_args.extend(["-set", f"USER_ID={os.getuid()}:{os.getgid()}"])
if test:
involucro_args.extend(["-set", f"TEST={test}"])
verbose = "--verbose" if verbose else "--quiet"
conda_bin = "conda"
if use_mamba:
conda_bin = "mamba"
if mamba_version is None:
mamba_version = ""
involucro_args.extend(["-set", f"CONDA_BIN={conda_bin}"])
if conda_version is not None or mamba_version is not None:
mamba_test = "true"
specs = []
if conda_version is not None:
specs.append(f"conda={conda_version}")
if mamba_version is not None:
specs.append(f"mamba={mamba_version}")
if mamba_version == "" and not specs:
# If nothing but mamba without a specific version is requested,
# then only run conda install if mamba is not already installed.
mamba_test = "[ '[]' = \"$( conda list --json --full-name mamba )\" ]"
conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}"""
involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"])
involucro_args.append(command)
if test_files:
test_bind = []
for test_file in test_files:
if ":" not in test_file:
if os.path.exists(test_file):
test_bind.append(f"{test_file}:{DEFAULT_WORKING_DIR}/{test_file}")
else:
if os.path.exists(test_file.split(":")[0]):
test_bind.append(test_file)
if test_bind:
involucro_args.insert(6, "-set")
involucro_args.insert(7, f"TEST_BINDS={','.join(test_bind)}")
cmd = involucro_context.build_command(involucro_args)
print(f"Executing: {shlex_join(cmd)}")
if dry_run:
return 0
ensure_installed(involucro_context, True)
if singularity:
if not os.path.exists(singularity_image_dir):
safe_makedirs(singularity_image_dir)
with open(os.path.join(singularity_image_dir, "Singularity.def"), "w+") as sin_def:
fill_template = SINGULARITY_TEMPLATE % {
"container_test": test,
"base_image": dest_base_image or DEFAULT_BASE_IMAGE,
}
sin_def.write(fill_template)
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')
return ret
def context_from_args(args):
verbose = "2" if not args.verbose else "3"
return InvolucroContext(involucro_bin=args.involucro_path, verbose=verbose)
class CondaInDockerContext(CondaContext):
def __init__(
self,
conda_prefix=None,
conda_exec=None,
shell_exec=None,
debug=False,
ensure_channels=DEFAULT_CHANNELS,
condarc_override=None,
):
if not conda_exec:
conda_image = CONDA_IMAGE or "continuumio/miniconda3:latest"
binds = []
for channel in ensure_channels:
if channel.startswith("file://"):
bind_path = channel[7:]
binds.extend(["-v", f"{bind_path}:{bind_path}"])
conda_exec = docker_command_list("run", binds + [conda_image, "conda"])
super().__init__(
conda_prefix=conda_prefix,
conda_exec=conda_exec,
shell_exec=shell_exec,
debug=debug,
ensure_channels=ensure_channels,
condarc_override=condarc_override,
)
class InvolucroContext(installable.InstallableContext):
installable_description = "Involucro"
def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"):
if involucro_bin is None:
if os.path.exists("./involucro"):
self.involucro_bin = "./involucro"
else:
self.involucro_bin = "involucro"
else:
self.involucro_bin = involucro_bin
self.shell_exec = shell_exec or commands.shell
self.verbose = verbose
def build_command(self, involucro_args):
return [self.involucro_bin, f"-v={self.verbose}"] + involucro_args
def exec_command(self, involucro_args):
cmd = self.build_command(involucro_args)
# Create ./build dir manually, otherwise Docker will do it as root
created_build_dir = False
if not os.path.exists("build"):
created_build_dir = True
os.mkdir("./build")
try:
res = self.shell_exec(cmd)
finally:
# delete build directory in any case
if created_build_dir:
shutil.rmtree("./build")
return res
def is_installed(self):
return os.path.exists(self.involucro_bin)
def can_install(self):
return True
@property
def parent_path(self):
return os.path.dirname(os.path.abspath(self.involucro_bin))
def ensure_installed(involucro_context, auto_init):
return installable.ensure_installed(involucro_context, install_involucro, auto_init)
def install_involucro(involucro_context):
install_path = os.path.abspath(involucro_context.involucro_bin)
involucro_context.involucro_bin = install_path
try:
download_to_file(involucro_link(), install_path)
except Exception:
log.exception(f"Failed to download involucro from url '{involucro_link()}'")
return 1
try:
os.chmod(install_path, os.stat(install_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
return 0
except Exception:
log.exception("Failed to make file '%s' executable", install_path)
return 1
def add_build_arguments(parser):
"""Base arguments describing how to 'mull'."""
parser.add_argument(
"--involucro-path",
dest="involucro_path",
default=None,
help="Path to involucro (if not set will look in working directory and on PATH).",
)
parser.add_argument(
"--dry-run", dest="dry_run", action="store_true", help="Just print commands instead of executing them."
)
parser.add_argument("--verbose", dest="verbose", action="store_true", help="Cause process to be verbose.")
parser.add_argument("--singularity", action="store_true", help="Additionally build a singularity image.")
parser.add_argument(
"--singularity-image-dir", dest="singularity_image_dir", help="Directory to write singularity images too."
)
parser.add_argument("--involucro-lua-file", dest="invfile", default=INVFILE, help="Path to invfile.lua")
parser.add_argument("-n", "--namespace", dest="namespace", default="biocontainers", help="quay.io namespace.")
parser.add_argument(
"-r",
"--repository_template",
dest="repository_template",
default=DEFAULT_REPOSITORY_TEMPLATE,
help="Docker repository target for publication (only quay.io or compat. API is currently supported).",
)
parser.add_argument(
"-c",
"--channels",
dest="channels",
default=",".join(DEFAULT_CHANNELS),
help="Comma separated list of target conda channels.",
)
parser.add_argument(
"--conda-version",
dest="conda_version",
default=None,
help="Change to specified version of Conda before installing packages.",
)
parser.add_argument(
"--mamba-version",
dest="mamba_version",
default=None,
help="Change to specified version of Mamba before installing packages.",
)
parser.add_argument(
"--use-mamba",
dest="use_mamba",
action="store_true",
help="Use Mamba instead of Conda for package installation.",
)
parser.add_argument(
"--oauth-token",
dest="oauth_token",
default=None,
help="If set, use this token when communicating with quay.io API.",
)
parser.add_argument("--check-published", dest="rebuild", action="store_false")
parser.add_argument("--hash", dest="hash", choices=["v1", "v2"], default="v2")
def add_single_image_arguments(parser):
parser.add_argument(
"--name-override",
dest="name_override",
default=None,
help="Override mulled image name - this is not recommended since metadata will not be detectable from the name of resulting images",
)
parser.add_argument(
"--image-build", dest="image_build", default=None, help="Build a versioned variant of this image."
)
def target_str_to_targets(targets_raw):
def parse_target(target_str):
if "=" in target_str:
package_name, version = target_str.split("=", 1)
build = None
if "=" in version:
version, build = version.split("=")
elif "--" in version:
version, build = version.split("--")
target = build_target(package_name, version, build)
else:
target = build_target(target_str)
return target
targets = [parse_target(_) for _ in targets_raw.split(",")]
return targets
def args_to_mull_targets_kwds(args):
kwds = {}
if hasattr(args, "image_build"):
kwds["image_build"] = args.image_build
if hasattr(args, "name_override"):
kwds["name_override"] = args.name_override
if hasattr(args, "namespace"):
kwds["namespace"] = args.namespace
if hasattr(args, "dry_run"):
kwds["dry_run"] = args.dry_run
if hasattr(args, "singularity"):
kwds["singularity"] = args.singularity
if hasattr(args, "test"):
kwds["test"] = args.test
if hasattr(args, "test_files"):
if args.test_files:
kwds["test_files"] = args.test_files.split(",")
if hasattr(args, "channels"):
kwds["channels"] = args.channels.split(",")
if hasattr(args, "command"):
kwds["command"] = args.command
if hasattr(args, "repository_template"):
kwds["repository_template"] = args.repository_template
if hasattr(args, "conda_version"):
kwds["conda_version"] = args.conda_version
if hasattr(args, "mamba_version"):
kwds["mamba_version"] = args.mamba_version
if hasattr(args, "use_mamba"):
kwds["use_mamba"] = args.use_mamba
if hasattr(args, "oauth_token"):
kwds["oauth_token"] = args.oauth_token
if hasattr(args, "rebuild"):
kwds["rebuild"] = args.rebuild
if hasattr(args, "hash"):
kwds["hash_func"] = args.hash
if hasattr(args, "singularity_image_dir") and args.singularity_image_dir:
kwds["singularity_image_dir"] = args.singularity_image_dir
if hasattr(args, "invfile"):
kwds["invfile"] = args.invfile
kwds["involucro_context"] = context_from_args(args)
return kwds
[docs]def main(argv=None):
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
add_build_arguments(parser)
add_single_image_arguments(parser)
parser.add_argument("command", metavar="COMMAND", help="Command (build-and-test, build, all)")
parser.add_argument(
"targets", metavar="TARGETS", default=None, help="Build a single container with specific package(s)."
)
parser.add_argument(
"--repository-name",
dest="repository_name",
default=None,
help="Name of mulled container (leave blank to auto-generate based on packages - recommended).",
)
parser.add_argument("--test", help="Provide a test command for the container.")
parser.add_argument(
"--test-files",
help="Provide test-files that may be required to run the test command. Individual mounts are separated by comma."
"The source:dest docker syntax is respected. If relative file paths are given, files will be mounted in /source/<relative_file_path>",
)
args = parser.parse_args()
targets = target_str_to_targets(args.targets)
sys.exit(mull_targets(targets, **args_to_mull_targets_kwds(args)))
__all__ = (
"main",
"build_target",
)
if __name__ == "__main__":
main()