Warning
This document is for an in-development version of Galaxy. You can alternatively view this page in the latest release if it exists or view the top of the latest release's documentation.
Source code for galaxy.tool_util.deps.mulled.mulled_build
#!/usr/bin/env python
"""Build a mulled image for specified conda targets.
Examples
Build a mulled image with:
mulled-build build 'samtools=1.3.1--4,bedtools=2.22'
"""
import json
import logging
import os
import shutil
import stat
import string
import subprocess
import sys
from sys import platform as _platform
from six.moves import shlex_quote
try:
import yaml
except ImportError:
yaml = None
from galaxy.tool_util.deps import installable
from galaxy.tool_util.deps.conda_util import best_search_result
from galaxy.tool_util.deps.docker_util import command_list as docker_command_list
from galaxy.util import (
commands,
safe_makedirs,
unicodify,
)
from ._cli import arg_parser
from .util import (
build_target,
conda_build_target_str,
create_repository,
get_file_from_recipe_url,
PrintProgress,
quay_repository,
v1_image_name,
v2_image_name,
)
from ..conda_compat import MetaData
log = logging.getLogger(__name__)
DIRNAME = os.path.dirname(__file__)
DEFAULT_BASE_IMAGE = "bgruening/busybox-bash:0.1"
DEFAULT_EXTENDED_BASE_IMAGE = "bioconda/extended-base-image:latest"
DEFAULT_CHANNELS = ["conda-forge", "bioconda"]
DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}"
DEFAULT_BINDS = ["build/dist:/usr/local/"]
DEFAULT_WORKING_DIR = '/source/'
IS_OS_X = _platform == "darwin"
INVOLUCRO_VERSION = "1.1.2"
DEST_BASE_IMAGE = os.environ.get('DEST_BASE_IMAGE', None)
CONDA_IMAGE = os.environ.get('CONDA_IMAGE', None)
SINGULARITY_TEMPLATE = """Bootstrap: docker
From: %(base_image)s
%%setup
echo "Copying conda environment"
mkdir -p /tmp/conda
cp -r /data/dist/* /tmp/conda/
%%post
rm -R /usr/local || true
mkdir -p /usr/local
cp -R /tmp/conda/* /usr/local/
%%test
%(container_test)s
"""
def involucro_link():
if IS_OS_X:
url = "https://github.com/mvdbeek/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION
else:
url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION
return url
def get_tests(args, pkg_path):
"""Extract test cases given a recipe's meta.yaml file."""
recipes_dir = args.recipes_dir
tests = []
input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
recipe_meta = MetaData(input_dir)
tests_commands = recipe_meta.get_value('test/commands')
tests_imports = recipe_meta.get_value('test/imports')
requirements = recipe_meta.get_value('requirements/run')
if tests_imports or tests_commands:
if tests_commands:
tests.append(' && '.join(tests_commands))
if tests_imports and 'python' in requirements:
tests.append(' && '.join('python -c "import %s"' % imp for imp in tests_imports))
elif tests_imports and ('perl' in requirements or 'perl-threaded' in requirements):
tests.append(' && '.join('''perl -e "use %s;"''' % imp for imp in tests_imports))
tests = ' && '.join(tests)
tests = tests.replace('$R ', 'Rscript ')
return tests
def get_pkg_name(args, pkg_path):
"""Extract the package name from a given meta.yaml file."""
recipes_dir = args.recipes_dir
input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
recipe_meta = MetaData(input_dir)
return recipe_meta.get_value('package/name')
def get_affected_packages(args):
"""Return a list of all meta.yaml file that where modified/created recently.
Length of time to check for indicated by the ``hours`` parameter.
"""
recipes_dir = args.recipes_dir
hours = args.diff_hours
cmd = ['git', 'log', '--diff-filter=ACMRTUXB', '--name-only', '--pretty=""', '--since="%s hours ago"' % hours]
changed_files = unicodify(subprocess.check_output(cmd, cwd=recipes_dir)).splitlines()
pkg_list = {x for x in changed_files if x.startswith('recipes/') and x.endswith('meta.yaml')}
for pkg in pkg_list:
if pkg and os.path.exists(os.path.join(recipes_dir, pkg)):
yield (get_pkg_name(args, pkg), get_tests(args, pkg))
def conda_versions(pkg_name, file_name):
"""Return all conda version strings for a specified package name."""
j = json.load(open(file_name))
ret = list()
for pkg in j['packages'].values():
if pkg['name'] == pkg_name:
ret.append('{}--{}'.format(pkg['version'], pkg['build']))
return ret
def get_conda_hits_for_targets(targets, conda_context):
search_results = (best_search_result(t, conda_context, platform='linux-64')[0] for t in targets)
return [r for r in search_results if r]
def base_image_for_targets(targets, conda_context=None):
hits = get_conda_hits_for_targets(targets, conda_context or CondaInDockerContext())
for hit in hits:
try:
tarball = get_file_from_recipe_url(hit['url'])
meta_content = unicodify(tarball.extractfile('info/about.json').read())
if json.loads(meta_content).get('extra', {}).get('container', {}).get('extended-base', False):
return DEFAULT_EXTENDED_BASE_IMAGE
elif yaml.safe_load(unicodify(tarball.extractfile('info/recipe/meta.yaml').read())).get('extra', {}).get('container', {}).get('extended-base', False):
return DEFAULT_EXTENDED_BASE_IMAGE
except Exception:
log.warning("Could not load metadata.yaml for '%s', version '%s'", hit['name'], hit['version'], exc_info=True)
return DEFAULT_BASE_IMAGE
class BuildExistsException(Exception):
"""Exception indicating mull_targets is skipping an existing build.
If mull_targets is called with rebuild=False and the target built is already published
an instance of this exception is thrown.
"""
def mull_targets(
targets, involucro_context=None,
command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers",
test='true', test_files=None, image_build=None, name_override=None,
repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True,
oauth_token=None, hash_func="v2", singularity=False,
singularity_image_dir="singularity_import", base_image=None,
determine_base_image=True,
):
targets = list(targets)
if involucro_context is None:
involucro_context = InvolucroContext()
image_function = v1_image_name if hash_func == "v1" else v2_image_name
if len(targets) > 1 and image_build is None:
# Force an image build in this case - this seems hacky probably
# shouldn't work this way but single case broken else wise.
image_build = "0"
repo_template_kwds = {
"namespace": namespace,
"image": image_function(targets, image_build=image_build, name_override=name_override)
}
repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
if not rebuild or "push" in command:
repo_name = repo_template_kwds["image"].split(":", 1)[0]
repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
if not rebuild:
tags = repo_data.get("tags", [])
target_tag = None
if ":" in repo_template_kwds["image"]:
image_name_parts = repo_template_kwds["image"].split(":")
assert len(image_name_parts) == 2, ": not allowed in image name [%s]" % repo_template_kwds["image"]
target_tag = image_name_parts[1]
if tags and (target_tag is None or target_tag in tags):
raise BuildExistsException()
if "push" in command and "error_type" in repo_data and oauth_token:
# Explicitly create the repository so it can be built as public.
create_repository(repo_template_kwds["namespace"], repo_name, oauth_token)
for channel in channels:
if channel.startswith('file://'):
bind_path = channel.lstrip('file://')
binds.append('/{}:/{}'.format(bind_path, bind_path))
channels = ",".join(channels)
target_str = ",".join(map(conda_build_target_str, targets))
bind_str = ",".join(binds)
involucro_args = [
'-f', '%s/invfile.lua' % DIRNAME,
'-set', "CHANNELS=%s" % channels,
'-set', "TARGETS=%s" % target_str,
'-set', "REPO=%s" % repo,
'-set', "BINDS=%s" % bind_str,
]
dest_base_image = None
if base_image:
dest_base_image = base_image
elif DEST_BASE_IMAGE:
dest_base_image = DEST_BASE_IMAGE
elif determine_base_image:
dest_base_image = base_image_for_targets(targets)
if dest_base_image:
involucro_args.extend(["-set", "DEST_BASE_IMAGE=%s" % dest_base_image])
if CONDA_IMAGE:
involucro_args.extend(["-set", "CONDA_IMAGE=%s" % CONDA_IMAGE])
if verbose:
involucro_args.extend(["-set", "VERBOSE=1"])
if singularity:
singularity_image_name = repo_template_kwds['image']
involucro_args.extend(["-set", "SINGULARITY=1"])
involucro_args.extend(["-set", "SINGULARITY_IMAGE_NAME=%s" % singularity_image_name])
involucro_args.extend(["-set", "SINGULARITY_IMAGE_DIR=%s" % singularity_image_dir])
involucro_args.extend(["-set", "USER_ID={}:{}".format(os.getuid(), os.getgid())])
if test:
involucro_args.extend(["-set", "TEST=%s" % test])
if conda_version is not None:
verbose = "--verbose" if verbose else "--quiet"
involucro_args.extend(["-set", "PREINSTALL=conda install {} --yes conda={}".format(verbose, conda_version)])
involucro_args.append(command)
if test_files:
test_bind = []
for test_file in test_files:
if ':' not in test_file:
if os.path.exists(test_file):
test_bind.append("{}:{}/{}".format(test_file, DEFAULT_WORKING_DIR, test_file))
else:
if os.path.exists(test_file.split(':')[0]):
test_bind.append(test_file)
if test_bind:
involucro_args.insert(6, '-set')
involucro_args.insert(7, "TEST_BINDS=%s" % ",".join(test_bind))
cmd = involucro_context.build_command(involucro_args)
print('Executing: ' + ' '.join(shlex_quote(_) for _ in cmd))
if dry_run:
return 0
ensure_installed(involucro_context, True)
if singularity:
if not os.path.exists(singularity_image_dir):
safe_makedirs(singularity_image_dir)
with open(os.path.join(singularity_image_dir, 'Singularity.def'), 'w+') as sin_def:
fill_template = SINGULARITY_TEMPLATE % {'container_test': test, 'base_image': dest_base_image or DEFAULT_BASE_IMAGE}
sin_def.write(fill_template)
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')
return ret
def context_from_args(args):
verbose = "2" if not args.verbose else "3"
return InvolucroContext(involucro_bin=args.involucro_path, verbose=verbose)
class CondaInDockerContext:
@property
def conda_exec(self):
conda_image = CONDA_IMAGE or 'continuumio/miniconda3:latest'
return docker_command_list('run', [conda_image, 'conda'])
@property
def _override_channels_args(self):
override_channels_args = ['--override-channels']
for channel in DEFAULT_CHANNELS:
override_channels_args.extend(["--channel", channel])
return override_channels_args
class InvolucroContext(installable.InstallableContext):
installable_description = "Involucro"
def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"):
if involucro_bin is None:
if os.path.exists("./involucro"):
self.involucro_bin = "./involucro"
else:
self.involucro_bin = "involucro"
else:
self.involucro_bin = involucro_bin
self.shell_exec = shell_exec or commands.shell
self.verbose = verbose
def build_command(self, involucro_args):
return [self.involucro_bin, "-v=%s" % self.verbose] + involucro_args
def exec_command(self, involucro_args):
cmd = self.build_command(involucro_args)
# Create ./build dir manually, otherwise Docker will do it as root
created_build_dir = False
if not os.path.exists('build'):
created_build_dir = True
os.mkdir('./build')
try:
res = self.shell_exec(cmd)
finally:
# delete build directory in any case
if created_build_dir:
shutil.rmtree('./build')
return res
def is_installed(self):
return os.path.exists(self.involucro_bin)
def can_install(self):
return True
@property
def parent_path(self):
return os.path.dirname(os.path.abspath(self.involucro_bin))
def ensure_installed(involucro_context, auto_init):
return installable.ensure_installed(involucro_context, install_involucro, auto_init)
def install_involucro(involucro_context):
install_path = os.path.abspath(involucro_context.involucro_bin)
involucro_context.involucro_bin = install_path
download_cmd = commands.download_command(involucro_link(), to=install_path)
exit_code = involucro_context.shell_exec(download_cmd)
if exit_code:
return exit_code
try:
os.chmod(install_path, os.stat(install_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
return 0
except Exception:
log.exception("Failed to make file '%s' executable", install_path)
return 1
def add_build_arguments(parser):
"""Base arguments describing how to 'mull'."""
parser.add_argument('--involucro-path', dest="involucro_path", default=None,
help="Path to involucro (if not set will look in working directory and on PATH).")
parser.add_argument('--dry-run', dest='dry_run', action="store_true",
help='Just print commands instead of executing them.')
parser.add_argument('--verbose', dest='verbose', action="store_true",
help='Cause process to be verbose.')
parser.add_argument('--singularity', action="store_true",
help='Additionally build a singularity image.')
parser.add_argument('--singularity-image-dir', dest="singularity_image_dir",
help="Directory to write singularity images too.")
parser.add_argument('-n', '--namespace', dest='namespace', default="biocontainers",
help='quay.io namespace.')
parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE,
help='Docker repository target for publication (only quay.io or compat. API is currently supported).')
parser.add_argument('-c', '--channels', dest='channels', default=",".join(DEFAULT_CHANNELS),
help='Comma separated list of target conda channels.')
parser.add_argument('--conda-version', dest="conda_version", default=None,
help="Change to specified version of Conda before installing packages.")
parser.add_argument('--oauth-token', dest="oauth_token", default=None,
help="If set, use this token when communicating with quay.io API.")
parser.add_argument('--check-published', dest="rebuild", action='store_false')
parser.add_argument('--hash', dest="hash", choices=["v1", "v2"], default="v2")
def add_single_image_arguments(parser):
parser.add_argument("--name-override", dest="name_override", default=None,
help="Override mulled image name - this is not recommended since metadata will not be detectable from the name of resulting images")
parser.add_argument("--image-build", dest="image_build", default=None,
help="Build a versioned variant of this image.")
def target_str_to_targets(targets_raw):
def parse_target(target_str):
if "=" in target_str:
package_name, version = target_str.split("=", 1)
build = None
if "=" in version:
version, build = version.split('=')
elif "--" in version:
version, build = version.split('--')
target = build_target(package_name, version, build)
else:
target = build_target(target_str)
return target
targets = [parse_target(_) for _ in targets_raw.split(",")]
return targets
def args_to_mull_targets_kwds(args):
kwds = {}
if hasattr(args, "image_build"):
kwds["image_build"] = args.image_build
if hasattr(args, "name_override"):
kwds["name_override"] = args.name_override
if hasattr(args, "namespace"):
kwds["namespace"] = args.namespace
if hasattr(args, "dry_run"):
kwds["dry_run"] = args.dry_run
if hasattr(args, "singularity"):
kwds["singularity"] = args.singularity
if hasattr(args, "test"):
kwds["test"] = args.test
if hasattr(args, "test_files"):
if args.test_files:
kwds["test_files"] = args.test_files.split(",")
if hasattr(args, "channels"):
kwds["channels"] = args.channels.split(',')
if hasattr(args, "command"):
kwds["command"] = args.command
if hasattr(args, "repository_template"):
kwds["repository_template"] = args.repository_template
if hasattr(args, "conda_version"):
kwds["conda_version"] = args.conda_version
if hasattr(args, "oauth_token"):
kwds["oauth_token"] = args.oauth_token
if hasattr(args, "rebuild"):
kwds["rebuild"] = args.rebuild
if hasattr(args, "hash"):
kwds["hash_func"] = args.hash
if hasattr(args, "singularity_image_dir") and args.singularity_image_dir:
kwds["singularity_image_dir"] = args.singularity_image_dir
kwds["involucro_context"] = context_from_args(args)
return kwds
[docs]def main(argv=None):
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
add_build_arguments(parser)
add_single_image_arguments(parser)
parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
parser.add_argument('targets', metavar="TARGETS", default=None, help="Build a single container with specific package(s).")
parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of mulled container (leave blank to auto-generate based on packages - recommended).")
parser.add_argument('--test', help='Provide a test command for the container.')
parser.add_argument('--test-files', help='Provide test-files that may be required to run the test command. Individual mounts are separated by comma.'
'The source:dest docker syntax is respected. If relative file paths are given, files will be mounted in /source/<relative_file_path>')
args = parser.parse_args()
targets = target_str_to_targets(args.targets)
sys.exit(mull_targets(targets, **args_to_mull_targets_kwds(args)))
__all__ = ("main", )
if __name__ == '__main__':
main()