Warning

This document is for an in-development version of Galaxy. You can alternatively view this page in the latest release if it exists or view the top of the latest release's documentation.

Source code for galaxy.webapps.galaxy.api.datasets

"""
API operations on the contents of a history dataset.
"""
import logging
import os

from six import string_types

from galaxy import (
    exceptions as galaxy_exceptions,
    managers,
    model,
    util,
    web
)
from galaxy.datatypes import dataproviders
from galaxy.util.path import (
    safe_walk
)
from galaxy.visualization.data_providers.genome import (
    BamDataProvider,
    FeatureLocationIndexDataProvider,
    SamDataProvider
)
from galaxy.web.base.controller import (
    BaseAPIController,
    UsesVisualizationMixin
)
from galaxy.web.framework.helpers import is_true

log = logging.getLogger(__name__)


[docs]class DatasetsController(BaseAPIController, UsesVisualizationMixin):
[docs] def __init__(self, app): super(DatasetsController, self).__init__(app) self.hda_manager = managers.hdas.HDAManager(app) self.hda_serializer = managers.hdas.HDASerializer(self.app) self.ldda_manager = managers.lddas.LDDAManager(app)
def _parse_serialization_params(self, kwd, default_view): view = kwd.get('view', None) keys = kwd.get('keys') if isinstance(keys, string_types): keys = keys.split(',') return dict(view=view, keys=keys, default_view=default_view)
[docs] @web.expose_api def index(self, trans, **kwd): """ GET /api/datasets Lists datasets. """ trans.response.status = 501 return 'not implemented'
[docs] @web.expose_api_anonymous def show(self, trans, id, hda_ldda='hda', data_type=None, provider=None, **kwd): """ GET /api/datasets/{encoded_dataset_id} Displays information about and/or content of a dataset. """ # Get dataset. try: dataset = self.get_hda_or_ldda(trans, hda_ldda=hda_ldda, dataset_id=id) except Exception as e: return str(e) # Use data type to return particular type of data. try: if data_type == 'state': rval = self._dataset_state(trans, dataset) elif data_type == 'converted_datasets_state': rval = self._converted_datasets_state(trans, dataset, kwd.get('chrom', None), is_true(kwd.get('retry', False))) elif data_type == 'data': rval = self._data(trans, dataset, **kwd) elif data_type == 'features': rval = self._search_features(trans, dataset, kwd.get('query')) elif data_type == 'raw_data': rval = self._raw_data(trans, dataset, provider, **kwd) elif data_type == 'track_config': rval = self.get_new_track_config(trans, dataset) elif data_type == 'genome_data': rval = self._get_genome_data(trans, dataset, kwd.get('dbkey', None)) else: # Default: return dataset as dict. if hda_ldda == 'hda': return self.hda_serializer.serialize_to_view(dataset, view=kwd.get('view', 'detailed'), user=trans.user, trans=trans) else: rval = dataset.to_dict() except Exception as e: rval = "Error in dataset API at listing contents: " + str(e) log.error(rval + ": %s" % str(e), exc_info=True) trans.response.status = 500 return rval
[docs] @web._future_expose_api def update_permissions(self, trans, dataset_id, payload, **kwd): """ PUT /api/datasets/{encoded_dataset_id}/permissions Updates permissions of a dataset. :rtype: dict :returns: dictionary containing new permissions """ if payload: kwd.update(payload) hda_ldda = kwd.get('hda_ldda', 'hda') dataset_assoc = self.get_hda_or_ldda(trans, hda_ldda=hda_ldda, dataset_id=dataset_id) if hda_ldda == "hda": self.hda_manager.update_permissions(trans, dataset_assoc, **kwd) return self.hda_manager.serialize_dataset_association_roles(trans, dataset_assoc) else: self.ldda_manager.update_permissions(trans, dataset_assoc, **kwd) return self.ldda_manager.serialize_dataset_association_roles(trans, dataset_assoc)
def _dataset_state(self, trans, dataset, **kwargs): """ Returns state of dataset. """ msg = self.hda_manager.data_conversion_status(dataset) if not msg: msg = dataset.conversion_messages.DATA return msg def _converted_datasets_state(self, trans, dataset, chrom=None, retry=False): """ Init-like method that returns state of dataset's converted datasets. Returns valid chroms for that dataset as well. """ msg = self.hda_manager.data_conversion_status(dataset) if msg: return msg # Get datasources and check for messages (which indicate errors). Retry if flag is set. data_sources = dataset.get_datasources(trans) messages_list = [data_source_dict['message'] for data_source_dict in data_sources.values()] msg = self._get_highest_priority_msg(messages_list) if msg: if retry: # Clear datasources and then try again. dataset.clear_associated_files() return self._converted_datasets_state(trans, dataset, chrom) else: return msg # If there is a chrom, check for data on the chrom. if chrom: data_provider = trans.app.data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='index') if not data_provider.has_data(chrom): return dataset.conversion_messages.NO_DATA # Have data if we get here return {"status": dataset.conversion_messages.DATA, "valid_chroms": None} def _search_features(self, trans, dataset, query): """ Returns features, locations in dataset that match query. Format is a list of features; each feature is a list itself: [name, location] """ if dataset.can_convert_to("fli"): converted_dataset = dataset.get_converted_dataset(trans, "fli") if converted_dataset: data_provider = FeatureLocationIndexDataProvider(converted_dataset=converted_dataset) if data_provider: return data_provider.get_data(query) return [] def _data(self, trans, dataset, chrom, low, high, start_val=0, max_vals=None, **kwargs): """ Provides a block of data from a dataset. """ # Parameter check. if not chrom: return dataset.conversion_messages.NO_DATA # Dataset check. msg = self.hda_manager.data_conversion_status(dataset) if msg: return msg # Get datasources and check for messages. data_sources = dataset.get_datasources(trans) messages_list = [data_source_dict['message'] for data_source_dict in data_sources.values()] return_message = self._get_highest_priority_msg(messages_list) if return_message: return return_message extra_info = None mode = kwargs.get("mode", "Auto") data_provider_registry = trans.app.data_provider_registry indexer = None # Coverage mode uses index data. if mode == "Coverage": # Get summary using minimal cutoffs. indexer = data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='index') return indexer.get_data(chrom, low, high, **kwargs) # TODO: # (1) add logic back in for no_detail # (2) handle scenario where mode is Squish/Pack but data requested is large, so reduced data needed to be returned. # If mode is Auto, need to determine what type of data to return. if mode == "Auto": # Get stats from indexer. indexer = data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='index') stats = indexer.get_data(chrom, low, high, stats=True) # If stats were requested, return them. if 'stats' in kwargs: if stats['data']['max'] == 0: return {'dataset_type': indexer.dataset_type, 'data': None} else: return stats # Stats provides features/base and resolution is bases/pixel, so # multiplying them yields features/pixel. features_per_pixel = stats['data']['max'] * float(kwargs['resolution']) # Use heuristic based on features/pixel and region size to determine whether to # return coverage data. When zoomed out and region is large, features/pixel # is determining factor. However, when sufficiently zoomed in and region is # small, coverage data is no longer provided. if int(high) - int(low) > 50000 and features_per_pixel > 1000: return indexer.get_data(chrom, low, high) # # Provide individual data points. # # Get data provider. data_provider = data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='data') # Allow max_vals top be data provider set if not passed if max_vals is None: max_vals = data_provider.get_default_max_vals() # Get reference sequence and mean depth for region; these is used by providers for aligned reads. region = None mean_depth = None if isinstance(data_provider, (SamDataProvider, BamDataProvider)): # Get reference sequence. if dataset.dbkey: # FIXME: increase region 1M each way to provide sequence for # spliced/gapped reads. Probably should provide refseq object # directly to data provider. region = self.app.genomes.reference(trans, dbkey=dataset.dbkey, chrom=chrom, low=(max(0, int(low) - 1000000)), high=(int(high) + 1000000)) # Get mean depth. if not indexer: indexer = data_provider_registry.get_data_provider(trans, original_dataset=dataset, source='index') stats = indexer.get_data(chrom, low, high, stats=True) mean_depth = stats['data']['mean'] # Get and return data from data_provider. result = data_provider.get_data(chrom, int(low), int(high), int(start_val), int(max_vals), ref_seq=region, mean_depth=mean_depth, **kwargs) result.update({'dataset_type': data_provider.dataset_type, 'extra_info': extra_info}) return result def _raw_data(self, trans, dataset, provider=None, **kwargs): """ Uses original (raw) dataset to return data. This method is useful when the dataset is not yet indexed and hence using data would be slow because indexes need to be created. """ # Dataset check. msg = self.hda_manager.data_conversion_status(dataset) if msg: return msg registry = trans.app.data_provider_registry # allow the caller to specify which provider is used # pulling from the original providers if possible, then the new providers if provider: if provider in registry.dataset_type_name_to_data_provider: data_provider = registry.dataset_type_name_to_data_provider[provider](dataset) elif dataset.datatype.has_dataprovider(provider): kwargs = dataset.datatype.dataproviders[provider].parse_query_string_settings(kwargs) # use dictionary to allow more than the data itself to be returned (data totals, other meta, etc.) return { 'data': list(dataset.datatype.dataprovider(dataset, provider, **kwargs)) } else: raise dataproviders.exceptions.NoProviderAvailable(dataset.datatype, provider) # no provider name: look up by datatype else: data_provider = registry.get_data_provider(trans, raw=True, original_dataset=dataset) # Return data. data = data_provider.get_data(**kwargs) return data
[docs] @web.expose_api_anonymous def extra_files(self, trans, history_content_id, history_id, **kwd): """ GET /api/histories/{encoded_history_id}/contents/{encoded_content_id}/extra_files Generate list of extra files. """ decoded_content_id = self.decode_id(history_content_id) hda = self.hda_manager.get_accessible(decoded_content_id, trans.user) extra_files_path = hda.extra_files_path rval = [] for root, directories, files in safe_walk(extra_files_path): for directory in directories: rval.append({"class": "Directory", "path": os.path.relpath(os.path.join(root, directory), extra_files_path)}) for file in files: rval.append({"class": "File", "path": os.path.relpath(os.path.join(root, file), extra_files_path)}) return rval
[docs] @web.expose_api_raw_anonymous def display(self, trans, history_content_id, history_id, preview=False, filename=None, to_ext=None, raw=False, **kwd): """ GET /api/histories/{encoded_history_id}/contents/{encoded_content_id}/display Displays history content (dataset). The query parameter 'raw' should be considered experimental and may be dropped at some point in the future without warning. Generally, data should be processed by its datatype prior to display (the defult if raw is unspecified or explicitly false. """ decoded_content_id = self.decode_id(history_content_id) raw = util.string_as_bool_or_none(raw) rval = '' try: hda = self.hda_manager.get_accessible(decoded_content_id, trans.user) if raw: if filename and filename != 'index': file_path = trans.app.object_store.get_filename(hda.dataset, extra_dir=('dataset_%s_files' % hda.dataset.id), alt_name=filename) else: file_path = hda.file_name rval = open(file_path, 'rb') else: display_kwd = kwd.copy() if 'key' in display_kwd: del display_kwd["key"] rval = hda.datatype.display_data(trans, hda, preview, filename, to_ext, **display_kwd) except Exception as exception: log.error("Error getting display data for dataset (%s) from history (%s): %s", history_content_id, history_id, str(exception), exc_info=True) trans.response.status = 500 rval = ("Could not get display data for dataset: " + str(exception)) return rval
[docs] @web.expose_api_raw_anonymous def get_metadata_file(self, trans, history_content_id, history_id, metadata_file=None, **kwd): decoded_content_id = self.decode_id(history_content_id) rval = '' try: hda = self.hda_manager.get_accessible(decoded_content_id, trans.user) file_ext = hda.metadata.spec.get(metadata_file).get("file_ext", metadata_file) fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in hda.name)[0:150] trans.response.headers["Content-Type"] = "application/octet-stream" trans.response.headers["Content-Disposition"] = 'attachment; filename="Galaxy%s-[%s].%s"' % (hda.hid, fname, file_ext) return open(hda.metadata.get(metadata_file).file_name, 'rb') except Exception as exception: log.error("Error getting metadata_file (%s) for dataset (%s) from history (%s): %s", metadata_file, history_content_id, history_id, str(exception), exc_info=True) trans.response.status = 500 rval = ("Could not get display data for dataset: " + str(exception)) return rval
[docs] @web._future_expose_api_anonymous def converted(self, trans, dataset_id, ext, **kwargs): """ converted( self, trans, dataset_id, ext, **kwargs ) * GET /api/datasets/{dataset_id}/converted/{ext} return information about datasets made by converting this dataset to a new format :type dataset_id: str :param dataset_id: the encoded id of the original HDA to check :type ext: str :param ext: file extension of the target format or None. If there is no existing converted dataset for the format in `ext`, one will be created. If `ext` is None, a dictionary will be returned of the form { <converted extension> : <converted id>, ... } containing all the *existing* converted datasets. ..note: `view` and `keys` are also available to control the serialization of individual datasets. They have no effect when `ext` is None. :rtype: dict :returns: dictionary containing detailed HDA information or (if `ext` is None) an extension->dataset_id map """ decoded_id = self.decode_id(dataset_id) hda = self.hda_manager.get_accessible(decoded_id, trans.user) if ext: converted = self._get_or_create_converted(trans, hda, ext, **kwargs) return self.hda_serializer.serialize_to_view(converted, user=trans.user, trans=trans, **self._parse_serialization_params(kwargs, 'detailed')) return self.hda_serializer.serialize_converted_datasets(hda, 'converted')
def _get_or_create_converted(self, trans, original, target_ext, **kwargs): try: original.get_converted_dataset(trans, target_ext) converted = original.get_converted_files_by_type(target_ext) return converted except model.NoConverterException: exc_data = dict(source=original.ext, target=target_ext, available=list(original.get_converter_types().keys())) raise galaxy_exceptions.RequestParameterInvalidException('Conversion not possible', **exc_data)