Source code for galaxy.tool_util.linters.tests

"""This module contains a linting functions for tool tests."""
import typing
from inspect import (
    Parameter,
    signature,
)

from galaxy.util import asbool
from ._util import is_datasource
from ..verify import asserts


[docs]def check_compare_attribs(element, lint_ctx, test_idx): COMPARE_COMPATIBILITY = { "sort": ["diff", "re_match", "re_match_multiline"], "lines_diff": ["diff", "re_match", "contains"], "decompress": ["diff"], "delta": ["sim_size"], "delta_frac": ["sim_size"], } compare = element.get("compare", "diff") for attrib in COMPARE_COMPATIBILITY: if attrib in element.attrib and compare not in COMPARE_COMPATIBILITY[attrib]: lint_ctx.error( f'Test {test_idx}: Attribute {attrib} is incompatible with compare="{compare}".', node=element )
[docs]def lint_tests(tool_xml, lint_ctx): # determine node to report for general problems with tests tests = tool_xml.findall("./tests/test") general_node = tool_xml.find("./tests") if general_node is None: general_node = tool_xml.getroot() datasource = is_datasource(tool_xml) if not tests: if not datasource: lint_ctx.warn("No tests found, most tools should define test cases.", node=general_node) elif datasource: lint_ctx.info("No tests found, that should be OK for data_sources.", node=general_node) return num_valid_tests = 0 for test_idx, test in enumerate(tests, start=1): has_test = False test_expect = ("expect_failure", "expect_exit_code", "expect_num_outputs") for te in test_expect: if te in test.attrib: has_test = True break test_assert = ("assert_stdout", "assert_stderr", "assert_command") for ta in test_assert: assertions = test.findall(ta) if len(assertions) == 0: continue if len(assertions) > 1: lint_ctx.error(f"Test {test_idx}: More than one {ta} found. Only the first is considered.", node=test) has_test = True _check_asserts(test_idx, assertions, lint_ctx) _check_asserts(test_idx, test.findall(".//assert_contents"), lint_ctx) # check if expect_num_outputs is set if there are outputs with filters # (except for tests with expect_failure .. which can't have test outputs) filter = tool_xml.findall("./outputs//filter") if len(filter) > 0 and "expect_num_outputs" not in test.attrib: if not asbool(test.attrib.get("expect_failure", False)): lint_ctx.warn("Test should specify 'expect_num_outputs' if outputs have filters", node=test) # really simple test that test parameters are also present in the inputs for param in test.findall("param"): name = param.attrib.get("name", None) if not name: lint_ctx.error(f"Test {test_idx}: Found test param tag without a name defined.", node=param) continue name = name.split("|")[-1] xpaths = [f"@name='{name}'", f"@argument='{name}'", f"@argument='-{name}'", f"@argument='--{name}'"] if "_" in name: xpaths += [f"@argument='-{name.replace('_', '-')}'", f"@argument='--{name.replace('_', '-')}'"] found = False for xp in xpaths: inxpath = f".//inputs//param[{xp}]" inparam = tool_xml.findall(inxpath) if len(inparam) > 0: found = True break if not found: lint_ctx.error(f"Test {test_idx}: Test param {name} not found in the inputs", node=param) output_data_or_collection = _collect_output_names(tool_xml) found_output_test = False for output in test.findall("output") + test.findall("output_collection"): found_output_test = True name = output.attrib.get("name", None) if not name: lint_ctx.error(f"Test {test_idx}: Found {output.tag} tag without a name defined.", node=output) continue if name not in output_data_or_collection: lint_ctx.error( f"Test {test_idx}: Found {output.tag} tag with unknown name [{name}], valid names {list(output_data_or_collection)}", node=output, ) continue if output.tag == "output": check_compare_attribs(output, lint_ctx, test_idx) elements = output.findall("./element") if elements: for element in elements: check_compare_attribs(element, lint_ctx, test_idx) # check that # - test/output corresponds to outputs/data and # - test/collection to outputs/output_collection corresponding_output = output_data_or_collection[name] if output.tag == "output" and corresponding_output.tag != "data": lint_ctx.error( f"Test {test_idx}: test output {name} does not correspond to a 'data' output, but a '{corresponding_output.tag}'", node=output, ) elif output.tag == "output_collection" and corresponding_output.tag != "collection": lint_ctx.error( f"Test {test_idx}: test collection output '{name}' does not correspond to a 'output_collection' output, but a '{corresponding_output.tag}'", node=output, ) # check that discovered data is tested sufficiently discover_datasets = corresponding_output.find(".//discover_datasets") if discover_datasets is not None: if output.tag == "output": if "count" not in output.attrib and output.find("./discovered_dataset") is None: lint_ctx.error( f"Test {test_idx}: test output '{name}' must have a 'count' attribute and/or 'discovered_dataset' children", node=output, ) else: if "count" not in output.attrib and len(elements) == 0: lint_ctx.error( f"Test {test_idx}: test collection '{name}' must have a 'count' attribute or 'element' children", node=output, ) if corresponding_output.get("type", "") in ["list:list", "list:paired"]: nested_elements = output.find("./element/element") element_with_count = output.find("./element[@count]") if nested_elements is None and element_with_count is None: lint_ctx.error( f"Test {test_idx}: test collection '{name}' must contain nested 'element' tags and/or element childen with a 'count' attribute", node=output, ) if asbool(test.attrib.get("expect_failure", False)): if found_output_test: lint_ctx.error(f"Test {test_idx}: Cannot specify outputs in a test expecting failure.", node=test) continue if "expect_num_outputs" in test.attrib: lint_ctx.error( f"Test {test_idx}: Cannot make assumptions on the number of outputs in a test expecting failure.", node=test, ) continue has_test = has_test or found_output_test if not has_test: lint_ctx.warn( f"Test {test_idx}: No outputs or expectations defined for tests, this test is likely invalid.", node=test, ) else: num_valid_tests += 1 if num_valid_tests or datasource: lint_ctx.valid(f"{num_valid_tests} test(s) found.", node=general_node) else: lint_ctx.warn("No valid test(s) found.", node=general_node)
def _check_asserts(test_idx, assertions, lint_ctx): """ assertions is a list of assert_contents, assert_stdout, assert_stderr, assert_command in practice only for the first case the list may be longer than one """ for assertion in assertions: for i, a in enumerate(assertion.iter()): if i == 0: # skip root note itself continue assert_function_name = "assert_" + a.tag if assert_function_name not in asserts.assertion_functions: lint_ctx.error(f"Test {test_idx}: unknown assertion '{a.tag}'", node=a) continue assert_function_sig = signature(asserts.assertion_functions[assert_function_name]) # check of the attributes for attrib in a.attrib: if attrib not in assert_function_sig.parameters: lint_ctx.error(f"Test {test_idx}: unknown attribute '{attrib}' for '{a.tag}'", node=a) continue # check missing required attributes for p in assert_function_sig.parameters: if p in ["output", "output_bytes", "verify_assertions_function", "children"]: continue if assert_function_sig.parameters[p].default is Parameter.empty and p not in a.attrib: lint_ctx.error(f"Test {test_idx}: missing attribute '{p}' for '{a.tag}'", node=a) # has_n_lines, has_n_columns, and has_size need to specify n/value, min, or max if a.tag in ["has_n_lines", "has_n_columns"]: if "n" not in a.attrib and "min" not in a.attrib and "max" not in a.attrib: lint_ctx.error(f"Test {test_idx}: '{a.tag}' needs to specify 'n', 'min', or 'max'", node=a) if a.tag == "has_size": if "value" not in a.attrib and "min" not in a.attrib and "max" not in a.attrib: lint_ctx.error(f"Test {test_idx}: '{a.tag}' needs to specify 'value', 'min', or 'max'", node=a) def _handle_optionals(annotation): as_dict = annotation.__dict__ if "__origin__" in as_dict and as_dict["__origin__"] == typing.Union: return as_dict["__args__"][0] return annotation def _collect_output_names(tool_xml): """ determine dict mapping the names of data and collection outputs to the corresponding nodes """ output_data_or_collection = {} outputs = tool_xml.findall("./outputs") if len(outputs) == 1: for output in list(outputs[0]): name = output.attrib.get("name", None) if not name: continue output_data_or_collection[name] = output return output_data_or_collection