diff --git a/cwltool/builder.py b/cwltool/builder.py index a417eeedb..d66b98888 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -6,7 +6,7 @@ from typing import Any, Union, AnyStr, Callable from .errors import WorkflowException from .stdfsaccess import StdFsAccess -from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs +from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs, normalizeFilesDirs CONTENT_LIMIT = 64 * 1024 @@ -31,7 +31,8 @@ def __init__(self): # type: () -> None self.resources = None # type: Dict[str, Union[int, str]] self.bindings = [] # type: List[Dict[str, Any]] self.timeout = None # type: int - self.pathmapper = None # type: PathMapper + self.pathmapper = None # type: PathMapper + self.stagedir = None # type: unicode def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]): # type: (Dict[unicode, Any], Any, List[int], List[int]) -> List[Dict[str, Any]] @@ -112,7 +113,8 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]): if isinstance(sf, dict) or "$(" in sf or "${" in sf: secondary_eval = self.do_eval(sf, context=datum) if isinstance(secondary_eval, basestring): - sfpath = {"location": secondary_eval, "class": "File"} + sfpath = {"location": secondary_eval, + "class": "File"} else: sfpath = secondary_eval else: @@ -121,6 +123,7 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]): datum["secondaryFiles"].extend(sfpath) else: datum["secondaryFiles"].append(sfpath) + normalizeFilesDirs(datum["secondaryFiles"]) def _capture_files(f): self.files.append(f) @@ -187,8 +190,14 @@ def generate_arg(self, binding): # type: (Dict[str,Any]) -> List[str] return [a for a in args if a is not None] - def do_eval(self, ex, context=None, pull_image=True): - # type: (Dict[str,str], Any, bool) -> Any + def do_eval(self, ex, context=None, pull_image=True, recursive=False): + # type: (Union[Dict[str, str], unicode], Any, bool, bool) -> Any + if recursive: + if isinstance(ex, dict): + return {k: self.do_eval(v, context, pull_image, recursive) for k,v in ex.iteritems()} + if isinstance(ex, list): + return [self.do_eval(v, context, pull_image, recursive) for v in ex] + return expression.do_eval(ex, self.job, self.requirements, self.outdir, self.tmpdir, self.resources, diff --git a/cwltool/cwltest.py b/cwltool/cwltest.py index d3a575eab..18d850773 100755 --- a/cwltool/cwltest.py +++ b/cwltool/cwltest.py @@ -58,9 +58,9 @@ def compare(a, b): # type: (Any, Any) -> bool raise CompareFail(u"%s not in %s" % (json.dumps(i, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True))) a = {k: v for k, v in a.iteritems() - if k not in ("path", "location", "listing")} + if k not in ("path", "location", "listing", "basename")} b = {k: v for k, v in b.iteritems() - if k not in ("path", "location", "listing")} + if k not in ("path", "location", "listing", "basename")} if len(a) != len(b): raise CompareFail(u"expected %s\ngot %s" % (json.dumps(a, indent=4, sort_keys=True), json.dumps(b, indent=4, sort_keys=True))) diff --git a/cwltool/draft2tool.py b/cwltool/draft2tool.py index f0029ce92..e5a8658ec 100644 --- a/cwltool/draft2tool.py +++ b/cwltool/draft2tool.py @@ -6,7 +6,6 @@ import glob import logging import hashlib -import random import re import urlparse import tempfile @@ -17,7 +16,7 @@ import shellescape from typing import Callable, Any, Union, Generator, cast -from .process import Process, shortname, uniquename, getListing +from .process import Process, shortname, uniquename, getListing, normalizeFilesDirs from .errors import WorkflowException from .utils import aslist from . import expression @@ -50,7 +49,9 @@ def __init__(self): # type: () -> None def run(self, **kwargs): # type: (**Any) -> None try: - self.output_callback(self.builder.do_eval(self.script), "success") + ev = self.builder.do_eval(self.script) + normalizeFilesDirs(ev) + self.output_callback(ev, "success") except Exception as e: _logger.warn(u"Failed to evaluate expression:\n%s", e, exc_info=(e if kwargs.get('debug') else False)) self.output_callback({}, "permanentFail") @@ -110,6 +111,17 @@ def run(self, **kwargs): self.cachebuilder, self.outdir), "success") +# map files to assigned path inside a container. We need to also explicitly +# walk over input as implicit reassignment doesn't reach everything in builder.bindings +def check_adjust(builder, f): + # type: (Builder, Dict[str, Any]) -> Dict[str,Any] + f["path"] = builder.pathmapper.mapper(f["location"])[1] + f["dirname"], f["basename"] = os.path.split(f["path"]) + if f["class"] == "File": + f["nameroot"], f["nameext"] = os.path.splitext(f["basename"]) + if not ACCEPTLIST_RE.match(f["basename"]): + raise WorkflowException("Invalid filename: '%s' contains illegal characters" % (f["basename"])) + return f class CommandLineTool(Process): def __init__(self, toolpath_object, **kwargs): @@ -120,7 +132,7 @@ def makeJobRunner(self): # type: () -> CommandLineJob return CommandLineJob() def makePathMapper(self, reffiles, stagedir, **kwargs): - # type: (Set[Any], unicode, **Any) -> PathMapper + # type: (List[Any], unicode, **Any) -> PathMapper dockerReq, _ = self.get_requirement("DockerRequirement") try: return PathMapper(reffiles, kwargs["basedir"], stagedir) @@ -141,8 +153,13 @@ def job(self, joborder, output_callback, **kwargs): cachebuilder = self._init_job(joborder, **cacheargs) cachebuilder.pathmapper = PathMapper(cachebuilder.files, kwargs["basedir"], - cachebuilder.stagedir) - + cachebuilder.stagedir, + separateDirs=False) + _check_adjust = partial(check_adjust, cachebuilder) + adjustFileObjs(cachebuilder.files, _check_adjust) + adjustFileObjs(cachebuilder.bindings, _check_adjust) + adjustDirObjs(cachebuilder.files, _check_adjust) + adjustDirObjs(cachebuilder.bindings, _check_adjust) cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings)) (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") if docker_req and kwargs.get("use_container") is not False: @@ -151,8 +168,9 @@ def job(self, joborder, output_callback, **kwargs): keydict = {u"cmdline": cmdline} for _,f in cachebuilder.pathmapper.items(): - st = os.stat(f[0]) - keydict[f[0]] = [st.st_size, int(st.st_mtime * 1000)] + if f.type == "File": + st = os.stat(f.resolved) + keydict[f.resolved] = [st.st_size, int(st.st_mtime * 1000)] interesting = {"DockerRequirement", "EnvVarRequirement", @@ -236,19 +254,10 @@ def rm_pending_output_callback(output_callback, jobcachepending, builder.pathmapper = self.makePathMapper(reffiles, builder.stagedir, **kwargs) builder.requirements = j.requirements - # map files to assigned path inside a container. We need to also explicitly - # walk over input as implicit reassignment doesn't reach everything in builder.bindings - def _check_adjust(f): # type: (Dict[str,Any]) -> Dict[str,Any] - f["path"] = builder.pathmapper.mapper(f["location"])[1] - f["dirname"], f["basename"] = os.path.split(f["path"]) - if f["class"] == "File": - f["nameroot"], f["nameext"] = os.path.splitext(f["basename"]) - if not ACCEPTLIST_RE.match(f["basename"]): - raise WorkflowException("Invalid filename: '%s' contains illegal characters" % (f["basename"])) - return f - _logger.debug(u"[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) + _check_adjust = partial(check_adjust, builder) + adjustFileObjs(builder.files, _check_adjust) adjustFileObjs(builder.bindings, _check_adjust) adjustDirObjs(builder.files, _check_adjust) @@ -260,27 +269,51 @@ def _check_adjust(f): # type: (Dict[str,Any]) -> Dict[str,Any] _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4)) - dockerReq, _ = self.get_requirement("DockerRequirement") + dockerReq = self.get_requirement("DockerRequirement")[0] if dockerReq and kwargs.get("use_container"): out_prefix = kwargs.get("tmp_outdir_prefix") j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix) tmpdir_prefix = kwargs.get('tmpdir_prefix') j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix) - j.stagedir = None + j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix) else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir j.stagedir = builder.stagedir initialWorkdir = self.get_requirement("InitialWorkDirRequirement")[0] - j.generatefiles = {"class": "Directory", "listing": []} + j.generatefiles = {"class": "Directory", "listing": [], "basename": ""} if initialWorkdir: + ls = [] # type: List[Dict[str, Any]] if isinstance(initialWorkdir["listing"], (str, unicode)): - j.generatefiles["listing"] = builder.do_eval(initialWorkdir["listing"]) + ls = builder.do_eval(initialWorkdir["listing"]) else: for t in initialWorkdir["listing"]: - j.generatefiles["listing"].append({"entryname": builder.do_eval(t["entryname"]), - "entry": copy.deepcopy(builder.do_eval(t["entry"]))}) + if "entry" in t: + et = {"entry": builder.do_eval(t["entry"])} + if "entryname" in t: + et["entryname"] = builder.do_eval(t["entryname"]) + else: + et["entryname"] = None + ls.append(et) + else: + ls.append(t) + for i,t in enumerate(ls): + if "entry" in t: + if isinstance(t["entry"], (str, unicode)): + ls[i] = { + "class": "File", + "basename": t["entryname"], + "contents": t["entry"] + } + else: + if t["entryname"]: + t = copy.deepcopy(t) + t["entry"]["basename"] = t["entryname"] + ls[i] = t["entry"] + j.generatefiles[u"listing"] = ls + + normalizeFilesDirs(j.generatefiles) j.environment = {} evr = self.get_requirement("EnvVarRequirement")[0] @@ -321,6 +354,8 @@ def collect_output_ports(self, ports, builder, outdir): # https://github.com/python/mypy/issues/797 partial(revmap_file, builder, outdir))) adjustFileObjs(ret, remove_path) + adjustDirObjs(ret, remove_path) + normalizeFilesDirs(ret) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret) return ret @@ -334,6 +369,7 @@ def collect_output_ports(self, ports, builder, outdir): if ret: adjustFileObjs(ret, remove_path) adjustDirObjs(ret, remove_path) + normalizeFilesDirs(ret) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret) return ret if ret is not None else {} except validate.ValidationException as e: diff --git a/cwltool/job.py b/cwltool/job.py index 539c2f37e..1a0b51548 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -58,11 +58,12 @@ def __init__(self): # type: () -> None self.outdir = None # type: str self.tmpdir = None # type: str self.environment = None # type: Dict[str,str] - self.generatefiles = None # type: Dict[str,Union[Dict[str,str],str]] + self.generatefiles = None # type: Dict[unicode, Union[List[Dict[str, str]], Dict[str,str], str]] + self.stagedir = None # type: unicode def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs="move", **kwargs): - # type: (bool, bool, bool, bool, bool, **Any) -> Union[Tuple[str,Dict[None,None]],None] + # type: (bool, bool, bool, bool, bool, unicode, **Any) -> Union[Tuple[str,Dict[None,None]],None] if not os.path.exists(self.outdir): os.makedirs(self.outdir) @@ -78,10 +79,12 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, (docker_req, docker_is_req) = get_feature(self, "DockerRequirement") - for f in self.pathmapper.files(): - p = self.pathmapper.mapper(f) + for knownfile in self.pathmapper.files(): + p = self.pathmapper.mapper(knownfile) if p.type == "File" and not os.path.isfile(p[0]): - raise WorkflowException(u"Input file %s (at %s) not found or is not a regular file." % (f, self.pathmapper.mapper(f)[0])) + raise WorkflowException( + u"Input file %s (at %s) not found or is not a regular file." + % (knownfile, self.pathmapper.mapper(knownfile)[0])) img_id = None if docker_req and kwargs.get("use_container") is not False: @@ -97,6 +100,11 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, vol = self.pathmapper.mapper(src) if vol.type == "File": runtime.append(u"--volume=%s:%s:ro" % (vol.resolved, vol.target)) + if vol.type == "CreateFile": + createtmp = os.path.join(self.stagedir, os.path.basename(vol.target)) + with open(createtmp, "w") as f: + f.write(vol.resolved.encode("utf-8")) + runtime.append(u"--volume=%s:%s:ro" % (createtmp, vol.target)) runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.outdir), "/var/spool/cwl")) runtime.append(u"--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp")) runtime.append(u"--workdir=%s" % ("/var/spool/cwl")) diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index f0f757dec..6a100b7ad 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -8,6 +8,7 @@ import urlparse from schema_salad.ref_resolver import Loader import schema_salad.validate as validate +from schema_salad.validate import ValidationException import schema_salad.schema as schema from avro.schema import Names from . import update @@ -37,13 +38,12 @@ def fetch_document(argsworkflow): workflowobj = argsworkflow uri = "#" + str(id(argsworkflow)) else: - raise validate.ValidationException( - "Must be URI or object: '%s'" % argsworkflow) + raise ValidationException("Must be URI or object: '%s'" % argsworkflow) return document_loader, workflowobj, uri def _convert_stdstreams_to_files(workflowobj): - # type: (Union[Dict[unicode, Any], List[Dict[unicode, Any]]) -> None + # type: (Union[Dict[unicode, Any], List[Dict[unicode, Any]]]) -> None if isinstance(workflowobj, dict): if ('class' in workflowobj @@ -53,7 +53,7 @@ def _convert_stdstreams_to_files(workflowobj): for streamtype in ['stdout', 'stderr']: if out['type'] == streamtype: if 'outputBinding' in out: - raise validate.ValidateException( + raise ValidationException( "Not allowed to specify outputBinding when" " using %s shortcut." % streamtype) if streamtype in workflowobj: @@ -109,12 +109,11 @@ def validate_document(document_loader, workflowobj, uri, workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (dict, list)): - raise validate.ValidationException("Workflow must be a dict or list.") + raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): - raise validate.ValidationException( - "Draft-2 workflows must be a dict.") + raise ValidationException("Draft-2 workflows must be a dict.") metadata = {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]} diff --git a/cwltool/main.py b/cwltool/main.py index 1bd3f14a7..4cb71f3eb 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -12,7 +12,6 @@ import urlparse import hashlib import pkg_resources # part of setuptools -import random import functools import rdflib @@ -26,7 +25,7 @@ from . import workflow from .errors import WorkflowException, UnsupportedRequirement from .cwlrdf import printrdf, printdot -from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps +from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps, normalizeFilesDirs from .load_tool import fetch_document, validate_document, make_tool from . import draft2tool from .builder import adjustFileObjs, adjustDirObjs @@ -155,6 +154,10 @@ def arg_parser(): # type: () -> argparse.ArgumentParser help="Will be passed to `docker run` as the '--net' " "parameter. Implies '--enable-net'.") + parser.add_argument("--on-error", type=str, + help="Desired workflow behavior when a step fails. One of 'stop' or 'continue'. " + "Default is 'stop.", default="stop") + parser.add_argument("workflow", type=str, nargs="?", default=None) parser.add_argument("job_order", nargs=argparse.REMAINDER) @@ -413,18 +416,20 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, return 1 if print_input_deps: - printdeps(job_order_object, loader, stdout, relative_deps, + printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=u"file://%s/" % input_basedir) return 0 def pathToLoc(p): - if "location" not in p: + if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] adjustDirObjs(job_order_object, pathToLoc) adjustFileObjs(job_order_object, pathToLoc) - adjustDirObjs(job_order_object, functools.partial(getListing, StdFsAccess(input_basedir))) + normalizeFilesDirs(job_order_object) + adjustDirObjs(job_order_object, cast(Callable[..., Any], + functools.partial(getListing, StdFsAccess(input_basedir)))) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] @@ -435,12 +440,12 @@ def pathToLoc(p): def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None): - # type: (Dict[unicode, Any], Loader, IO[Any], bool, str) -> None + # type: (Dict[unicode, Any], Loader, IO[Any], bool, unicode, str) -> None deps = {"class": "File", - "location": uri} + "location": uri} # type: Dict[unicode, Any] def loadref(b, u): - return document_loader.resolve_ref(u, base_url=b)[0] + return document_loader.fetch(urlparse.urljoin(b, u)) sf = scandeps(basedir if basedir else uri, obj, set(("$import", "run")), @@ -591,7 +596,8 @@ def main(argsl=None, 'tool_help': False, 'workflow': None, 'job_order': None, - 'pack': False}.iteritems(): + 'pack': False, + 'on_error': 'continue'}.iteritems(): if not hasattr(args, k): setattr(args, k, v) diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 8455d648a..3d707fce4 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -1,15 +1,18 @@ import os -import random import logging import stat import collections -from typing import Tuple, Set, Union, Any +import uuid +import urlparse +from functools import partial +from typing import Any, Callable, Set, Tuple, Union +import schema_salad.validate as validate _logger = logging.getLogger("cwltool") -MapperEnt = collections.namedtuple("MapperEnt", ("resolved", "target", "type")) +MapperEnt = collections.namedtuple("MapperEnt", ["resolved", "target", "type"]) -def adjustFiles(rec, op): # type: (Any, Callable[..., Any]) -> None +def adjustFiles(rec, op): # type: (Any, Union[Callable[..., Any], partial[Any]]) -> None """Apply a mapping function to each File path in the object `rec`.""" if isinstance(rec, dict): @@ -21,7 +24,7 @@ def adjustFiles(rec, op): # type: (Any, Callable[..., Any]) -> None for d in rec: adjustFiles(d, op) -def adjustFileObjs(rec, op): # type: (Any, Callable[[Any], Any]) -> None +def adjustFileObjs(rec, op): # type: (Any, Union[Callable[..., Any], partial[Any]]) -> None """Apply an update function to each File object in the object `rec`.""" if isinstance(rec, dict): @@ -33,18 +36,38 @@ def adjustFileObjs(rec, op): # type: (Any, Callable[[Any], Any]) -> None for d in rec: adjustFileObjs(d, op) -def adjustDirObjs(rec, op): # type: (Any, Callable[[Any], Any]) -> None +def adjustDirObjs(rec, op): + # type: (Any, Union[Callable[..., Any], partial[Any]]) -> None """Apply an update function to each Directory object in the object `rec`.""" if isinstance(rec, dict): if rec.get("class") == "Directory": op(rec) - for d in rec: - adjustDirObjs(rec[d], op) + for key in rec: + adjustDirObjs(rec[key], op) if isinstance(rec, list): for d in rec: adjustDirObjs(d, op) +def normalizeFilesDirs(job): + # type: (Union[List[Dict[unicode, Any]], Dict[unicode, Any]]) -> None + def addLocation(d): + if "location" not in d: + if d["class"] == "File" and ("contents" not in d): + raise validate.ValidationException("Anonymous file object must have 'contents' and 'basename' fields.") + if d["class"] == "Directory" and ("listing" not in d or "basename" not in d): + raise validate.ValidationException("Anonymous directory object must have 'listing' and 'basename' fields.") + d["location"] = "_:" + unicode(uuid.uuid4()) + if "basename" not in d: + d["basename"] = unicode(uuid.uuid4()) + + if "basename" not in d: + parse = urlparse.urlparse(d["location"]) + d["basename"] = os.path.basename(parse.path) + + adjustFileObjs(job, addLocation) + adjustDirObjs(job, addLocation) + def abspath(src, basedir): # type: (unicode, unicode) -> unicode if src.startswith(u"file://"): @@ -60,61 +83,50 @@ class PathMapper(object): (absolute local path, absolute container path)""" def __init__(self, referenced_files, basedir, stagedir, separateDirs=True): - # type: (Set[Any], unicode, unicode) -> None - self._pathmap = {} # type: Dict[unicode, Tuple[unicode, unicode]] + # type: (List[Any], unicode, unicode, bool) -> None + self._pathmap = {} # type: Dict[unicode, MapperEnt] self.stagedir = stagedir self.separateDirs = separateDirs self.setup(referenced_files, basedir) def visitlisting(self, listing, stagedir, basedir): + # type: (List[Dict[unicode, Any]], unicode, unicode) -> None for ld in listing: - if "entryname" in ld: - tgt = os.path.join(stagedir, ld["entryname"]) - if isinstance(ld["entry"], (str, unicode)): - self._pathmap[str(id(ld["entry"]))] = MapperEnt(ld["entry"], tgt, "CreateFile") - else: - if ld["entry"]["class"] == "Directory": - self.visit(ld["entry"], tgt, basedir, copy=ld.get("writable", False)) - else: - self.visit(ld["entry"], stagedir, basedir, entryname=ld["entryname"], copy=ld.get("writable", False)) - #ab = ld["entry"]["location"] - #if ab.startswith("file://"): - # ab = ab[7:] - #self._pathmap[ld["entry"]["location"]] = MapperEnt(ab, tgt, ld["entry"]["class"]) - elif ld.get("class") == "File": + tgt = os.path.join(stagedir, ld["basename"]) + if ld["class"] == "Directory": + self.visit(ld, tgt, basedir, copy=ld.get("writable", False)) + else: self.visit(ld, stagedir, basedir, copy=ld.get("writable", False)) - def visit(self, obj, stagedir, basedir, entryname=None, copy=False): + def visit(self, obj, stagedir, basedir, copy=False): + # type: (Dict[unicode, Any], unicode, unicode, bool) -> None if obj["class"] == "Directory": - if "location" in obj: - self._pathmap[obj["location"]] = MapperEnt(obj["location"], stagedir, "Directory") - else: - self._pathmap[str(id(obj))] = MapperEnt(str(id(obj)), stagedir, "Directory") + self._pathmap[obj["location"]] = MapperEnt(obj["location"], stagedir, "Directory") self.visitlisting(obj.get("listing", []), stagedir, basedir) elif obj["class"] == "File": path = obj["location"] if path in self._pathmap: return ab = abspath(path, basedir) - if entryname: - tgt = os.path.join(stagedir, entryname) + tgt = os.path.join(stagedir, obj["basename"]) + if "contents" in obj and obj["location"].startswith("_:"): + self._pathmap[obj["location"]] = MapperEnt(obj["contents"], tgt, "CreateFile") else: - tgt = os.path.join(stagedir, os.path.basename(path)) - if copy: - self._pathmap[path] = MapperEnt(ab, tgt, "WritableFile") - else: - self._pathmap[path] = MapperEnt(ab, tgt, "File") - self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir) + if copy: + self._pathmap[path] = MapperEnt(ab, tgt, "WritableFile") + else: + self._pathmap[path] = MapperEnt(ab, tgt, "File") + self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir) def setup(self, referenced_files, basedir): - # type: (Set[Any], unicode) -> None + # type: (List[Any], unicode) -> None # Go through each file and set the target to its own directory along # with any secondary files. stagedir = self.stagedir for fob in referenced_files: if self.separateDirs: - stagedir = os.path.join(self.stagedir, "stg%x" % random.randint(1, 1000000000)) + stagedir = os.path.join(self.stagedir, "stg%s" % uuid.uuid4()) self.visit(fob, stagedir, basedir) # Dereference symbolic links @@ -131,18 +143,18 @@ def setup(self, referenced_files, basedir): self._pathmap[path] = MapperEnt(deref, tgt, "File") - def mapper(self, src): # type: (unicode) -> Tuple[unicode, unicode] + def mapper(self, src): # type: (unicode) -> MapperEnt if u"#" in src: i = src.index(u"#") p = self._pathmap[src[:i]] - return (p.resolved, p.target + src[i:]) + return MapperEnt(p.resolved, p.target + src[i:], None) else: return self._pathmap[src] def files(self): # type: () -> List[unicode] return self._pathmap.keys() - def items(self): # type: () -> List[Tuple[unicode, Tuple[unicode, unicode]]] + def items(self): # type: () -> List[Tuple[unicode, MapperEnt]] return self._pathmap.items() def reversemap(self, target): # type: (unicode) -> Tuple[unicode, unicode] diff --git a/cwltool/process.py b/cwltool/process.py index ea708c36c..243d89b22 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -11,8 +11,8 @@ import pprint from collections import Iterable import errno -import random import shutil +import uuid import abc import schema_salad.validate as validate @@ -30,7 +30,7 @@ from .stdfsaccess import StdFsAccess from .builder import Builder, adjustFileObjs, adjustDirObjs from .errors import WorkflowException, UnsupportedRequirement -from .pathmapper import PathMapper, abspath +from .pathmapper import PathMapper, abspath, normalizeFilesDirs _logger = logging.getLogger("cwltool") @@ -80,7 +80,6 @@ SCHEMA_CACHE = {} # type: Dict[str, Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[unicode, Any], Loader]] SCHEMA_FILE = None # type: Dict[unicode, Any] SCHEMA_DIR = None # type: Dict[unicode, Any] -SCHEMA_DIRENT = None # type: Dict[unicode, Any] SCHEMA_ANY = None # type: Dict[unicode, Any] def get_schema(version): @@ -154,6 +153,7 @@ def adjustFilesWithSecondary(rec, op, primary=None): adjustFilesWithSecondary(d, op, primary) def getListing(fs_access, rec): + # type: (StdFsAccess, Dict[str, Any]) -> None if "listing" not in rec: listing = [] loc = rec["location"] @@ -162,14 +162,13 @@ def getListing(fs_access, rec): ent = {"class": "Directory", "location": ld} getListing(fs_access, ent) - listing.append({"entryname": os.path.basename(ld), - "entry": ent}) + listing.append(ent) else: - listing.append({"entryname": os.path.basename(ld), - "entry": {"class": "File", "location": ld}}) + listing.append({"class": "File", "location": ld}) rec["listing"] = listing def stageFiles(pm, stageFunc): + # type: (PathMapper, Callable[..., Any]) -> None for f, p in pm.items(): if not os.path.exists(os.path.dirname(p.target)): os.makedirs(os.path.dirname(p.target), 0755) @@ -182,6 +181,7 @@ def stageFiles(pm, stageFunc): n.write(p.resolved.encode("utf-8")) def collectFilesAndDirs(obj, out): + # type: (Union[Dict[unicode, Any], List[Dict[unicode, Any]]], List[Dict[unicode, Any]]) -> None if isinstance(obj, dict): if obj.get("class") in ("File", "Directory"): out.append(obj) @@ -193,6 +193,7 @@ def collectFilesAndDirs(obj, out): collectFilesAndDirs(l, out) def relocateOutputs(outputObj, outdir, output_dirs, action): + # type: (Union[Dict[unicode, Any], List[Dict[unicode, Any]]], unicode, Set[unicode], unicode) -> Union[Dict[unicode, Any], List[Dict[unicode, Any]]] if action not in ("move", "copy"): return outputObj @@ -206,7 +207,7 @@ def moveIt(src, dst): _logger.debug("Copying %s to %s", src, dst) shutil.copy(src, dst) - outfiles = [] + outfiles = [] # type: List[Dict[unicode, Any]] collectFilesAndDirs(outputObj, outfiles) pm = PathMapper(outfiles, "", outdir, separateDirs=False) stageFiles(pm, moveIt) @@ -220,7 +221,7 @@ def _check_adjust(f): return outputObj -def cleanIntermediate(output_dirs): +def cleanIntermediate(output_dirs): # type: (Set[unicode]) -> None for a in output_dirs: if os.path.exists(a) and empty_subtree(a): _logger.debug(u"Removing intermediate output directory %s", a) @@ -286,20 +287,23 @@ def fillInDefaults(inputs, job): def avroize_type(field_type, name_prefix=""): + # type: (Union[List[Dict[unicode, Any]], Dict[unicode, Any]], unicode) -> Any """ adds missing information to a type so that CWL types are valid in schema_salad. """ - if type(field_type) == list: - field_type_result = [] - for idx, field_type_item in enumerate(field_type): - field_type_result.append(avroize_type(field_type_item, name_prefix+"_"+str(idx))) - return field_type_result - elif type(field_type) == dict and "type" in field_type and field_type["type"] == "enum": - if "name" not in field_type: - field_type["name"] = name_prefix+"_type_enum" + if isinstance(field_type, list): + for f in field_type: + avroize_type(f, name_prefix) + elif isinstance(field_type, dict): + if field_type["type"] in ("enum", "record"): + if "name" not in field_type: + field_type["name"] = name_prefix+unicode(uuid.uuid4()) + if field_type["type"] == "record": + avroize_type(field_type["fields"], name_prefix) + if field_type["type"] == "array": + avroize_type(field_type["items"], name_prefix) return field_type - class Process(object): __metaclass__ = abc.ABCMeta @@ -308,8 +312,8 @@ def __init__(self, toolpath_object, **kwargs): self.metadata = kwargs.get("metadata", {}) # type: Dict[str,Any] self.names = None # type: avro.schema.Names + global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY # pylint: disable=global-statement if SCHEMA_FILE is None: - global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_DIRENT, SCHEMA_ANY # pylint: disable=global-statement get_schema("draft-4") SCHEMA_ANY = cast(Dict[unicode, Any], SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/salad#Any"]) @@ -317,10 +321,8 @@ def __init__(self, toolpath_object, **kwargs): SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/cwl#File"]) SCHEMA_DIR = cast(Dict[unicode, Any], SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/cwl#Directory"]) - SCHEMA_DIRENT = cast(Dict[unicode, Any], - SCHEMA_CACHE["draft-4"][3].idx["https://w3id.org/cwl/cwl#Dirent"]) - names = schema_salad.schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_DIRENT, SCHEMA_ANY], + names = schema_salad.schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY], schema_salad.ref_resolver.Loader({}))[0] if isinstance(names, avro.schema.SchemaParseException): raise names @@ -368,7 +370,7 @@ def __init__(self, toolpath_object, **kwargs): c["type"] = ["null"] + aslist(c["type"]) else: c["type"] = c["type"] - c["type"] = avroize_type(c["type"],c["name"]) + c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": self.inputs_record_schema["fields"].append(c) # type: ignore elif key == "outputs": @@ -394,6 +396,7 @@ def _init_job(self, joborder, **kwargs): unicode]], copy.deepcopy(joborder)) fillInDefaults(self.tool[u"inputs"], builder.job) + normalizeFilesDirs(builder.job) # Validate job order try: @@ -564,6 +567,7 @@ def uniquename(stem): # type: (unicode) -> unicode return u def nestdir(base, deps): + # type: (unicode, Dict[unicode, Any]) -> Dict[unicode, Any] dirname = os.path.dirname(base) + "/" subid = deps["location"] if subid.startswith(dirname): @@ -573,34 +577,30 @@ def nestdir(base, deps): while sp: nx = sp.pop() deps = { - "entryname": nx, - "entry": { - "class": "Directory", - "listing": [deps] - } + "class": "Directory", + "basename": nx, + "listing": [deps] } return deps def mergedirs(listing): - r = [] - ents = {} + # type: (List[Dict[unicode, Any]]) -> List[Dict[unicode, Any]] + r = [] # type: List[Dict[unicode, Any]] + ents = {} # type: Dict[unicode, Any] for e in listing: - if "entryname" in e: - if e["entryname"] not in ents: - ents[e["entryname"]] = e - elif e["entry"]["class"] == "Directory": - ents[e["entryname"]]["entry"]["listing"].extend(e["entry"]["listing"]) - else: - r.append(e) + if e["basename"] not in ents: + ents[e["basename"]] = e + elif e["class"] == "Directory": + ents[e["basename"]]["listing"].extend(e["listing"]) for e in ents.itervalues(): - if e["entry"]["class"] == "Directory": - e["entry"]["listing"] = mergedirs(e["entry"]["listing"]) + if e["class"] == "Directory": + e["listing"] = mergedirs(e["listing"]) r.extend(ents.itervalues()) return r def scandeps(base, doc, reffields, urlfields, loadref): - # type: (unicode, Any, Set[str], Set[str], Callable[[unicode, str], Any]) -> List[Dict[str, str]] - r = [] + # type: (unicode, Any, Set[unicode], Set[unicode], Callable[[unicode, unicode], Any]) -> List[Dict[unicode, unicode]] + r = [] # type: List[Dict[unicode, unicode]] if isinstance(doc, dict): if "id" in doc: if doc["id"].startswith("file://"): @@ -623,7 +623,7 @@ def scandeps(base, doc, reffields, urlfields, loadref): deps = { "class": "File", "location": subid - } # type: Dict[str, Any] + } # type: Dict[unicode, Any] sf = scandeps(subid, sub, reffields, urlfields, loadref) if sf: deps["secondaryFiles"] = sf @@ -643,5 +643,6 @@ def scandeps(base, doc, reffields, urlfields, loadref): for d in doc: r.extend(scandeps(base, d, reffields, urlfields, loadref)) + normalizeFilesDirs(r) r = mergedirs(r) return r diff --git a/cwltool/schemas/draft-4/CommandLineTool.yml b/cwltool/schemas/draft-4/CommandLineTool.yml index f33dadfc4..9d4fa1bec 100644 --- a/cwltool/schemas/draft-4/CommandLineTool.yml +++ b/cwltool/schemas/draft-4/CommandLineTool.yml @@ -591,7 +591,46 @@ $graph: Docker container. -- name: DirentExt +- type: record + name: SoftwareRequirement + extends: "#ProcessRequirement" + doc: | + Software that should be configured in the environment of the defined + process. + + While absolute portability between platforms can be achieved with container + technologies like Docker, this mechanism provides an inexact and imprecise + fallback. The inexact nature is due to this standard not specifying the + base operating environment of the process or how the that environment is to + be configured as a result of these hints. The imprecise nature of these + hints is due to the `name` and `version` fields being unstructured and not + namespaced. The optional `identifier` field is recommended so that users + and platforms can distinguish between software that has the same name. + fields: + - name: spec + type: + type: array + items: SoftwareSpec + doc: "The list of software to be configured." + +- name: SoftwareSpec + type: record + fields: + - name: name + type: string + doc: "The name of the software to be configured." + jsonldPredicate: "@id" + - name: "version" + type: string? + doc: "The (optional) version of the software to configured." + - name: "identifier" + type: string? + doc: | + The (optional) identifier of the software to be configured. Should be a + IRI such as an [RRID](http://www.identifiers.org/rrid/SCR_001156). + Example: `http://identifiers.org/rrid/RRID:SCR_001156` + +- name: Dirent type: record doc: | Define a file or subdirectory that must be placed in the designated output @@ -600,11 +639,12 @@ $graph: template. fields: - name: entryname - type: [string, Expression] + type: ["null", string, Expression] jsonldPredicate: _id: cwl:entryname doc: | The name of the file or subdirectory to create in the output directory. + If `entry` is a File or Directory, this overrides `basename`. Optional. - name: entry type: [string, Expression] jsonldPredicate: @@ -633,6 +673,7 @@ $graph: the original file or directory. Default false (files and directories read-only by default). + - name: InitialWorkDirRequirement type: record extends: ProcessRequirement @@ -644,19 +685,18 @@ $graph: - name: listing type: - type: array - items: [File, DirentExt] + items: [File, Directory, Dirent, string, Expression] - string - Expression jsonldPredicate: _id: "cwl:listing" - mapSubject: entryname - mapPredicate: entry doc: | The list of files or subdirectories that must be placed in the designated output directory prior to executing the command line tool. May be an expression. If so, the expression return value must validate - as `{type: array, items: [File, Dirent]}`. + as `{type: array, items: [File, Directory]}`. + - name: EnvVarRequirement type: record diff --git a/cwltool/schemas/draft-4/Process.yml b/cwltool/schemas/draft-4/Process.yml index 351bab9d2..6c74cacd5 100644 --- a/cwltool/schemas/draft-4/Process.yml +++ b/cwltool/schemas/draft-4/Process.yml @@ -65,7 +65,7 @@ $graph: _type: "@vocab" doc: Must be `File` to indicate this object describes a file. - name: location - type: string + type: string? doc: | A URI that identifies the file resource. This may be a relative reference, in which case it must be resolved using the base URI of the @@ -75,6 +75,10 @@ $graph: remote resource (due to unsupported protocol, access denied, or other issue) it must signal an error. + If the `location' field is not provided, the `contents` field must be + provided. The implementation must assign a unique identifier for + the `location` field. + If the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, then follow the rules above. @@ -84,24 +88,34 @@ $graph: - name: path type: string? doc: | - The local path where the File is made available prior to executing a - CommandLineTool. This must be set by the implementation. This field + The local host path where the File is available when a CommandLineTool is + executed. This field must be set by the implementation. The final + path component must match the value of `basename`. This field must not be used in any other context. The command line tool being executed must be able to to access the file at `path` using the POSIX `open(2)` syscall. + + As a special case, if the `path` field is provided but the `location` + field is not, an implementation may assign the value of the `path` + field to `location`, and remove the `path` field. jsonldPredicate: "_id": "cwl:path" "_type": "@id" - name: basename - type: string? + type: string doc: | - The base name of the file, that is, the path component following the - final slash in the path. - - The implementation must set this field based on the value of `path` - prior to evaluating parameter references or expressions in a - CommandLineTool document. This field must not be used in any other - context. + The base name of the file, that is, the name of the file without any + leading directory path. The base name must not contain a slash `/`. + + If not provided, the implementation must set this field based on the + `location` field by taking the final path component after parsing + `location` as an IRI. If `basename` is provided, it is not required to + match the value from `location`. + + When this file is made available to a CommandLineTool, it must be named + with `basename`, i.e. the final component of the `path` field must match + `basename`. + jsonldPredicate: "cwl:basename" - name: dirname type: string? doc: | @@ -121,10 +135,8 @@ $graph: period. Leading periods on the basename are ignored; a basename of `.cshrc` will have a nameroot of `.cshrc`. - The implementation must set this field based on the value of `path` - prior to evaluating parameter references or expressions in a - CommandLineTool document. This field must not be used in any other - context. + The implementation must set this field automatically based on the value + of `basename` prior to evaluating parameter references or expressions. - name: nameext type: string? doc: | @@ -133,10 +145,8 @@ $graph: period. Leading periods on the basename are ignored; a basename of `.cshrc` will have an empty `nameext`. - The implementation must set this field based on the value of `path` - prior to evaluating parameter references or expressions in a - CommandLineTool document. This field must not be used in any other - context. + The implementation must set this field automatically based on the value + of `basename` prior to evaluating parameter references or expressions. - name: checksum type: ["null", string] doc: | @@ -149,7 +159,7 @@ $graph: type: - "null" - type: array - items: [File, Dirent] + items: [File, Directory] jsonldPredicate: "cwl:secondaryFiles" doc: | A list of additional files that are associated with the primary file @@ -179,26 +189,26 @@ $graph: File format ontologies may be provided in the "$schema" metadata at the root of the document. If no ontologies are specified in `$schema`, the runtime may perform exact file format matches. + - name: contents + type: string? + doc: | + File contents literal. Maximum of 64 KiB. + If neither `location` nor `path` is provided, `contents` must be + non-null. The implementation must assign a unique identifier for the + `location` field. When the file is staged as input to CommandLineTool, + the value of `contents` must be written to a file. + + If `loadContents` of `Binding` is true and `location` is valid, the + implementation must read up to the first 64 KiB of text from the file + and place it in the "contents" field. -- name: Dirent - type: record - fields: - - name: entryname - type: string - jsonldPredicate: - "_id": cwl:entryname - - name: entry - type: [File, Directory] - jsonldPredicate: - "_id": cwl:entry - name: Directory type: record docParent: "#CWLType" doc: | - Represents a directory to present to a command line tool. This could be a virtual - directory, made of files assembled from multiple locations. + Represents a directory to present to a command line tool. fields: - name: class type: @@ -211,7 +221,7 @@ $graph: _type: "@vocab" doc: Must be `Directory` to indicate this object describes a Directory. - name: location - type: string + type: string? doc: | A URI that identifies the directory resource. This may be a relative reference, in which case it must be resolved using the base URI of the @@ -222,6 +232,10 @@ $graph: unsupported protocol, access denied, or other issue) it must signal an error. + If the `location' field is not provided, the `listing` field must be + provided. The implementation must assign a unique identifier for + the `location` field. + If the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, then follow the rules above. @@ -239,21 +253,63 @@ $graph: jsonldPredicate: _id: "cwl:path" _type: "@id" + - name: basename + type: string + doc: | + The base name of the directory, that is, the name of the file without any + leading directory path. The base name must not contain a slash `/`. + + If not provided, the implementation must set this field based on the + `location` field by taking the final path component after parsing + `location` as an IRI. If `basename` is provided, it is not required to + match the value from `location`. + + When this file is made available to a CommandLineTool, it must be named + with `basename`, i.e. the final component of the `path` field must match + `basename`. + jsonldPredicate: "cwl:basename" - name: listing type: - "null" - type: array - items: [File, Dirent] - doc: List of files or subdirectories contained in this directory + items: [File, Directory] + doc: | + List of files or subdirectories contained in this directory. The name + of each file or subdirectory is determined by the `basename` field of + each `File` or `Directory` object. It is an error if a `File` shares a + `basename` with any other entry in `listing`. If two or more + `Directory` object share the same `basename`, this must be treated as + equivalent to a single subdirectory with the listings recursively + merged. jsonldPredicate: _id: "cwl:listing" - mapSubject: entryname - mapPredicate: entry - name: SchemaBase type: record abstract: true + + +- name: Parameter + type: record + extends: "#SchemaBase" + abstract: true + doc: | + Define an input or output parameter to a process. + fields: + - name: label + type: + - "null" + - string + jsonldPredicate: "rdfs:label" + doc: "A short, human-readable label of this parameter object." + + - name: description + type: + - "null" + - string + jsonldPredicate: "rdfs:comment" + doc: "A long, human-readable description of this parameter object." - name: secondaryFiles type: - "null" @@ -312,29 +368,6 @@ $graph: pipe. Default: `false`. -- name: Parameter - type: record - extends: "#SchemaBase" - abstract: true - doc: | - Define an input or output parameter to a process. - - fields: - - name: label - type: - - "null" - - string - jsonldPredicate: "rdfs:label" - doc: "A short, human-readable label of this parameter object." - - - name: description - type: - - "null" - - string - jsonldPredicate: "rdfs:comment" - doc: "A long, human-readable description of this parameter object." - - - type: enum name: Expression doc: | diff --git a/cwltool/schemas/draft-4/Workflow.yml b/cwltool/schemas/draft-4/Workflow.yml index fbd1bef66..d1f00fc32 100644 --- a/cwltool/schemas/draft-4/Workflow.yml +++ b/cwltool/schemas/draft-4/Workflow.yml @@ -255,10 +255,12 @@ $graph: the value of the parameter(s) specified in the `source` field, or null if there is no `source` field. - The value of `inputs` in the parameter reference or expression is the - input object to the workflow step after assigning the `source` values, - but before evaluating any step with `valueFrom`. The order of - evaluating `valueFrom` among step input parameters is undefined. + The value of `inputs` in the parameter reference or expression must be + the input object to the workflow step after assigning the `source` + values and then scattering. The order of evaluating `valueFrom` among + step input parameters is undefined and the result of evaluating + `valueFrom` on a parameter must not be visible to evaluation of + `valueFrom` on other parameters. - type: record @@ -502,7 +504,7 @@ $graph: extends: ProcessRequirement doc: | Indicates that the workflow platform must support nested workflows in - the `run` field of (WorkflowStep)(#WorkflowStep). + the `run` field of [WorkflowStep](#WorkflowStep). - name: ScatterFeatureRequirement type: record diff --git a/cwltool/schemas/draft-4/conformance_test_draft-4.yaml b/cwltool/schemas/draft-4/conformance_test_draft-4.yaml index 7769640b8..7d4b31a6d 100644 --- a/cwltool/schemas/draft-4/conformance_test_draft-4.yaml +++ b/cwltool/schemas/draft-4/conformance_test_draft-4.yaml @@ -78,28 +78,24 @@ doc: Test command execution in Docker with stdout redirection - job: - output: { - "output_file": { - "checksum": "sha1$cec7b8746a78c42060c96505887449bca0142976", - "size": 84, - "location": "error.txt", - "class": "File" - } - } tool: draft-4/egrep-stderr.cwl doc: Test command line with stderr redirection + output: + output_file: + class: File + checksum: sha1$cec7b8746a78c42060c96505887449bca0142976 + size: 84 + location: error.txt - job: - output: { - "output_file": { - "checksum": "sha1$cec7b8746a78c42060c96505887449bca0142976", - "size": 84, - "location": "", - "class": "File" - } - } tool: draft-4/egrep-stderr-shortcut.cwl doc: Test command line with stderr redirection, brief syntax + output: + output_file: + class: File + checksum: sha1$cec7b8746a78c42060c96505887449bca0142976 + size: 84 + location: Any - output: output_file: @@ -690,6 +686,12 @@ tool: "draft-4/scatter-valuefrom-wf4.cwl#main" doc: Test workflow scatter with two scatter parameters and dotproduct join method and valueFrom on step input +- job: draft-4/scatter-valuefrom-job1.json + output: + out: ["foo one one", "foo two two", "foo three three", "foo four four"] + tool: draft-4/scatter-valuefrom-wf5.cwl + doc: Test workflow scatter with single scatter parameter and valueFrom on step input + - job: draft-4/conflict-job.json output: { "fileout": { @@ -771,3 +773,15 @@ } tool: draft-4/dir5.cwl doc: Test dynamic initial work dir + +- job: draft-4/stagefile-job.yml + output: { + "outfile": { + "checksum": "sha1$2d6c5e4430c4b200227cc77d3b0025082505ee19", + "size": 1107, + "location": "whale.txt", + "class": "File" + } + } + tool: draft-4/stagefile.cwl + doc: Test writable staged files. diff --git a/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl b/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl index 2ccde1553..4b14f7bda 100755 --- a/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/cat1-tool.cwl @@ -5,6 +5,8 @@ description: "Print the contents of a file to stdout using 'cat' running in a do hints: DockerRequirement: dockerPull: debian:wheezy + SoftwareRequirement: + name: cat inputs: file1: type: File diff --git a/cwltool/schemas/draft-4/draft-4/dir4-job.yml b/cwltool/schemas/draft-4/draft-4/dir4-job.yml index 8da860d27..f4ef8c7d5 100644 --- a/cwltool/schemas/draft-4/draft-4/dir4-job.yml +++ b/cwltool/schemas/draft-4/draft-4/dir4-job.yml @@ -4,7 +4,6 @@ inf: secondaryFiles: - class: File location: index.py - - entryname: xtestdir - entry: - class: Directory - location: testdir + - class: Directory + basename: xtestdir + location: testdir diff --git a/cwltool/schemas/draft-4/draft-4/rename.cwl b/cwltool/schemas/draft-4/draft-4/rename.cwl index e97743ee0..ca3ae58f4 100644 --- a/cwltool/schemas/draft-4/draft-4/rename.cwl +++ b/cwltool/schemas/draft-4/draft-4/rename.cwl @@ -4,8 +4,7 @@ baseCommand: "true" requirements: InitialWorkDirRequirement: listing: - - entryname: $(inputs.newname) - entry: $(inputs.srcfile) + $(inputs.newname): $(inputs.srcfile.location) inputs: srcfile: File newname: string diff --git a/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf5.cwl b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf5.cwl new file mode 100644 index 000000000..417ea4997 --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/scatter-valuefrom-wf5.cwl @@ -0,0 +1,58 @@ +#!/usr/bin/env cwl-runner +cwlVersion: cwl:draft-4.dev3 +class: Workflow +inputs: + inp: + type: + type: array + items: + type: record + name: instr + fields: + - name: instr + type: string +outputs: + out: + type: + type: array + items: string + outputSource: step1/echo_out + +requirements: + - class: ScatterFeatureRequirement + - class: StepInputExpressionRequirement + +steps: + step1: + in: + echo_in: + source: inp + valueFrom: $(self.instr) + first: + source: inp + valueFrom: $(inputs.echo_in.instr) + out: [echo_out] + scatter: echo_in + run: + class: CommandLineTool + inputs: + first: + type: string + inputBinding: + position: 1 + echo_in: + type: string + inputBinding: + position: 2 + outputs: + echo_out: + type: string + outputBinding: + glob: "step1_out" + loadContents: true + outputEval: $(self[0].contents) + baseCommand: "echo" + arguments: + - "-n" + - "foo" + stdout: "step1_out" diff --git a/cwltool/schemas/draft-4/draft-4/stagefile-job.yml b/cwltool/schemas/draft-4/draft-4/stagefile-job.yml new file mode 100644 index 000000000..b9852bce0 --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/stagefile-job.yml @@ -0,0 +1,3 @@ +infile: + class: File + location: whale.txt \ No newline at end of file diff --git a/cwltool/schemas/draft-4/draft-4/stagefile.cwl b/cwltool/schemas/draft-4/draft-4/stagefile.cwl new file mode 100644 index 000000000..bf73c73db --- /dev/null +++ b/cwltool/schemas/draft-4/draft-4/stagefile.cwl @@ -0,0 +1,16 @@ +class: CommandLineTool +cwlVersion: draft-4.dev3 +requirements: + InitialWorkDirRequirement: + listing: + - entry: $(inputs.infile) + writable: true +inputs: + infile: File +outputs: + outfile: + type: File + outputBinding: + glob: $(inputs.infile.basename) +baseCommand: "sed" +arguments: ["-i", "s/Ishmael/Bob/", $(inputs.infile.basename)] diff --git a/cwltool/schemas/draft-4/draft-4/template-tool.cwl b/cwltool/schemas/draft-4/draft-4/template-tool.cwl index f7a00024f..b3ec7b85d 100755 --- a/cwltool/schemas/draft-4/draft-4/template-tool.cwl +++ b/cwltool/schemas/draft-4/draft-4/template-tool.cwl @@ -10,8 +10,7 @@ requirements: - "var t = function(s) { return _.template(s)({'inputs': inputs}); };" - class: InitialWorkDirRequirement listing: - foo.txt: > - $(t("The file is <%= inputs.file1.path.split('/').slice(-1)[0] %>\n")) + foo.txt: $(t("The file is <%= inputs.file1.path.split('/').slice(-1)[0] %>\n")) inputs: - id: file1 type: File diff --git a/cwltool/schemas/draft-4/salad/schema_salad/jsonld_context.py b/cwltool/schemas/draft-4/salad/schema_salad/jsonld_context.py index 79d98333f..225b686f5 100755 --- a/cwltool/schemas/draft-4/salad/schema_salad/jsonld_context.py +++ b/cwltool/schemas/draft-4/salad/schema_salad/jsonld_context.py @@ -175,3 +175,52 @@ def salad_to_jsonld_context(j, schema_ctx): process_type(t, g, context, defaultBase, namespaces, defaultPrefix) return (context, g) + +def fix_jsonld_ids(obj, ids): + # type: (Union[Dict[unicode, Any], List[Dict[unicode, Any]]], List[unicode]) -> None + if isinstance(obj, dict): + for i in ids: + if i in obj: + obj["@id"] = obj[i] + for v in obj.values(): + fix_jsonld_ids(v, ids) + if isinstance(obj, list): + for entry in obj: + fix_jsonld_ids(entry, ids) + +def makerdf(workflow, wf, ctx): + # type: (Union[str, unicode], Union[List[Dict[unicode, Any]], Dict[unicode, Any]], Loader.ContextType) -> Graph + prefixes = {} + idfields = [] + for k,v in ctx.iteritems(): + if isinstance(v, dict): + url = v["@id"] + else: + url = v + if url == "@id": + idfields.append(k) + doc_url, frg = urlparse.urldefrag(url) + if "/" in frg: + p, _ = frg.split("/") + prefixes[p] = u"%s#%s/" % (doc_url, p) + + if isinstance(wf, list): + wf = { + "@context": ctx, + "@graph": wf + } + else: + wf["@context"] = ctx + + fix_jsonld_ids(wf, idfields) + + g = Graph().parse(data=json.dumps(wf), format='json-ld', location=workflow) + + # Bug in json-ld loader causes @id fields to be added to the graph + for s,p,o in g.triples((None, URIRef("@id"), None)): + g.remove((s, p, o)) + + for k2,v2 in prefixes.iteritems(): + g.namespace_manager.bind(k2, v2) + + return g diff --git a/cwltool/schemas/draft-4/salad/schema_salad/main.py b/cwltool/schemas/draft-4/salad/schema_salad/main.py index 109e225ba..3c9a634e5 100644 --- a/cwltool/schemas/draft-4/salad/schema_salad/main.py +++ b/cwltool/schemas/draft-4/salad/schema_salad/main.py @@ -23,9 +23,8 @@ def printrdf(workflow, wf, ctx, sr): - # type: (str, Union[List, Dict[Any, Any], str, unicode], Dict[unicode, Any], str) -> None - g = Graph().parse(data=json.dumps(wf), format='json-ld', - location=workflow, context=ctx) + # type: (str, Union[List[Dict[unicode, Any]], Dict[unicode, Any]], Dict[unicode, Any], str) -> None + g = jsonld_context.makerdf(workflow, wf, ctx) print(g.serialize(format=sr)) @@ -210,8 +209,12 @@ def main(argsl=None): # type: (List[str]) -> int # Optionally convert the document to RDF if args.print_rdf: - printrdf(args.document, document, schema_ctx, args.rdf_serializer) - return 0 + if isinstance(document, (dict, list)): + printrdf(args.document, document, schema_ctx, args.rdf_serializer) + return 0 + else: + print("Document must be a dictionary or list.") + return 1 if args.print_metadata: print(json.dumps(doc_metadata, indent=4)) diff --git a/cwltool/schemas/draft-4/salad/schema_salad/metaschema/metaschema_base.yml b/cwltool/schemas/draft-4/salad/schema_salad/metaschema/metaschema_base.yml index 42901a60e..9b5c2aebc 100644 --- a/cwltool/schemas/draft-4/salad/schema_salad/metaschema/metaschema_base.yml +++ b/cwltool/schemas/draft-4/salad/schema_salad/metaschema/metaschema_base.yml @@ -43,19 +43,19 @@ $graph: type: record doc: A field of a record. fields: - - name: name + name: type: string jsonldPredicate: "@id" doc: | The name of the field - - name: doc + doc: type: string? doc: | A documentation string for this field jsonldPredicate: "sld:doc" - - name: type + type: type: - PrimitiveType - RecordSchema @@ -81,7 +81,7 @@ $graph: - name: RecordSchema type: record fields: - - name: type + type: doc: "Must be `record`" type: name: Record_symbol @@ -93,9 +93,12 @@ $graph: _type: "@vocab" typeDSL: true refScope: 2 - - name: "fields" + fields: type: RecordField[]? - jsonldPredicate: "sld:fields" + jsonldPredicate: + _id: sld:fields + mapSubject: name + mapPredicate: type doc: "Defines the fields of the record." @@ -104,7 +107,7 @@ $graph: doc: | Define an enumerated type. fields: - - name: type + type: doc: "Must be `enum`" type: name: Enum_symbol @@ -116,7 +119,7 @@ $graph: _type: "@vocab" typeDSL: true refScope: 2 - - name: "symbols" + symbols: type: string[] jsonldPredicate: _id: "sld:symbols" @@ -128,7 +131,7 @@ $graph: - name: ArraySchema type: record fields: - - name: type + type: doc: "Must be `array`" type: name: Array_symbol @@ -140,7 +143,7 @@ $graph: _type: "@vocab" typeDSL: true refScope: 2 - - name: items + items: type: - PrimitiveType - RecordSchema diff --git a/cwltool/schemas/draft-4/salad/schema_salad/ref_resolver.py b/cwltool/schemas/draft-4/salad/schema_salad/ref_resolver.py index 7051dbc53..4c4bb9db0 100644 --- a/cwltool/schemas/draft-4/salad/schema_salad/ref_resolver.py +++ b/cwltool/schemas/draft-4/salad/schema_salad/ref_resolver.py @@ -135,8 +135,9 @@ def expand_url(self, url, base_url, scoped_id=False, vocab_term=False, scoped_re frg = splitbase.fragment + u"/" + split.path else: frg = split.path + pt = splitbase.path if splitbase.path else "/" url = urlparse.urlunsplit( - (splitbase.scheme, splitbase.netloc, splitbase.path, splitbase.query, frg)) + (splitbase.scheme, splitbase.netloc, pt, splitbase.query, frg)) elif scoped_ref is not None and not split.fragment: pass else: @@ -156,7 +157,7 @@ def _add_properties(self, s): # type: (unicode) -> None u"http://www.w3.org/2000/01/rdf-schema#Literal") if not literal: self.url_fields.add(unicode(s)) - self.foreign_properties.add(s) + self.foreign_properties.add(unicode(s)) def add_namespaces(self, ns): # type: (Dict[unicode, unicode]) -> None self.vocab.update(ns) diff --git a/cwltool/schemas/draft-4/salad/schema_salad/schema.py b/cwltool/schemas/draft-4/salad/schema_salad/schema.py index 4c997ba61..c90c3446b 100644 --- a/cwltool/schemas/draft-4/salad/schema_salad/schema.py +++ b/cwltool/schemas/draft-4/salad/schema_salad/schema.py @@ -97,7 +97,11 @@ def get_metaschema(): "@type": "@id", "refScope": 1 }, - "fields": "sld:fields", + "fields": { + "@id": "sld:fields", + "mapSubject": "name", + "mapPredicate": "type" + }, "float": "http://www.w3.org/2001/XMLSchema#float", "identity": "https://w3id.org/cwl/salad#JsonldPredicate/identity", "int": "http://www.w3.org/2001/XMLSchema#int", diff --git a/cwltool/schemas/draft-4/salad/setup.py b/cwltool/schemas/draft-4/salad/setup.py index 9c12ddaf3..e049f9f84 100755 --- a/cwltool/schemas/draft-4/salad/setup.py +++ b/cwltool/schemas/draft-4/salad/setup.py @@ -41,7 +41,7 @@ extras_require = {} # TODO: to be removed when the above is added setup(name='schema-salad', - version='1.12', + version='1.13', description='Schema Annotations for Linked Avro Data (SALAD)', long_description=open(README).read(), author='Common workflow language working group', diff --git a/cwltool/schemas/draft-4/salad/tests/test_examples.py b/cwltool/schemas/draft-4/salad/tests/test_examples.py index 36c823a6c..f49e9089b 100644 --- a/cwltool/schemas/draft-4/salad/tests/test_examples.py +++ b/cwltool/schemas/draft-4/salad/tests/test_examples.py @@ -2,8 +2,11 @@ import schema_salad.ref_resolver import schema_salad.main import schema_salad.schema +from schema_salad.jsonld_context import makerdf import rdflib import ruamel.yaml as yaml +import json + try: from ruamel.yaml import CSafeLoader as SafeLoader except ImportError: @@ -249,5 +252,75 @@ def test_typedsl_ref(self): ra, _ = ldr.resolve_all({"type": "File[]?"}, "") self.assertEqual({'type': ['null', {'items': 'File', 'type': 'array'}]}, ra) + def test_scoped_id(self): + ldr = schema_salad.ref_resolver.Loader({}) + ctx = { + "id": "@id", + "location": { + "@id": "@id", + "@type": "@id" + }, + "bar": "http://example.com/bar", + "ex": "http://example.com/" + } + ldr.add_context(ctx) + + ra, _ = ldr.resolve_all({ + "id": "foo", + "bar": { + "id": "baz" + } + }, "http://example.com") + self.assertEqual({'id': 'http://example.com/#foo', + 'bar': { + 'id': 'http://example.com/#foo/baz'}, + }, ra) + + g = makerdf(None, ra, ctx) + print(g.serialize(format="n3")) + + ra, _ = ldr.resolve_all({ + "location": "foo", + "bar": { + "location": "baz" + } + }, "http://example.com", checklinks=False) + self.assertEqual({'location': 'http://example.com/foo', + 'bar': { + 'location': 'http://example.com/baz'}, + }, ra) + + g = makerdf(None, ra, ctx) + print(g.serialize(format="n3")) + + ra, _ = ldr.resolve_all({ + "id": "foo", + "bar": { + "location": "baz" + } + }, "http://example.com", checklinks=False) + self.assertEqual({'id': 'http://example.com/#foo', + 'bar': { + 'location': 'http://example.com/baz'}, + }, ra) + + g = makerdf(None, ra, ctx) + print(g.serialize(format="n3")) + + ra, _ = ldr.resolve_all({ + "location": "foo", + "bar": { + "id": "baz" + } + }, "http://example.com", checklinks=False) + self.assertEqual({'location': 'http://example.com/foo', + 'bar': { + 'id': 'http://example.com/#baz'}, + }, ra) + + g = makerdf(None, ra, ctx) + print(g.serialize(format="n3")) + + if __name__ == '__main__': unittest.main() diff --git a/cwltool/schemas/site/cwlsite-draft3-job.json b/cwltool/schemas/site/cwlsite-draft3-job.json index 2311b68dd..24fc20202 100644 --- a/cwltool/schemas/site/cwlsite-draft3-job.json +++ b/cwltool/schemas/site/cwlsite-draft3-job.json @@ -50,11 +50,7 @@ "primtype": "#CWLType" }, { - "source": { - "class": "File", - "path": "../draft-3/salad/schema_salad/metaschema/metaschema.yml", - "secondaryFiles": {"$import": "draft3-deps.json"} - }, + "source": {$import: draft3-metaschema.json}, "target": "draft-3/SchemaSalad.html", "renderlist": [ "https://w3id.org/cwl/salad#Semantic_Annotations_for_Linked_Avro_Data", diff --git a/cwltool/schemas/site/cwlsite-draft3-schemas.json b/cwltool/schemas/site/cwlsite-draft3-schemas.json index 33c8f43cf..cc6f9cf27 100644 --- a/cwltool/schemas/site/cwlsite-draft3-schemas.json +++ b/cwltool/schemas/site/cwlsite-draft3-schemas.json @@ -7,11 +7,7 @@ "context_target": "draft-3/cwl-context.json", "rdfs_target": "draft-3/cwl.ttl" }, { - "schema_in": { - "class": "File", - "path": "../draft-3/salad/schema_salad/metaschema/metaschema.yml", - "secondaryFiles": {"$import": "draft3-deps.json"} - }, + "schema_in": {$import: draft3-metaschema.json}, "context_target": "draft-3/salad-context.json", "rdfs_target": "draft-3/salad.ttl" }] diff --git a/cwltool/schemas/site/cwlsite-draft4-job.json b/cwltool/schemas/site/cwlsite-draft4-job.json index 273b5536e..1f3078d09 100644 --- a/cwltool/schemas/site/cwlsite-draft4-job.json +++ b/cwltool/schemas/site/cwlsite-draft4-job.json @@ -50,11 +50,7 @@ "primtype": "#CWLType" }, { - "source": { - "class": "File", - "path": "../draft-4/salad/schema_salad/metaschema/metaschema.yml", - "secondaryFiles": {"$import": "draft4-deps.json"} - }, + "source": {$import: draft4-metaschema.json}, "target": "draft-4/SchemaSalad.html", "renderlist": [ "https://w3id.org/cwl/salad#Semantic_Annotations_for_Linked_Avro_Data", diff --git a/cwltool/schemas/site/cwlsite-draft4-schemas.json b/cwltool/schemas/site/cwlsite-draft4-schemas.json index c01c31dd5..6681827f1 100644 --- a/cwltool/schemas/site/cwlsite-draft4-schemas.json +++ b/cwltool/schemas/site/cwlsite-draft4-schemas.json @@ -7,11 +7,7 @@ "context_target": "draft-4/cwl-context.json", "rdfs_target": "draft-4/cwl.ttl" }, { - "schema_in": { - "class": "File", - "path": "../draft-4/salad/schema_salad/metaschema/metaschema.yml", - "secondaryFiles": {"$import": "draft4-deps.json"} - }, + "schema_in": {$import: draft4-metaschema.json}, "context_target": "draft-4/salad-context.json", "rdfs_target": "draft-4/salad.ttl" } diff --git a/cwltool/schemas/site/cwlsite.cwl b/cwltool/schemas/site/cwlsite.cwl index d9af1e436..394c99e92 100755 --- a/cwltool/schemas/site/cwlsite.cwl +++ b/cwltool/schemas/site/cwlsite.cwl @@ -1,71 +1,41 @@ #!/usr/bin/env cwl-runner -cwlVersion: cwl:draft-3 +cwlVersion: cwl:draft-4.dev3 class: Workflow inputs: - - id: render + render: type: type: array items: - name: render type: record fields: - - name: source - type: File - - name: renderlist - type: - type: array - items: string - - name: redirect - type: - type: array - items: string - - name: target - type: string - - name: brandlink - type: string - - name: brandimg - type: string - - - id: schemas + source: File + renderlist: string[] + redirect: string[] + target: string + brandlink: string + brandimg: string + schemas: type: type: array items: - name: rdfs type: record fields: - - name: schema_in - type: File - - name: context_target - type: string - - name: rdfs_target - type: string - - id: brandimg - type: File - - id: empty + schema_in: File + context_target: string + rdfs_target: string + brandimg: File + empty: type: string default: "" outputs: - - id: doc_out - type: - type: array - items: File - source: ["#docs/out", "#brandimg"] - linkMerge: merge_flattened - - id: report + doc_out: type: File - source: "#report/out" - - id: context - type: - type: array - items: File - source: "#context/out" - - id: rdfs - type: - type: array - items: File - source: "#rdfs/out" + outputSource: merge/dir + report: + type: File + outputSource: report/out requirements: - class: ScatterFeatureRequirement @@ -73,62 +43,63 @@ requirements: - class: SubworkflowFeatureRequirement - class: MultipleInputFeatureRequirement - class: InlineJavascriptRequirement - expressionLib: - - $include: cwlpath.js hints: - class: DockerRequirement dockerPull: commonworkflowlanguage/cwltool_module steps: - - id: rdfs - inputs: - - {id: schema, source: "#schemas", valueFrom: $(self.schema_in) } - - {id: target, source: "#schemas", valueFrom: $(self.rdfs_target) } - outputs: - - { id: out } - scatter: ["#rdfs/schema", "#rdfs/target"] - scatterMethod: dotproduct + rdfs: + scatter: schemas + in: + schemas: schemas + schema: { valueFrom: $(inputs.schemas.schema_in) } + target: { valueFrom: $(inputs.schemas.rdfs_target) } + out: [out, targetdir] run: makerdfs.cwl - - id: context - inputs: - - {id: schema, source: "#schemas", valueFrom: $(self.schema_in) } - - {id: target, source: "#schemas", valueFrom: $(self.context_target) } - outputs: - - { id: out } - scatter: ["#context/schema", "#context/target"] - scatterMethod: dotproduct + context: + scatter: schemas + in: + schemas: schemas + schema: { valueFrom: $(inputs.schemas.schema_in) } + target: { valueFrom: $(inputs.schemas.context_target) } + out: [out, targetdir] run: makecontext.cwl - - id: docs - inputs: - - { id: source, source: "#render", valueFrom: $(self.source) } - - { id: target, source: "#render", valueFrom: $(self.target) } - - { id: renderlist, source: "#render", valueFrom: $(self.renderlist) } - - { id: redirect, source: "#render", valueFrom: $(self.redirect) } - - { id: brandlink, source: "#render", valueFrom: $(self.brandlink) } - - { id: brand, source: "#render", valueFrom: $(self.brandimg) } - - { id: primtype, source: "#render", valueFrom: $(self.primtype) } - outputs: - - { id: out } - - { id: targetdir } - scatter: - - "#docs/source" - - "#docs/target" - - "#docs/renderlist" - - "#docs/redirect" - - "#docs/brandlink" - - "#docs/primtype" - - "#docs/brand" - scatterMethod: dotproduct + docs: + scatter: render + in: + render: render + source: { valueFrom: $(inputs.render.source) } + target: { valueFrom: $(inputs.render.target) } + renderlist: { valueFrom: $(inputs.render.renderlist) } + redirect: { valueFrom: $(inputs.render.redirect) } + brandlink: { valueFrom: $(inputs.render.brandlink) } + brand: { valueFrom: $(inputs.render.brandimg) } + primtype: { valueFrom: $(inputs.render.primtype) } + out: [out, targetdir] run: makedoc.cwl - - id: report - inputs: - - {id: inp, source: ["#docs/out", "#brandimg"], linkMerge: merge_flattened } - - {id: dirs, source: ["#docs/targetdir", "#empty"], linkMerge: merge_flattened } - - {id: target, default: "linkchecker-report.txt"} - outputs: - - id: out + merge: + in: + primary: + source: docs/out + valueFrom: $(self[0]) + secondary: + source: [docs/out, rdfs/out, context/out, brandimg] + linkMerge: merge_flattened + valueFrom: $(self.slice(1)) + dirs: + source: [docs/targetdir, rdfs/targetdir, context/targetdir, empty] + linkMerge: merge_flattened + valueFrom: $(self.slice(1)) + out: [dir] + run: mergesecondary.cwl + + report: + in: + inp: merge/dir + target: { default: "linkchecker-report.txt"} + out: [out] run: linkchecker.cwl diff --git a/cwltool/schemas/site/draft3-deps.json b/cwltool/schemas/site/draft3-deps.json index 4cdcd0e2f..98b90f6e8 100644 --- a/cwltool/schemas/site/draft3-deps.json +++ b/cwltool/schemas/site/draft3-deps.json @@ -1,138 +1,77 @@ [ - { - "path": "../draft-3/Process.yml", - "class": "File", - "secondaryFiles": [ - { - "path": "../draft-3/concepts.md", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/metaschema.yml", - "class": "File", - "secondaryFiles": [ - { - "path": "../draft-3/salad/schema_salad/metaschema/salad.md", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/field_name.yml", - "class": "File", - "secondaryFiles": [ + { + "secondaryFiles": [ + { + "class": "File", + "location": "../draft-3/concepts.md" + }, + { + "entry": { + "class": "Directory", + "listing": [ { - "path": "../draft-3/salad/schema_salad/metaschema/field_name_schema.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/field_name_src.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/field_name_proc.yml", - "class": "File" - } - ] - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/ident_res.yml", - "class": "File", - "secondaryFiles": [ - { - "path": "../draft-3/salad/schema_salad/metaschema/ident_res_schema.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/ident_res_src.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/ident_res_proc.yml", - "class": "File" - } - ] - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/link_res.yml", - "class": "File", - "secondaryFiles": [ - { - "path": "../draft-3/salad/schema_salad/metaschema/link_res_schema.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/link_res_src.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/link_res_proc.yml", - "class": "File" - } - ] - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/vocab_res.yml", - "class": "File", - "secondaryFiles": [ - { - "path": "../draft-3/salad/schema_salad/metaschema/vocab_res_schema.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/vocab_res_src.yml", - "class": "File" - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/vocab_res_proc.yml", - "class": "File" + "entry": { + "class": "Directory", + "listing": [ + { + "entry": { + "class": "Directory", + "listing": [ + {$import: draft3-metaschema.json} + ] + }, + "entryname": "metaschema" + } + ] + }, + "entryname": "schema_salad" } ] - }, - { - "path": "../draft-3/salad/schema_salad/metaschema/import_include.md", - "class": "File" - } - ] - } - ] - }, - { - "path": "../draft-3/CommandLineTool.yml", - "class": "File", - "secondaryFiles": [ - { - "path": "../draft-3/contrib.md", - "class": "File" - }, - { - "path": "../draft-3/intro.md", - "class": "File" - }, - { - "path": "../draft-3/concepts.md", - "class": "File" - }, - { - "path": "../draft-3/invocation.md", - "class": "File" - } - ] - }, - { - "path": "../draft-3/Workflow.yml", - "class": "File", - "secondaryFiles": [ - { - "path": "../draft-3/contrib.md", - "class": "File" - }, - { - "path": "../draft-3/intro.md", - "class": "File" - }, - { - "path": "../draft-3/concepts.md", - "class": "File" - } - ] - } -] + }, + "entryname": "salad" + } + ], + "class": "File", + "location": "../draft-3/Process.yml" + }, + { + "secondaryFiles": [ + { + "class": "File", + "location": "../draft-3/contrib.md" + }, + { + "class": "File", + "location": "../draft-3/intro.md" + }, + { + "class": "File", + "location": "../draft-3/concepts.md" + }, + { + "class": "File", + "location": "../draft-3/invocation.md" + } + ], + "class": "File", + "location": "../draft-3/CommandLineTool.yml" + }, + { + "secondaryFiles": [ + { + "class": "File", + "location": "../draft-3/contrib.md" + }, + { + "class": "File", + "location": "../draft-3/intro.md" + }, + { + "class": "File", + "location": "../draft-3/concepts.md" + } + ], + "class": "File", + "location": "../draft-3/Workflow.yml" + } + ] diff --git a/cwltool/schemas/site/draft4-deps.json b/cwltool/schemas/site/draft4-deps.json index 157ccb1ec..3cddeb186 100644 --- a/cwltool/schemas/site/draft4-deps.json +++ b/cwltool/schemas/site/draft4-deps.json @@ -1,126 +1,80 @@ [ { - "path": "../draft-4/Process.yml", - "class": "File" - }, - { - "path": "../draft-4/concepts.md", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/metaschema.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/metaschema_base.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/salad.md", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/field_name.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/field_name_schema.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/field_name_src.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/field_name_proc.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/ident_res.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/ident_res_schema.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/ident_res_src.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/ident_res_proc.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/link_res.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/link_res_schema.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/link_res_src.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/link_res_proc.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/vocab_res.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/vocab_res_schema.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/vocab_res_src.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/vocab_res_proc.yml", - "class": "File" - }, - { - "path": "../draft-4/salad/schema_salad/metaschema/import_include.md", - "class": "File" - }, - { - "path": "../draft-4/CommandLineTool.yml", - "class": "File" - }, - { - "path": "../draft-4/contrib.md", - "class": "File" - }, - { - "path": "../draft-4/intro.md", - "class": "File" - }, - { - "path": "../draft-4/concepts.md", - "class": "File" - }, - { - "path": "../draft-4/invocation.md", - "class": "File" - }, - { - "path": "../draft-4/Workflow.yml", - "class": "File" - }, - { - "path": "../draft-4/contrib.md", - "class": "File" - }, - { - "path": "../draft-4/intro.md", - "class": "File" - }, - { - "path": "../draft-4/concepts.md", - "class": "File" + "secondaryFiles": [ + { + "class": "File", + "location": "../draft-4/concepts.md" + }, + { + "entry": { + "class": "Directory", + "listing": [ + { + "entry": { + "class": "Directory", + "listing": [ + { + "entry": { + "class": "Directory", + "listing": [ + { + "class": "File", + "location": "../draft-4/salad/schema_salad/metaschema/metaschema_base.yml" + } + ] + }, + "entryname": "metaschema" + } + ] + }, + "entryname": "schema_salad" + } + ] + }, + "entryname": "salad" + } + ], + "class": "File", + "location": "../draft-4/Process.yml" + }, + { + "secondaryFiles": [ + { + "class": "File", + "location": "../draft-4/contrib.md" + }, + { + "class": "File", + "location": "../draft-4/intro.md" + }, + { + "class": "File", + "location": "../draft-4/concepts.md" + }, + { + "class": "File", + "location": "../draft-4/invocation.md" + } + ], + "class": "File", + "location": "../draft-4/CommandLineTool.yml" + }, + { + "secondaryFiles": [ + { + "class": "File", + "location": "../draft-4/contrib.md" + }, + { + "class": "File", + "location": "../draft-4/intro.md" + }, + { + "class": "File", + "location": "../draft-4/concepts.md" + } + ], + "class": "File", + "location": "../draft-4/Workflow.yml" } ] diff --git a/cwltool/schemas/site/linkchecker.cwl b/cwltool/schemas/site/linkchecker.cwl index ec5f3eecc..59218329f 100644 --- a/cwltool/schemas/site/linkchecker.cwl +++ b/cwltool/schemas/site/linkchecker.cwl @@ -1,76 +1,29 @@ class: CommandLineTool -cwlVersion: draft-3 -requirements: - - class: ShellCommandRequirement - - class: InlineJavascriptRequirement - expressionLib: - - $include: cwlpath.js +cwlVersion: draft-4.dev3 hints: - - class: DockerRequirement + DockerRequirement: dockerFile: | FROM debian:8 RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get -yq install w3c-linkchecker \ dockerImageId: commonworkflowlanguage/checklink - inputs: - - id: inp - type: - type: array - items: File - - id: dirs - type: - type: array - items: string - - id: target - type: string + inp: + type: File + inputBinding: {position: 1} + target: string outputs: - - id: out + out: type: File outputBinding: glob: $(inputs.target) -baseCommand: [] -arguments: - - "mkdir" - - "-p" - - valueFrom: | - ${ - var r = []; - for (var i=0; i < inputs.dirs.length; i++) { - if (inputs.dirs[i] != "") { - r.push(inputs.dirs[i]); - } - } - return r; - } - - {valueFrom: "&&", shellQuote: false} - - valueFrom: | - ${ - var r = []; - for (var i=0; i < inputs.inp.length; i++) { - if (i > 0) { - r.push("&&"); - } - r.push("ln"); - r.push("-s"); - r.push(inputs.inp[i].path); - r.push(runtime.outdir + "/" + inputs.dirs[i]); - } - return r; - } - - {valueFrom: "&&", shellQuote: false} - - "checklink" - - "-X(http.*|mailto:.*)" - - "-q" - - valueFrom: | - ${ - var r = []; - for (var i=0; i < inputs.inp.length; i++) { - r.push(cwl.path.basename(inputs.inp[i].path)); - } - return r; - } - - {valueFrom: " > ", shellQuote: false} - - valueFrom: $(inputs.target) - - {valueFrom: " && ! test -s", shellQuote: false} - - valueFrom: $(inputs.target) \ No newline at end of file + loadContents: true + #outputEval: | + # ${ + # return if (self.contents.length > 0) { + # + # } + # } +baseCommand: checklink +arguments: ["-X(http.*|mailto:.*)", "-q"] +stdout: $(inputs.target) diff --git a/cwltool/schemas/site/makecontext.cwl b/cwltool/schemas/site/makecontext.cwl index ae025e14f..bdf07c2c4 100644 --- a/cwltool/schemas/site/makecontext.cwl +++ b/cwltool/schemas/site/makecontext.cwl @@ -1,15 +1,15 @@ -cwlVersion: "cwl:draft-3" +cwlVersion: draft-4.dev3 class: CommandLineTool inputs: - - id: schema + schema: type: File inputBinding: {position: 1} - - id: target - type: string + target: string outputs: - - id: out - type: File + out: stdout + targetdir: + type: string outputBinding: - glob: $(inputs.target) + outputEval: $(inputs.target.match(/^([^/]+)\/[^/]/)[1]) baseCommand: [python, "-mschema_salad", "--print-jsonld-context"] stdout: $(inputs.target) diff --git a/cwltool/schemas/site/makedoc.cwl b/cwltool/schemas/site/makedoc.cwl index c046800c0..ad8004b07 100644 --- a/cwltool/schemas/site/makedoc.cwl +++ b/cwltool/schemas/site/makedoc.cwl @@ -1,46 +1,50 @@ -cwlVersion: "cwl:draft-3" +cwlVersion: draft-4.dev3 class: CommandLineTool requirements: - class: InlineJavascriptRequirement expressionLib: - $include: cwlpath.js inputs: - - id: source + source: type: File inputBinding: {position: 1} - - id: renderlist + renderlist: type: - "null" - type: array items: string inputBinding: {prefix: "--only"} inputBinding: {position: 2} - - id: redirect + redirect: type: - "null" - type: array items: string inputBinding: {prefix: "--redirect"} inputBinding: {position: 2} - - id: brand + brand: type: string inputBinding: {prefix: "--brand"} - - id: brandlink + brandlink: type: string inputBinding: {prefix: "--brandlink"} - - id: target + target: type: string - - id: primtype + primtype: type: ["null", string] inputBinding: {prefix: "--primtype"} outputs: - - id: out - type: File - outputBinding: - glob: $(inputs.target) - - id: targetdir + out: stdout + targetdir: type: string outputBinding: - outputEval: $(cwl.path.dirname(inputs.target)) + outputEval: | + ${ + var m = inputs.target.match(/^([^/]+)\/[^/]/); + if (m) + return m[1]; + else + return ""; + } baseCommand: [python, "-mschema_salad.makedoc"] stdout: $(inputs.target) diff --git a/cwltool/schemas/site/makerdfs.cwl b/cwltool/schemas/site/makerdfs.cwl index afbb1e121..d8b59d29f 100644 --- a/cwltool/schemas/site/makerdfs.cwl +++ b/cwltool/schemas/site/makerdfs.cwl @@ -1,15 +1,15 @@ -cwlVersion: "cwl:draft-3" +cwlVersion: draft-4.dev3 class: CommandLineTool inputs: - - id: schema + schema: type: File inputBinding: {position: 1} - - id: target - type: string + target: string outputs: - - id: out - type: File + out: stdout + targetdir: + type: string outputBinding: - glob: $(inputs.target) + outputEval: $(inputs.target.match(/^([^/]+)\/[^/]/)[1]) baseCommand: [python, "-mschema_salad", "--print-rdfs"] stdout: $(inputs.target) diff --git a/cwltool/schemas/site/makespec.cwl b/cwltool/schemas/site/makespec.cwl index 0eb34ce32..f2fe8bba1 100644 --- a/cwltool/schemas/site/makespec.cwl +++ b/cwltool/schemas/site/makespec.cwl @@ -1,13 +1,13 @@ -cwlVersion: "cwl:draft-3" +cwlVersion: draft-4.dev3 class: Workflow inputs: - - {id: "#schema_in", type: string} - - {id: "#context_target", type: string} - - {id: "#rdfs_target", type: string} + schema_in: string + context_target: string + rdfs_target: string outputs: - - id: index_out + index_out: type: File source: "#doc/out" diff --git a/cwltool/schemas/site/userguide-input.json b/cwltool/schemas/site/userguide-input.json index f393dad56..86e500051 100644 --- a/cwltool/schemas/site/userguide-input.json +++ b/cwltool/schemas/site/userguide-input.json @@ -1,134 +1,142 @@ { - "path": "../draft-3/UserGuide.yml", - "class": "File", "secondaryFiles": [ { - "path": "../draft-3/userguide-intro.md", - "class": "File" - }, - { - "path": "../draft-3/examples/1st-tool.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/echo-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/inp.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/inp-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/tar.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/tar-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/stdout.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/echo-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/tar-param.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/tar-param-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/docker.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/docker-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/arguments.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/arguments-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/array-inputs.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/array-inputs-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/array-outputs.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/array-outputs-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/record.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/record-job1.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/record-job2.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/record-job2.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/env.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/echo-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/expression.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/createfile.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/echo-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/linkfile.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/arguments-job.yml", - "class": "File" - }, - { - "path": "../draft-3/examples/1st-workflow.cwl", - "class": "File" - }, - { - "path": "../draft-3/examples/1st-workflow-job.yml", - "class": "File" + "class": "File", + "location": "../draft-3/userguide-intro.md" + }, + { + "entry": { + "class": "Directory", + "listing": [ + { + "class": "File", + "location": "../draft-3/examples/1st-tool.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/echo-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/inp.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/inp-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/tar.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/tar-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/stdout.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/echo-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/tar-param.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/tar-param-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/docker.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/docker-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/arguments.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/arguments-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/array-inputs.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/array-inputs-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/array-outputs.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/array-outputs-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/record.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/record-job1.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/record-job2.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/record-job2.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/env.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/echo-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/expression.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/createfile.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/echo-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/linkfile.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/arguments-job.yml" + }, + { + "class": "File", + "location": "../draft-3/examples/1st-workflow.cwl" + }, + { + "class": "File", + "location": "../draft-3/examples/1st-workflow-job.yml" + } + ] + }, + "entryname": "examples" } - ] + ], + "class": "File", + "location": "../draft-3/UserGuide.yml" } \ No newline at end of file diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index cfdcb657b..53498013c 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -21,11 +21,11 @@ def open(self, fn, mode): # type: (unicode, str) -> BinaryIO def exists(self, fn): # type: (unicode) -> bool return os.path.exists(self._abs(fn)) - def isfile(self, fn): + def isfile(self, fn): # type: (unicode) -> bool return os.path.isfile(self._abs(fn)) - def isdir(self, fn): + def isdir(self, fn): # type: (unicode) -> bool return os.path.isdir(self._abs(fn)) - def listdir(self, fn): + def listdir(self, fn): # type: (unicode) -> List[unicode] return [abspath(l, fn) for l in os.listdir(self._abs(fn))] diff --git a/cwltool/update.py b/cwltool/update.py index dfe76b14e..dd0dc4090 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -324,7 +324,7 @@ def _draft3toDraft4dev1(doc, loader, baseuri): # type: (Any, Loader, str) -> Any if isinstance(doc, dict): if "class" in doc and doc["class"] == "Workflow": - def fixup(f): + def fixup(f): # type: (str) -> str doc, frg = urlparse.urldefrag(f) frg = '/'.join(frg.rsplit('.', 1)) return doc + "#" + frg diff --git a/cwltool/workflow.py b/cwltool/workflow.py index e793d901a..475c26aa4 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -91,6 +91,8 @@ def match_types(sinktype, src, iid, inputobj, linkMerge, valueFrom): def can_assign_src_to_sink(src, sink): # type: (Any, Any) -> bool """Check for identical type specifications, ignoring extra keys like inputBinding. """ + if sink == "Any": + return True if isinstance(src, dict) and isinstance(sink, dict): if src["type"] == "array" and sink["type"] == "array": return can_assign_src_to_sink(src["items"], sink["items"]) @@ -243,6 +245,7 @@ def try_make_job(self, step, **kwargs): vfinputs = {shortname(k): v for k,v in inputobj.iteritems()} def postScatterEval(io): + # type: (Dict[unicode, Any]) -> Dict[unicode, Any] shortio = {shortname(k): v for k,v in io.iteritems()} def valueFromFunc(k, v): # type: (Any, Any) -> Any if k in valueFrom: @@ -299,7 +302,7 @@ def run(self, **kwargs): _logger.debug(u"[%s] workflow starting", self.name) def job(self, joborder, output_callback, **kwargs): - # type: (Dict[unicode, Any], Callable[[Any, Any], Any], bool, **Any) -> Generator[WorkflowJob, None, None] + # type: (Dict[unicode, Any], Callable[[Any, Any], Any], **Any) -> Generator[WorkflowJob, None, None] self.state = {} self.processStatus = "success" @@ -324,11 +327,16 @@ def job(self, joborder, output_callback, **kwargs): made_progress = False for step in self.steps: + if kwargs["on_error"] == "stop" and self.processStatus != "success": + break + if not step.submitted: step.iterable = self.try_make_job(step, **kwargs) if step.iterable: for newjob in step.iterable: + if kwargs["on_error"] == "stop" and self.processStatus != "success": + break if newjob: made_progress = True yield newjob diff --git a/tests/echo.cwl b/tests/echo.cwl index da6943328..3333e57a8 100644 --- a/tests/echo.cwl +++ b/tests/echo.cwl @@ -1,4 +1,4 @@ -cwlVersion: cwl:draft-4.dev2 +cwlVersion: cwl:draft-4.dev3 class: CommandLineTool inputs: - id: inp diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index dfa108318..f21343b09 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -8,7 +8,7 @@ class ToolArgparse(unittest.TestCase): script=''' #!/usr/bin/env cwl-runner -cwlVersion: "draft-4.dev2" +cwlVersion: "draft-4.dev3" class: CommandLineTool description: "This tool is developed for SMC-RNA Challenge for detecting gene fusions (STAR fusion)" inputs: @@ -28,7 +28,7 @@ class ToolArgparse(unittest.TestCase): script2=''' #!/usr/bin/env cwl-runner -cwlVersion: 'cwl:draft-4.dev2' +cwlVersion: 'cwl:draft-4.dev3' class: CommandLineTool inputs: - id: bdg