Major refactor of document loading to validate draft-3 and draft-4 according to

the declared schemas.  This avoids the problem of naively updating to the
latest before validation, which doesn't catch invalid documents such as
(for example) use draft-4 features in documents declared draft-3.

Author: Peter Amstutz

cwltool/factory.py

@@ -24,7 +24,7 @@ def __init__(self, makeTool=workflow.defaultMakeTool,
         self.execkwargs = execkwargs
 
     def make(self, cwl, frag=None, debug=False):
-        l = main.load_tool(cwl, False, True, self.makeTool, debug, urifrag=frag)
+        l = main.load_tool(cwl, self.makeTool)
         if type(l) == int:
             raise Exception("Error loading tool")
         return Callable(l, self)

cwltool/load_tool.py

@@ -1,23 +1,17 @@
 import os
-from schema_salad.ref_resolver import Loader
-import schema_salad.validate as validate
+import logging
+import re
 import urlparse
 import sys
-import update
-import process
-
-def load_tool(argsworkflow, updateonly, strict, makeTool, debug,
-              print_pre=False,
-              print_rdf=False,
-              print_dot=False,
-              print_deps=False,
-              relative_deps=False,
-              rdf_serializer=None,
-              enable_dev=False,
-              stdout=sys.stdout,
-              urifrag=None):
-    # type: (Union[str,unicode,dict[unicode,Any]], bool, bool, Callable[...,Process], bool, bool, bool, bool, bool, bool, Any, Any, Any) -> Any
+from schema_salad.ref_resolver import Loader
+import schema_salad.validate as validate
+import schema_salad.schema as schema
+from . import update
+from . import process
+
+_logger = logging.getLogger("cwltool")
 
+def fetch_document(argsworkflow):
     document_loader = Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"})
 
     jobobj = None
@@ -33,87 +27,105 @@ def load_tool(argsworkflow, updateonly, strict, makeTool, debug,
         workflowobj = document_loader.fetch(fileuri)
     elif isinstance(argsworkflow, dict):
         workflowobj = argsworkflow
-        uri = urifrag
-        fileuri = "#"
+        uri = "#" + str(id(argsworkflow))
     else:
-        raise validate.ValidationException("Must be URI or dict")
+        raise validate.ValidationException("Must be URI or object: '%s'" % argsworkflow)
+
+    return document_loader, workflowobj, uri
 
+
+def validate_document(document_loader, workflowobj, uri, defaultVersion=None, enable_dev=False, strict=True):
+    jobobj = None
     if "cwl:tool" in workflowobj:
         jobobj = workflowobj
         uri = urlparse.urljoin(uri, jobobj["cwl:tool"])
-        fileuri, urifrag = urlparse.urldefrag(uri)
-        workflowobj = document_loader.fetch(fileuri)
         del jobobj["cwl:tool"]
+        workflowobj = fetch_document(uri)
 
-    if isinstance(workflowobj, list):
+    if not isinstance(workflowobj, dict):
         if enable_dev:
-            # bare list without a version must be treated as draft-2
-            workflowobj = {"cwlVersion": "https://w3id.org/cwl/cwl#draft-2",
-                           "id": fileuri,
-                           "@graph": workflowobj}
+            workflowobj = {
+                "cwlVersion": "draft-2",
+                "@graph": workflowobj
+            }
         else:
             raise validate.ValidationException("Missing 'cwlVersion'")
 
-    workflowobj = update.update(workflowobj, document_loader, fileuri, enable_dev)
-    document_loader.idx.clear()
+    fileuri, urifrag = urlparse.urldefrag(uri)
+
+    if "cwlVersion" in workflowobj:
+        workflowobj["cwlVersion"] = re.sub(r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"])
+    elif defaultVersion:
+        workflowobj["cwlVersion"] = defaultVersion
+    else:
+        raise validate.ValidationException("Missing 'cwlVersion'")
+
+    if workflowobj["cwlVersion"] == "draft-2" or ".dev" in workflowobj["cwlVersion"]:
+        # can't validate draft-2 directly, must run updater
+        workflowobj = update.update(workflowobj, document_loader, fileuri, enable_dev, {})
 
     (document_loader, avsc_names, schema_metadata) = process.get_schema(workflowobj["cwlVersion"])
 
     if isinstance(avsc_names, Exception):
         raise avsc_names
 
-    if updateonly:
-        stdout.write(json.dumps(workflowobj, indent=4))
-        return 0
+    workflowobj["id"] = fileuri
+    processobj, metadata = schema.load_and_validate(document_loader, avsc_names, workflowobj, strict)
+
+    if not metadata:
+        metadata = {"$namespaces": processobj.get("$namespaces", {}),
+                   "$schemas": processobj.get("$schemas", []),
+                   "cwlVersion": processobj["cwlVersion"]}
 
-    if print_deps:
-        printdeps(workflowobj, document_loader, stdout, relative_deps)
-        return 0
+    if metadata.get("cwlVersion") != update.latest:
+        processobj = update.update(processobj, document_loader, fileuri, enable_dev, metadata)
 
-    try:
-        processobj, metadata = schema_salad.schema.load_and_validate(document_loader, avsc_names, workflowobj, strict)
-    except (validate.ValidationException, RuntimeError) as e:
-        _logger.error(u"Tool definition failed validation:\n%s", e, exc_info=(e if debug else False))
-        return 1
+    if jobobj:
+        metadata["cwl:defaults"] = jobobj
 
-    if print_pre:
-        stdout.write(json.dumps(processobj, indent=4))
-        return 0
+    return document_loader, avsc_names, processobj, metadata, uri
 
-    if print_rdf:
-        printrdf(argsworkflow, processobj, document_loader.ctx, rdf_serializer, stdout)
-        return 0
 
-    if print_dot:
-        printdot(argsworkflow, processobj, document_loader.ctx, stdout)
-        return 0
+def make_tool(document_loader, avsc_names, processobj, metadata, uri, makeTool, kwargs):
+    processobj, _ = document_loader.resolve_ref(uri)
 
-    if urifrag:
-        processobj, _ = document_loader.resolve_ref(uri)
-    elif isinstance(processobj, list):
+    if isinstance(processobj, list):
         if 1 == len(processobj):
             processobj = processobj[0]
         else:
-            _logger.error(u"Tool file contains graph of multiple objects, must specify one of #%s",
-                          ", #".join(urlparse.urldefrag(i["id"])[1]
-                                     for i in processobj if "id" in i))
-            return 1
-
-    if not metadata:
-        metadata = {"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), 'cwlVersion': processobj["cwlVersion"]}
-
-    try:
-        t = makeTool(processobj, strict=strict, makeTool=makeTool, loader=document_loader, avsc_names=avsc_names, metadata=metadata)
-    except (validate.ValidationException) as e:
-        _logger.error(u"Tool definition failed validation:\n%s", e, exc_info=(e if debug else False))
-        return 1
-    except (RuntimeError, workflow.WorkflowException) as e:
-        _logger.error(u"Tool definition failed initialization:\n%s", e, exc_info=(e if debug else False))
-        return 1
-
-    if jobobj:
+            raise WorkflowException(u"Tool file contains graph of multiple objects, "
+                                   "must specify one of #%s" %
+                                   ", #".join(urlparse.urldefrag(i["id"])[1]
+                                              for i in processobj if "id" in i))
+
+    kwargs = kwargs.copy()
+    kwargs.update({
+        "makeTool": makeTool,
+        "loader": document_loader,
+        "avsc_names": avsc_names,
+        "metadata": metadata
+    })
+    t = makeTool(processobj, **kwargs)
+
+    if "cwl:defaults" in metadata:
+        jobobj = metadata["cwl:defaults"]
         for inp in t.tool["inputs"]:
             if shortname(inp["id"]) in jobobj:
                 inp["default"] = jobobj[shortname(inp["id"])]
 
     return t
+
+
+def load_tool(argsworkflow, makeTool, kwargs,
+              defaultVersion=None,
+              enable_dev=False,
+              strict=True):
+
+    document_loader, workflowobj, uri = fetch_document(argsworkflow)
+    document_loader, avsc_names, processobj, metadata, uri = validate_document(document_loader,
+                                                                               workflowobj,
+                                                                               uri,
+                                                                               defaultVersion=defaultVersion,
+                                                                               enable_dev=enable_dev,
+                                                                               strict=strict)
+    return make_tool(document_loader, avsc_names, processobj, metadata, uri, makeTool, kwargs)

cwltool/main.py

@@ -9,6 +9,7 @@
 import sys
 import logging
 from . import workflow
+from .errors import WorkflowException
 import schema_salad.validate as validate
 import tempfile
 import schema_salad.jsonld_context
@@ -22,7 +23,7 @@
 from . import update
 from .process import shortname, Process
 import rdflib
-from load_tool import load_tool
+from load_tool import load_tool, fetch_document, validate_document, make_tool
 import hashlib
 from .utils import aslist
 from typing import Union, Any, cast, Callable, Dict, Tuple, IO
@@ -116,7 +117,6 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--print-deps", action="store_true", help="Print CWL document dependencies.")
     exgroup.add_argument("--print-input-deps", action="store_true", help="Print input object document dependencies.")
     exgroup.add_argument("--version", action="store_true", help="Print version and exit")
-    exgroup.add_argument("--update", action="store_true", help="Update to latest CWL version, print and exit")
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--strict", action="store_true", help="Strict validation (unrecognized or out of place fields are error)",
@@ -134,6 +134,8 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--relative-deps", choices=['primary', 'cwd'], default="primary",
                          help="When using --print-deps, print paths relative to primary file or current working directory.")
 
+    parser.add_argument("--enable-dev", action="store_true", help="Allow loading and running development versions of CWL spec.", default=False)
+
     parser.add_argument("--enable-net", action="store_true",
             help="Use docker's default networking for containers; the default is "
             "to disable networking.")
@@ -188,15 +190,15 @@ def output_callback(out, processStatus):
                 if r:
                     r.run(**kwargs)
                 else:
-                    raise workflow.WorkflowException("Workflow cannot make any more progress.")
-        except workflow.WorkflowException:
+                    raise WorkflowException("Workflow cannot make any more progress.")
+        except WorkflowException:
             raise
         except Exception as e:
             _logger.exception("Got workflow error")
-            raise workflow.WorkflowException(unicode(e))
+            raise WorkflowException(unicode(e))
 
         if final_status[0] != "success":
-            raise workflow.WorkflowException(u"Process status is %s" % (final_status))
+            raise WorkflowException(u"Process status is %s" % (final_status))
 
         return final_output[0]
 
@@ -458,15 +460,37 @@ def main(argsl=None,
         return 1
 
     try:
-        t = load_tool(args.workflow, args.update, args.strict, makeTool, args.debug,
-                      print_pre=args.print_pre,
-                      print_rdf=args.print_rdf,
-                      print_dot=args.print_dot,
-                      print_deps=args.print_deps,
-                      relative_deps=args.relative_deps,
-                      rdf_serializer=args.rdf_serializer,
-                      enable_dev=args.enable_dev,
-                      stdout=stdout)
+        document_loader, workflowobj, uri = fetch_document(args.workflow)
+
+        if args.print_deps:
+            printdeps(workflowobj, document_loader, stdout, args.relative_deps)
+            return 0
+
+        document_loader, avsc_names, processobj, metadata, uri = validate_document(document_loader,
+                                                                                   workflowobj, uri,
+                                                                                   enable_dev=args.enable_dev,
+                                                                                   strict=args.strict)
+
+        if args.print_pre:
+            stdout.write(json.dumps(processobj, indent=4))
+            return 0
+
+        if args.print_rdf:
+            printrdf(uri, processobj, document_loader.ctx, args.rdf_serializer, stdout)
+            return 0
+
+        if args.print_dot:
+            printdot(uri, processobj, document_loader.ctx, stdout)
+            return 0
+
+        t = make_tool(document_loader, avsc_names, processobj,
+                      metadata, uri, makeTool, {})
+    except (validate.ValidationException) as e:
+        _logger.error(u"Tool definition failed validation:\n%s", e, exc_info=(e if args.debug else False))
+        return 1
+    except (RuntimeError, WorkflowException) as e:
+        _logger.error(u"Tool definition failed initialization:\n%s", e, exc_info=(e if args.debug else False))
+        return 1
     except Exception as e:
         _logger.error(u"I'm sorry, I couldn't load this CWL file, try again with --debug for more information.\n%s\n", e, exc_info=(e if args.debug else False))
         return 1
@@ -533,7 +557,7 @@ def main(argsl=None,
     except (validate.ValidationException) as e:
         _logger.error(u"Input object failed validation:\n%s", e, exc_info=(e if args.debug else False))
         return 1
-    except workflow.WorkflowException as e:
+    except WorkflowException as e:
         _logger.error(u"Workflow error, try again with --debug for more information:\n  %s", e, exc_info=(e if args.debug else False))
         return 1
     except Exception as e: