From bfe5fff787f4fa5662c45dcec44dd504b0fca153 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Thu, 3 Aug 2017 21:50:16 +0530 Subject: [PATCH 01/10] removing byte encoding from arguments passed to subprocess --- cwltool/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltool/job.py b/cwltool/job.py index 4e82b7e22..24566866a 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -207,7 +207,7 @@ def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"): os.makedirs(dn) stdout_path = absout - commands = [Text(x).encode('utf-8') for x in runtime + self.command_line] + commands = [Text(x) for x in (runtime + self.command_line)] job_script_contents = None # type: Text builder = getattr(self, "builder", None) # type: Builder if builder is not None: From d8d9b6647bf5dcdb89182c96842f989ccbf6dd5a Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Thu, 3 Aug 2017 22:07:23 +0530 Subject: [PATCH 02/10] addding SystemRoot variable to env in job.py --- cwltool/job.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cwltool/job.py b/cwltool/job.py index 24566866a..64bbc3540 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -297,6 +297,8 @@ def run(self, pull_image=True, rm_container=True, env["TMPDIR"] = str(self.tmpdir) if onWindows() else self.tmpdir if "PATH" not in env: env["PATH"] = str(os.environ["PATH"]) if onWindows() else os.environ["PATH"] + if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ: + env["SYSTEMROOT"] = str(os.environ["SYSTEMROOT"]) if onWindows() else os.environ["SYSTEMROOT"] stageFiles(self.pathmapper, ignoreWritable=True, symLink=True) if self.generatemapper: From 96b41963f8bbda3aaa27022b4b31df26da5c9721 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Wed, 2 Aug 2017 19:47:28 +0530 Subject: [PATCH 03/10] fix for python 3 testing which was falling back to python 2 --- appveyor.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b4cc1930a..b2f5ea595 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,16 +11,16 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python33_64" - PYTHON_VERSION: "3.3" + - PYTHON: "C:\\Python33-x64" + PYTHON_VERSION: "3.3.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python34_64" - PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python35_64" - PYTHON_VERSION: "3.5" + - PYTHON: "C:\\Python35-x64" + PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "64" From 5076bc001cdfeca87d693d3f5d0f11898e32bbc0 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Wed, 2 Aug 2017 20:20:37 +0530 Subject: [PATCH 04/10] remove fetching schema-salad from local git repo in appveyor.yml --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b2f5ea595..583e47e0c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,8 +28,6 @@ install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - "python --version" - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" - - "pip install --disable-pip-version-check --user --upgrade pip" - - "pip install --upgrade --no-deps --force-reinstall git+git://github.com/kapilkd13/schema_salad@windows#egg=schema_salad-2.4.201706261942" build_script: - "%CMD_IN_ENV% python setup.py install" From 03884dd6705bbb22689c890ab3fce54797bdb014 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Thu, 3 Aug 2017 22:18:26 +0530 Subject: [PATCH 05/10] fixing mypy errors --- cwltool/builder.py | 4 ++-- cwltool/job.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 20f67b258..82288bb7f 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -62,8 +62,8 @@ def __init__(self): # type: () -> None self.job_script_provider = None # type: Any def build_job_script(self, commands): - # type: (List[bytes]) -> Text - build_job_script_method = getattr(self.job_script_provider, "build_job_script", None) # type: Callable[[Builder, List[bytes]], Text] + # type: (List[str]) -> Text + build_job_script_method = getattr(self.job_script_provider, "build_job_script", None) # type: Callable[[Builder, List[str]], Text] if build_job_script_method: return build_job_script_method(self, commands) else: diff --git a/cwltool/job.py b/cwltool/job.py index 64bbc3540..0430c26ac 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -424,7 +424,7 @@ def run(self, pull_image=True, rm_container=True, def _job_popen( - commands, # type: List[bytes] + commands, # type: List[str] stdin_path, # type: Text stdout_path, # type: Text stderr_path, # type: Text From 369a8533449eb74464158bdd4578e78871622ed6 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Fri, 4 Aug 2017 02:22:51 +0530 Subject: [PATCH 06/10] setting Systemroot environment variable on Appveyor --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index 583e47e0c..e9e0d1380 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,7 @@ version: .{build}-{branch} environment: + SYSTEMROOT: "C:\\WINDOWS" matrix: - PYTHON: "C:\\Python27" From db47719b823177285e7acea16bdb47df6c1847f2 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Fri, 4 Aug 2017 15:13:12 +0530 Subject: [PATCH 07/10] decoding byte data before sending to Subprocess and encoding \n before comparing --- cwltool/sandboxjs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cwltool/sandboxjs.py b/cwltool/sandboxjs.py index fa0851958..d76691954 100644 --- a/cwltool/sandboxjs.py +++ b/cwltool/sandboxjs.py @@ -57,7 +57,8 @@ def new_js_proc(force_docker_pull=False): # type: (bool) -> subprocess.Popen res = resource_stream(__name__, 'cwlNodeEngine.js') - nodecode = res.read() + nodecode = res.read().decode('utf-8') + required_node_version, docker = (False,)*2 nodejs = None trynodes = ("nodejs", "node") @@ -238,7 +239,7 @@ def get_error(error_queue): no_more_error.release() output_thread.join() error_thread.join() - if stdout_buf.getvalue().endswith("\n"): + if stdout_buf.getvalue().endswith("\n".encode()): rselect = [] no_more_output.release() no_more_error.release() From 533b7381bf54bb644c8e6dab2769c8ceaecba858 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Thu, 10 Aug 2017 21:09:06 +0530 Subject: [PATCH 08/10] fixing mypy error --- cwltool/builder.py | 2 +- cwltool/job.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 82288bb7f..699bbf30d 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -62,7 +62,7 @@ def __init__(self): # type: () -> None self.job_script_provider = None # type: Any def build_job_script(self, commands): - # type: (List[str]) -> Text + # type: (Union[List[str],List[bytes]]) -> Text build_job_script_method = getattr(self.job_script_provider, "build_job_script", None) # type: Callable[[Builder, List[str]], Text] if build_job_script_method: return build_job_script_method(self, commands) diff --git a/cwltool/job.py b/cwltool/job.py index 0430c26ac..c6577866c 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -424,7 +424,7 @@ def run(self, pull_image=True, rm_container=True, def _job_popen( - commands, # type: List[str] + commands, # type: Union[List[str],List[bytes]] stdin_path, # type: Text stdout_path, # type: Text stderr_path, # type: Text From edfb44d52c7a4ea9997dbf1cb77f90b961329084 Mon Sep 17 00:00:00 2001 From: kapilkd13 Date: Fri, 11 Aug 2017 12:23:58 +0530 Subject: [PATCH 09/10] pip installing cwltool instead of setup.py inside appveyor --- appveyor.yml | 2 +- cwltool/builder.py | 4 ++-- cwltool/job.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e9e0d1380..c2479d99e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -31,7 +31,7 @@ install: - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" build_script: - - "%CMD_IN_ENV% python setup.py install" + - "%CMD_IN_ENV% pip install ." test_script: diff --git a/cwltool/builder.py b/cwltool/builder.py index 699bbf30d..8fe64c8e9 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -62,8 +62,8 @@ def __init__(self): # type: () -> None self.job_script_provider = None # type: Any def build_job_script(self, commands): - # type: (Union[List[str],List[bytes]]) -> Text - build_job_script_method = getattr(self.job_script_provider, "build_job_script", None) # type: Callable[[Builder, List[str]], Text] + # type: (Union[List[str],List[unicode]]) -> Text + build_job_script_method = getattr(self.job_script_provider, "build_job_script", None) # type: Callable[[Builder, Union[List[str],List[unicode]]], Text] if build_job_script_method: return build_job_script_method(self, commands) else: diff --git a/cwltool/job.py b/cwltool/job.py index c6577866c..3055cda84 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -424,7 +424,7 @@ def run(self, pull_image=True, rm_container=True, def _job_popen( - commands, # type: Union[List[str],List[bytes]] + commands, # type: Union[List[str],List[unicode]] stdin_path, # type: Text stdout_path, # type: Text stderr_path, # type: Text From 02f3c8564171a04c3abe4700a58150acd0fe70e9 Mon Sep 17 00:00:00 2001 From: Manvendra Singh Date: Fri, 11 Aug 2017 19:38:09 +0530 Subject: [PATCH 10/10] mypy: Use Text keyword instead of unicode --- cwltool/builder.py | 4 ++-- cwltool/job.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cwltool/builder.py b/cwltool/builder.py index 8fe64c8e9..ad9d58869 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -62,8 +62,8 @@ def __init__(self): # type: () -> None self.job_script_provider = None # type: Any def build_job_script(self, commands): - # type: (Union[List[str],List[unicode]]) -> Text - build_job_script_method = getattr(self.job_script_provider, "build_job_script", None) # type: Callable[[Builder, Union[List[str],List[unicode]]], Text] + # type: (List[Text]) -> Text + build_job_script_method = getattr(self.job_script_provider, "build_job_script", None) # type: Callable[[Builder, Union[List[str],List[Text]]], Text] if build_job_script_method: return build_job_script_method(self, commands) else: diff --git a/cwltool/job.py b/cwltool/job.py index 3055cda84..aee86732e 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -424,7 +424,7 @@ def run(self, pull_image=True, rm_container=True, def _job_popen( - commands, # type: Union[List[str],List[unicode]] + commands, # type: List[Text] stdin_path, # type: Text stdout_path, # type: Text stderr_path, # type: Text