From 8ce93e995afca38c00b3674fcf3affd65e99b276 Mon Sep 17 00:00:00 2001 From: vishal Date: Thu, 11 Jul 2019 14:48:36 +0000 Subject: [PATCH 1/3] Cortex namespace python files --- examples/pipelines/fraud/resources/apis.yaml | 3 +-- images/onnx-serve/Dockerfile | 16 ++++++++-------- images/python-packager/Dockerfile | 12 ++++++------ images/spark/Dockerfile | 10 +++++----- images/spark/run.sh | 2 +- images/test/Dockerfile | 2 +- images/test/run.sh | 10 +++------- images/tf-api/Dockerfile | 16 ++++++++-------- images/tf-train-gpu/Dockerfile | 12 ++++++------ images/tf-train/Dockerfile | 12 ++++++------ pkg/operator/workloads/data_job.go | 4 ++-- pkg/workloads/{ => cortex}/consts.py | 0 pkg/workloads/{ => cortex}/lib/__init__.py | 2 +- pkg/workloads/{ => cortex}/lib/context.py | 14 +++++++------- pkg/workloads/{ => cortex}/lib/exceptions.py | 0 pkg/workloads/{ => cortex}/lib/log.py | 0 pkg/workloads/{ => cortex}/lib/package.py | 8 ++++---- pkg/workloads/{ => cortex}/lib/requirements.txt | 0 pkg/workloads/{ => cortex}/lib/resources.py | 4 ++-- .../{ => cortex}/lib/storage/__init__.py | 0 pkg/workloads/{ => cortex}/lib/storage/local.py | 4 ++-- pkg/workloads/{ => cortex}/lib/storage/s3.py | 4 ++-- .../{ => cortex}/lib/test/resources_test.py | 2 +- pkg/workloads/{ => cortex}/lib/test/util_test.py | 2 +- pkg/workloads/{ => cortex}/lib/tf_lib.py | 5 +++-- pkg/workloads/{ => cortex}/lib/util.py | 5 ++--- pkg/workloads/{ => cortex}/onnx_serve/api.py | 10 +++++----- .../{ => cortex}/onnx_serve/requirements.txt | 0 pkg/workloads/{ => cortex}/spark_job/__init__.py | 0 .../{ => cortex}/spark_job/spark_job.py | 11 +++++------ .../{ => cortex}/spark_job/spark_util.py | 10 +++++----- .../{ => cortex}/spark_job/test/__init__.py | 0 .../{ => cortex}/spark_job/test/conftest.py | 7 ++++--- .../spark_job/test/integration/__init__.py | 0 .../test/integration/insurance_context.py | 6 +++--- .../spark_job/test/integration/insurance_test.py | 15 ++++++++------- .../{ => cortex}/spark_job/test/unit/__init__.py | 0 .../spark_job/test/unit/spark_util_test.py | 9 +++++---- pkg/workloads/{ => cortex}/tf_api/__init__.py | 0 pkg/workloads/{ => cortex}/tf_api/api.py | 10 +++++----- .../{ => cortex}/tf_api/requirements.txt | 0 pkg/workloads/{ => cortex}/tf_train/__init__.py | 0 pkg/workloads/{ => cortex}/tf_train/train.py | 6 +++--- .../{ => cortex}/tf_train/train_util.py | 6 +++--- 44 files changed, 118 insertions(+), 121 deletions(-) rename pkg/workloads/{ => cortex}/consts.py (100%) rename pkg/workloads/{ => cortex}/lib/__init__.py (93%) rename pkg/workloads/{ => cortex}/lib/context.py (99%) rename pkg/workloads/{ => cortex}/lib/exceptions.py (100%) rename pkg/workloads/{ => cortex}/lib/log.py (100%) rename pkg/workloads/{ => cortex}/lib/package.py (98%) rename pkg/workloads/{ => cortex}/lib/requirements.txt (100%) rename pkg/workloads/{ => cortex}/lib/resources.py (98%) rename pkg/workloads/{ => cortex}/lib/storage/__init__.py (100%) rename pkg/workloads/{ => cortex}/lib/storage/local.py (97%) rename pkg/workloads/{ => cortex}/lib/storage/s3.py (98%) rename pkg/workloads/{ => cortex}/lib/test/resources_test.py (97%) rename pkg/workloads/{ => cortex}/lib/test/util_test.py (99%) rename pkg/workloads/{ => cortex}/lib/tf_lib.py (98%) rename pkg/workloads/{ => cortex}/lib/util.py (99%) rename pkg/workloads/{ => cortex}/onnx_serve/api.py (97%) rename pkg/workloads/{ => cortex}/onnx_serve/requirements.txt (100%) rename pkg/workloads/{ => cortex}/spark_job/__init__.py (100%) rename pkg/workloads/{ => cortex}/spark_job/spark_job.py (98%) rename pkg/workloads/{ => cortex}/spark_job/spark_util.py (99%) rename pkg/workloads/{ => cortex}/spark_job/test/__init__.py (100%) rename pkg/workloads/{ => cortex}/spark_job/test/conftest.py (98%) rename pkg/workloads/{ => cortex}/spark_job/test/integration/__init__.py (100%) rename pkg/workloads/{ => cortex}/spark_job/test/integration/insurance_context.py (99%) rename pkg/workloads/{ => cortex}/spark_job/test/integration/insurance_test.py (95%) rename pkg/workloads/{ => cortex}/spark_job/test/unit/__init__.py (100%) rename pkg/workloads/{ => cortex}/spark_job/test/unit/spark_util_test.py (99%) rename pkg/workloads/{ => cortex}/tf_api/__init__.py (100%) rename pkg/workloads/{ => cortex}/tf_api/api.py (98%) rename pkg/workloads/{ => cortex}/tf_api/requirements.txt (100%) rename pkg/workloads/{ => cortex}/tf_train/__init__.py (100%) rename pkg/workloads/{ => cortex}/tf_train/train.py (95%) rename pkg/workloads/{ => cortex}/tf_train/train_util.py (98%) diff --git a/examples/pipelines/fraud/resources/apis.yaml b/examples/pipelines/fraud/resources/apis.yaml index 6c67398ae1..5973bf5bd3 100644 --- a/examples/pipelines/fraud/resources/apis.yaml +++ b/examples/pipelines/fraud/resources/apis.yaml @@ -1,5 +1,4 @@ - kind: api name: fraud model: @dnn - compute: - replicas: 1 + model_format: tensorflow diff --git a/images/onnx-serve/Dockerfile b/images/onnx-serve/Dockerfile index 4acabf1614..ea66059698 100644 --- a/images/onnx-serve/Dockerfile +++ b/images/onnx-serve/Dockerfile @@ -26,14 +26,14 @@ RUN apt-get update -qq && apt-get install -y -q \ ENV PYTHONPATH="/src:${PYTHONPATH}" -COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt -COPY pkg/workloads/onnx_serve/requirements.txt /src/onnx_serve/requirements.txt -RUN pip3 install -r /src/lib/requirements.txt && \ - pip3 install -r /src/onnx_serve/requirements.txt && \ +COPY pkg/workloads/cortex/lib/requirements.txt /src/cortex/lib/requirements.txt +COPY pkg/workloads/cortex/onnx_serve/requirements.txt /src/cortex/onnx_serve/requirements.txt +RUN pip3 install -r /src/cortex/lib/requirements.txt && \ + pip3 install -r /src/cortex/onnx_serve/requirements.txt && \ rm -rf /root/.cache/pip* -COPY pkg/workloads/consts.py /src/ -COPY pkg/workloads/lib /src/lib -COPY pkg/workloads/onnx_serve /src/onnx_serve +COPY pkg/workloads/cortex/consts.py /src/cortex +COPY pkg/workloads/cortex/lib /src/cortex/lib +COPY pkg/workloads/cortex/onnx_serve /src/cortex/onnx_serve -ENTRYPOINT ["/usr/bin/python3", "/src/onnx_serve/api.py"] +ENTRYPOINT ["/usr/bin/python3", "/src/cortex/onnx_serve/api.py"] diff --git a/images/python-packager/Dockerfile b/images/python-packager/Dockerfile index c013718f3e..e59aba113d 100644 --- a/images/python-packager/Dockerfile +++ b/images/python-packager/Dockerfile @@ -23,15 +23,15 @@ RUN apt-get update -qq && apt-get install -y -q \ zlib1g-dev \ && apt-get clean -qq && rm -rf /var/lib/apt/lists/* -COPY pkg/workloads/consts.py /src/ -COPY pkg/workloads/lib /src/lib +COPY pkg/workloads/cortex/consts.py /src/cortex/ +COPY pkg/workloads/cortex/lib /src/cortex/lib -COPY pkg/workloads/tf_api/requirements.txt /src/tf_api/requirements.txt +COPY pkg/workloads/cortex/tf_api/requirements.txt /src/cortex/tf_api/requirements.txt -RUN pip3 install -r /src/lib/requirements.txt && \ - pip3 install -r /src/tf_api/requirements.txt && \ +RUN pip3 install -r /src/cortex/lib/requirements.txt && \ + pip3 install -r /src/cortex/tf_api/requirements.txt && \ rm -rf /root/.cache/pip* ENV PYTHONPATH="/src:${PYTHONPATH}" -ENTRYPOINT ["/usr/bin/python3", "/src/lib/package.py"] +ENTRYPOINT ["/usr/bin/python3", "/src/cortex/lib/package.py"] diff --git a/images/spark/Dockerfile b/images/spark/Dockerfile index 57f5d936fa..bf30e3a2cc 100644 --- a/images/spark/Dockerfile +++ b/images/spark/Dockerfile @@ -35,13 +35,13 @@ RUN sed -i "/^set -ex$/c\set -e" /opt/entrypoint.sh # Our code ENV PYTHONPATH="/src:${PYTHONPATH}" -COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt -RUN pip3 install -r /src/lib/requirements.txt && \ +COPY pkg/workloads/cortex/lib/requirements.txt /src/cortex/lib/requirements.txt +RUN pip3 install -r /src/cortex/lib/requirements.txt && \ rm -rf /root/.cache/pip* -COPY pkg/workloads/consts.py /src/ -COPY pkg/workloads/lib /src/lib -COPY pkg/workloads/spark_job /src/spark_job +COPY pkg/workloads/cortex/consts.py /src/cortex/ +COPY pkg/workloads/cortex/lib /src/cortex/lib +COPY pkg/workloads/cortex/spark_job /src/cortex/spark_job # $SPARK_HOME/conf gets clobbered by a volume that spark-on-k8s mounts (KubernetesClientApplication.scala) RUN cp -r $SPARK_HOME/conf $SPARK_HOME/conf-custom diff --git a/images/spark/run.sh b/images/spark/run.sh index 90a165c46c..311f7dafc7 100644 --- a/images/spark/run.sh +++ b/images/spark/run.sh @@ -37,7 +37,7 @@ echo "" echo "Starting" echo "" -/usr/bin/python3 /src/lib/package.py --workload-id=$CORTEX_WORKLOAD_ID --context=$CORTEX_CONTEXT_S3_PATH --cache-dir=$CORTEX_CACHE_DIR +/usr/bin/python3 /src/cortex/lib/package.py --workload-id=$CORTEX_WORKLOAD_ID --context=$CORTEX_CONTEXT_S3_PATH --cache-dir=$CORTEX_CACHE_DIR # Run the intended command /opt/entrypoint.sh "$@" diff --git a/images/test/Dockerfile b/images/test/Dockerfile index fff92acae7..206fafffd6 100644 --- a/images/test/Dockerfile +++ b/images/test/Dockerfile @@ -9,7 +9,7 @@ COPY pkg/estimators /estimators COPY images/test/run.sh /src/run.sh -WORKDIR /src +WORKDIR /src/cortex ENTRYPOINT ["/bin/bash"] CMD ["/src/run.sh"] diff --git a/images/test/run.sh b/images/test/run.sh index b527e98c34..176a2fe42d 100644 --- a/images/test/run.sh +++ b/images/test/run.sh @@ -18,13 +18,9 @@ err=0 trap 'err=1' ERR -cd lib -pytest -cd .. +pytest lib/test -cd spark_job -pytest test/unit -pytest test/integration -cd .. +pytest spark_job/test/unit +pytest spark_job/test/integration test $err = 0 diff --git a/images/tf-api/Dockerfile b/images/tf-api/Dockerfile index 186b9353fb..7ccde1f514 100644 --- a/images/tf-api/Dockerfile +++ b/images/tf-api/Dockerfile @@ -2,14 +2,14 @@ FROM cortexlabs/tf-base ENV PYTHONPATH="/src:${PYTHONPATH}" -COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt -COPY pkg/workloads/tf_api/requirements.txt /src/tf_api/requirements.txt -RUN pip3 install -r /src/lib/requirements.txt && \ - pip3 install -r /src/tf_api/requirements.txt && \ +COPY pkg/workloads/cortex/lib/requirements.txt /src/cortex/lib/requirements.txt +COPY pkg/workloads/cortex/tf_api/requirements.txt /src/cortex/tf_api/requirements.txt +RUN pip3 install -r /src/cortex/lib/requirements.txt && \ + pip3 install -r /src/cortex/tf_api/requirements.txt && \ rm -rf /root/.cache/pip* -COPY pkg/workloads/consts.py /src/ -COPY pkg/workloads/lib /src/lib -COPY pkg/workloads/tf_api /src/tf_api +COPY pkg/workloads/cortex/consts.py /src/cortex/ +COPY pkg/workloads/cortex/lib /src/cortex/lib +COPY pkg/workloads/cortex/tf_api /src/cortex/tf_api -ENTRYPOINT ["/usr/bin/python3", "/src/tf_api/api.py"] +ENTRYPOINT ["/usr/bin/python3", "/src/cortex/tf_api/api.py"] diff --git a/images/tf-train-gpu/Dockerfile b/images/tf-train-gpu/Dockerfile index c9bbd2d840..0cf4717d96 100644 --- a/images/tf-train-gpu/Dockerfile +++ b/images/tf-train-gpu/Dockerfile @@ -2,12 +2,12 @@ FROM cortexlabs/tf-base-gpu ENV PYTHONPATH="/src:${PYTHONPATH}" -COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt -RUN pip3 install -r /src/lib/requirements.txt && \ +COPY pkg/workloads/cortex/lib/requirements.txt /src/cortex/lib/requirements.txt +RUN pip3 install -r /src/cortex/lib/requirements.txt && \ rm -rf /root/.cache/pip* -COPY pkg/workloads/consts.py /src/ -COPY pkg/workloads/lib /src/lib -COPY pkg/workloads/tf_train /src/tf_train +COPY pkg/workloads/cortex/consts.py /src/cortex/ +COPY pkg/workloads/cortex/lib /src/cortex/lib +COPY pkg/workloads/cortex/tf_train /src/cortex/tf_train -ENTRYPOINT ["/usr/bin/python3", "/src/tf_train/train.py"] +ENTRYPOINT ["/usr/bin/python3", "/src/cortex/tf_train/train.py"] diff --git a/images/tf-train/Dockerfile b/images/tf-train/Dockerfile index c926ccc651..7c98de7e69 100644 --- a/images/tf-train/Dockerfile +++ b/images/tf-train/Dockerfile @@ -2,12 +2,12 @@ FROM cortexlabs/tf-base ENV PYTHONPATH="/src:${PYTHONPATH}" -COPY pkg/workloads/lib/requirements.txt /src/lib/requirements.txt -RUN pip3 install -r /src/lib/requirements.txt && \ +COPY pkg/workloads/cortex/lib/requirements.txt /src/cortex/lib/requirements.txt +RUN pip3 install -r /src/cortex/lib/requirements.txt && \ rm -rf /root/.cache/pip* -COPY pkg/workloads/consts.py /src/ -COPY pkg/workloads/lib /src/lib -COPY pkg/workloads/tf_train /src/tf_train +COPY pkg/workloads/cortex/consts.py /src/cortex/ +COPY pkg/workloads/cortex/lib /src/cortex/lib +COPY pkg/workloads/cortex/tf_train /src/cortex/tf_train -ENTRYPOINT ["/usr/bin/python3", "/src/tf_train/train.py"] +ENTRYPOINT ["/usr/bin/python3", "/src/cortex/tf_train/train.py"] diff --git a/pkg/operator/workloads/data_job.go b/pkg/operator/workloads/data_job.go index 71bb71133d..0e119ed122 100644 --- a/pkg/operator/workloads/data_job.go +++ b/pkg/operator/workloads/data_job.go @@ -95,7 +95,7 @@ func sparkSpec(workloadID string, ctx *context.Context, workloadType string, spa Mode: sparkop.ClusterMode, Image: &config.Cortex.SparkImage, ImagePullPolicy: pointer.String("Always"), - MainApplicationFile: pointer.String("local:///src/spark_job/spark_job.py"), + MainApplicationFile: pointer.String("local:///src/cortex/spark_job/spark_job.py"), RestartPolicy: sparkop.RestartPolicy{Type: sparkop.Never}, MemoryOverheadFactor: memOverheadFactor, Arguments: []string{ @@ -106,7 +106,7 @@ func sparkSpec(workloadID string, ctx *context.Context, workloadType string, spa " " + strings.Join(args, " ")), }, Deps: sparkop.Dependencies{ - PyFiles: []string{"local:///src/spark_job/spark_util.py", "local:///src/lib/*.py"}, + PyFiles: []string{"local:///src/cortex/spark_job/spark_util.py", "local:///src/cortex/lib/*.py"}, }, Driver: sparkop.DriverSpec{ SparkPodSpec: sparkop.SparkPodSpec{ diff --git a/pkg/workloads/consts.py b/pkg/workloads/cortex/consts.py similarity index 100% rename from pkg/workloads/consts.py rename to pkg/workloads/cortex/consts.py diff --git a/pkg/workloads/lib/__init__.py b/pkg/workloads/cortex/lib/__init__.py similarity index 93% rename from pkg/workloads/lib/__init__.py rename to pkg/workloads/cortex/lib/__init__.py index ad9380458d..eb2d7338e3 100644 --- a/pkg/workloads/lib/__init__.py +++ b/pkg/workloads/cortex/lib/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .context import Context +from cortex.lib.context import Context diff --git a/pkg/workloads/lib/context.py b/pkg/workloads/cortex/lib/context.py similarity index 99% rename from pkg/workloads/lib/context.py rename to pkg/workloads/cortex/lib/context.py index f2e81a815a..7e31b5955a 100644 --- a/pkg/workloads/lib/context.py +++ b/pkg/workloads/cortex/lib/context.py @@ -19,14 +19,14 @@ import importlib from datetime import datetime from copy import deepcopy - -import consts -from lib import util -from lib.storage import S3, LocalStorage -from lib.exceptions import CortexException, UserException from botocore.exceptions import ClientError -from lib.resources import ResourceMap -from lib.log import get_logger + +from cortex import consts +from cortex.lib import util +from cortex.lib.storage import S3, LocalStorage +from cortex.lib.exceptions import CortexException, UserException +from cortex.lib.resources import ResourceMap +from cortex.lib.log import get_logger logger = get_logger() diff --git a/pkg/workloads/lib/exceptions.py b/pkg/workloads/cortex/lib/exceptions.py similarity index 100% rename from pkg/workloads/lib/exceptions.py rename to pkg/workloads/cortex/lib/exceptions.py diff --git a/pkg/workloads/lib/log.py b/pkg/workloads/cortex/lib/log.py similarity index 100% rename from pkg/workloads/lib/log.py rename to pkg/workloads/cortex/lib/log.py diff --git a/pkg/workloads/lib/package.py b/pkg/workloads/cortex/lib/package.py similarity index 98% rename from pkg/workloads/lib/package.py rename to pkg/workloads/cortex/lib/package.py index f32e8d3996..208b068f76 100644 --- a/pkg/workloads/lib/package.py +++ b/pkg/workloads/cortex/lib/package.py @@ -18,13 +18,13 @@ import glob from subprocess import run -from lib import util, Context -from lib.log import get_logger -from lib.exceptions import UserException, CortexException - import requirements from packaging.requirements import Requirement +from cortex.lib import util, Context +from cortex.lib.log import get_logger +from cortex.lib.exceptions import UserException, CortexException + logger = get_logger() LOCAL_PACKAGE_PATH = "/packages" diff --git a/pkg/workloads/lib/requirements.txt b/pkg/workloads/cortex/lib/requirements.txt similarity index 100% rename from pkg/workloads/lib/requirements.txt rename to pkg/workloads/cortex/lib/requirements.txt diff --git a/pkg/workloads/lib/resources.py b/pkg/workloads/cortex/lib/resources.py similarity index 98% rename from pkg/workloads/lib/resources.py rename to pkg/workloads/cortex/lib/resources.py index 69524b7f2b..717e25e334 100644 --- a/pkg/workloads/lib/resources.py +++ b/pkg/workloads/cortex/lib/resources.py @@ -13,10 +13,10 @@ # limitations under the License. import collections - -from lib import util from copy import deepcopy +from cortex.lib import util + class ResourceMap(dict): def __init__(self, resource_name_map): diff --git a/pkg/workloads/lib/storage/__init__.py b/pkg/workloads/cortex/lib/storage/__init__.py similarity index 100% rename from pkg/workloads/lib/storage/__init__.py rename to pkg/workloads/cortex/lib/storage/__init__.py diff --git a/pkg/workloads/lib/storage/local.py b/pkg/workloads/cortex/lib/storage/local.py similarity index 97% rename from pkg/workloads/lib/storage/local.py rename to pkg/workloads/cortex/lib/storage/local.py index 7ec974ebe4..89535e2d1b 100644 --- a/pkg/workloads/lib/storage/local.py +++ b/pkg/workloads/cortex/lib/storage/local.py @@ -22,8 +22,8 @@ from pathlib import Path import shutil -from lib import util -from lib.exceptions import CortexException +from cortex.lib import util +from cortex.lib.exceptions import CortexException class LocalStorage(object): diff --git a/pkg/workloads/lib/storage/s3.py b/pkg/workloads/cortex/lib/storage/s3.py similarity index 98% rename from pkg/workloads/lib/storage/s3.py rename to pkg/workloads/cortex/lib/storage/s3.py index 06cc48ccf9..9ee19a878d 100644 --- a/pkg/workloads/lib/storage/s3.py +++ b/pkg/workloads/cortex/lib/storage/s3.py @@ -20,8 +20,8 @@ import json import msgpack -from lib import util -from lib.exceptions import CortexException +from cortex.lib import util +from cortex.lib.exceptions import CortexException class S3(object): diff --git a/pkg/workloads/lib/test/resources_test.py b/pkg/workloads/cortex/lib/test/resources_test.py similarity index 97% rename from pkg/workloads/lib/test/resources_test.py rename to pkg/workloads/cortex/lib/test/resources_test.py index c26db25f38..05231ab2ba 100644 --- a/pkg/workloads/lib/test/resources_test.py +++ b/pkg/workloads/cortex/lib/test/resources_test.py @@ -14,7 +14,7 @@ import pytest -from lib.resources import ResourceMap +from cortex.lib.resources import ResourceMap def test_resource_map_empty_resource(): diff --git a/pkg/workloads/lib/test/util_test.py b/pkg/workloads/cortex/lib/test/util_test.py similarity index 99% rename from pkg/workloads/lib/test/util_test.py rename to pkg/workloads/cortex/lib/test/util_test.py index ef2ed60d73..60c003f962 100644 --- a/pkg/workloads/lib/test/util_test.py +++ b/pkg/workloads/cortex/lib/test/util_test.py @@ -17,7 +17,7 @@ from copy import deepcopy import pytest -from lib import util +from cortex.lib import util import logging diff --git a/pkg/workloads/lib/tf_lib.py b/pkg/workloads/cortex/lib/tf_lib.py similarity index 98% rename from pkg/workloads/lib/tf_lib.py rename to pkg/workloads/cortex/lib/tf_lib.py index 86eb2b4791..f325ee21c5 100644 --- a/pkg/workloads/lib/tf_lib.py +++ b/pkg/workloads/cortex/lib/tf_lib.py @@ -14,10 +14,11 @@ import os import sys + import tensorflow as tf -from lib import util -import consts +from cortex.lib import util +from cortex import consts CORTEX_TYPE_TO_TF_TYPE = { diff --git a/pkg/workloads/lib/util.py b/pkg/workloads/cortex/lib/util.py similarity index 99% rename from pkg/workloads/lib/util.py rename to pkg/workloads/cortex/lib/util.py index b4b8f3094d..09a16d0c0e 100644 --- a/pkg/workloads/lib/util.py +++ b/pkg/workloads/cortex/lib/util.py @@ -29,9 +29,8 @@ from copy import deepcopy from datetime import datetime -import consts - -from lib.log import get_logger +from cortex import consts +from cortex.lib.log import get_logger logger = get_logger() diff --git a/pkg/workloads/onnx_serve/api.py b/pkg/workloads/cortex/onnx_serve/api.py similarity index 97% rename from pkg/workloads/onnx_serve/api.py rename to pkg/workloads/cortex/onnx_serve/api.py index 34f9831d71..de2267a153 100644 --- a/pkg/workloads/onnx_serve/api.py +++ b/pkg/workloads/cortex/onnx_serve/api.py @@ -22,13 +22,13 @@ from flask_api import status from waitress import serve import onnxruntime as rt -from lib.storage import S3 import numpy as np -import consts -from lib import util, package, Context -from lib.log import get_logger -from lib.exceptions import CortexException, UserRuntimeException, UserException +from cortex.lib.storage import S3 +from cortex import consts +from cortex.lib import util, package, Context +from cortex.lib.log import get_logger +from cortex.lib.exceptions import CortexException, UserRuntimeException, UserException logger = get_logger() logger.propagate = False # prevent double logging (flask modifies root logger) diff --git a/pkg/workloads/onnx_serve/requirements.txt b/pkg/workloads/cortex/onnx_serve/requirements.txt similarity index 100% rename from pkg/workloads/onnx_serve/requirements.txt rename to pkg/workloads/cortex/onnx_serve/requirements.txt diff --git a/pkg/workloads/spark_job/__init__.py b/pkg/workloads/cortex/spark_job/__init__.py similarity index 100% rename from pkg/workloads/spark_job/__init__.py rename to pkg/workloads/cortex/spark_job/__init__.py diff --git a/pkg/workloads/spark_job/spark_job.py b/pkg/workloads/cortex/spark_job/spark_job.py similarity index 98% rename from pkg/workloads/spark_job/spark_job.py rename to pkg/workloads/cortex/spark_job/spark_job.py index ea0ed60345..4da150014c 100644 --- a/pkg/workloads/spark_job/spark_job.py +++ b/pkg/workloads/cortex/spark_job/spark_job.py @@ -19,14 +19,13 @@ import traceback from pyspark.sql import SparkSession - -from lib import util, Context -from lib.log import get_logger -from lib.exceptions import UserException, CortexException, UserRuntimeException -import spark_util import pyspark.sql.functions as F -import consts +from cortex import consts +from cortex.lib import util, Context +from cortex.lib.log import get_logger +from cortex.lib.exceptions import UserException, CortexException, UserRuntimeException +from cortex.spark_job import spark_util logger = get_logger() diff --git a/pkg/workloads/spark_job/spark_util.py b/pkg/workloads/cortex/spark_job/spark_util.py similarity index 99% rename from pkg/workloads/spark_job/spark_util.py rename to pkg/workloads/cortex/spark_job/spark_util.py index 04c3cdbaae..21ad8892ef 100644 --- a/pkg/workloads/spark_job/spark_util.py +++ b/pkg/workloads/cortex/spark_job/spark_util.py @@ -20,11 +20,11 @@ from pyspark.sql.dataframe import DataFrame import pyspark.sql.functions as F -from lib import util -from lib.context import create_transformer_inputs_from_map, create_transformer_inputs_from_lists -from lib.exceptions import CortexException, UserException, UserRuntimeException -from lib.log import get_logger -import consts +from cortex.lib import util +from cortex.lib.context import create_transformer_inputs_from_map, create_transformer_inputs_from_lists +from cortex.lib.exceptions import CortexException, UserException, UserRuntimeException +from cortex.lib.log import get_logger +from cortex import consts logger = get_logger() diff --git a/pkg/workloads/spark_job/test/__init__.py b/pkg/workloads/cortex/spark_job/test/__init__.py similarity index 100% rename from pkg/workloads/spark_job/test/__init__.py rename to pkg/workloads/cortex/spark_job/test/__init__.py diff --git a/pkg/workloads/spark_job/test/conftest.py b/pkg/workloads/cortex/spark_job/test/conftest.py similarity index 98% rename from pkg/workloads/spark_job/test/conftest.py rename to pkg/workloads/cortex/spark_job/test/conftest.py index 7f1a274739..efa859b812 100644 --- a/pkg/workloads/spark_job/test/conftest.py +++ b/pkg/workloads/cortex/spark_job/test/conftest.py @@ -19,12 +19,13 @@ import pytest import uuid import os +import shutil from pyspark import SparkConf from pyspark.sql import SparkSession -from lib import Context -import consts -import shutil + +from cortex.lib import Context +from cortex import consts def quiet_py4j(): diff --git a/pkg/workloads/spark_job/test/integration/__init__.py b/pkg/workloads/cortex/spark_job/test/integration/__init__.py similarity index 100% rename from pkg/workloads/spark_job/test/integration/__init__.py rename to pkg/workloads/cortex/spark_job/test/integration/__init__.py diff --git a/pkg/workloads/spark_job/test/integration/insurance_context.py b/pkg/workloads/cortex/spark_job/test/integration/insurance_context.py similarity index 99% rename from pkg/workloads/spark_job/test/integration/insurance_context.py rename to pkg/workloads/cortex/spark_job/test/integration/insurance_context.py index 779e8b50ea..f9c2021ff3 100644 --- a/pkg/workloads/spark_job/test/integration/insurance_context.py +++ b/pkg/workloads/cortex/spark_job/test/integration/insurance_context.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import consts +from cortex import consts """ HOW TO GENERATE CONTEXT @@ -23,8 +23,8 @@ docker run -it --entrypoint "/bin/bash" cortexlabs/spark 4. run the following in python3 shell -from lib import util -from lib.storage import S3 +from cortex.lib import util +from cortex.lib.storage import S3 bucket = "cortex-cluster-" key = "apps//contexts/.msgpack" diff --git a/pkg/workloads/spark_job/test/integration/insurance_test.py b/pkg/workloads/cortex/spark_job/test/integration/insurance_test.py similarity index 95% rename from pkg/workloads/spark_job/test/integration/insurance_test.py rename to pkg/workloads/cortex/spark_job/test/integration/insurance_test.py index 50bea0d0a3..975a5a0e40 100644 --- a/pkg/workloads/spark_job/test/integration/insurance_test.py +++ b/pkg/workloads/cortex/spark_job/test/integration/insurance_test.py @@ -12,12 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import math - -import spark_util -from spark_job import spark_job -from lib.exceptions import UserException -from lib import Context -from test.integration import insurance_context +import os import pytest from pyspark.sql.types import * @@ -26,7 +21,13 @@ from mock import MagicMock, call from py4j.protocol import Py4JJavaError from pathlib import Path -import os + +from cortex.spark_job import spark_util +from cortex.spark_job import spark_job +from cortex.lib.exceptions import UserException +from cortex.lib import Context +from cortex.spark_job.test.integration import insurance_context + pytestmark = pytest.mark.usefixtures("spark") diff --git a/pkg/workloads/spark_job/test/unit/__init__.py b/pkg/workloads/cortex/spark_job/test/unit/__init__.py similarity index 100% rename from pkg/workloads/spark_job/test/unit/__init__.py rename to pkg/workloads/cortex/spark_job/test/unit/__init__.py diff --git a/pkg/workloads/spark_job/test/unit/spark_util_test.py b/pkg/workloads/cortex/spark_job/test/unit/spark_util_test.py similarity index 99% rename from pkg/workloads/spark_job/test/unit/spark_util_test.py rename to pkg/workloads/cortex/spark_job/test/unit/spark_util_test.py index dae0610065..b94a3ea49d 100644 --- a/pkg/workloads/spark_job/test/unit/spark_util_test.py +++ b/pkg/workloads/cortex/spark_job/test/unit/spark_util_test.py @@ -13,10 +13,6 @@ # limitations under the License. import math -import spark_util -import consts -from lib.exceptions import UserException -from lib import util import pytest from pyspark.sql.types import * from pyspark.sql import Row @@ -25,6 +21,11 @@ from py4j.protocol import Py4JJavaError from datetime import datetime +from cortex import consts +from cortex.lib.exceptions import UserException +from cortex.lib import util +from cortex.spark_job import spark_util + pytestmark = pytest.mark.usefixtures("spark") diff --git a/pkg/workloads/tf_api/__init__.py b/pkg/workloads/cortex/tf_api/__init__.py similarity index 100% rename from pkg/workloads/tf_api/__init__.py rename to pkg/workloads/cortex/tf_api/__init__.py diff --git a/pkg/workloads/tf_api/api.py b/pkg/workloads/cortex/tf_api/api.py similarity index 98% rename from pkg/workloads/tf_api/api.py rename to pkg/workloads/cortex/tf_api/api.py index e2e3e4f8fd..f6a56eeea2 100644 --- a/pkg/workloads/tf_api/api.py +++ b/pkg/workloads/cortex/tf_api/api.py @@ -28,11 +28,11 @@ from tensorflow_serving.apis import prediction_service_pb2_grpc from google.protobuf import json_format -import consts -from lib import util, tf_lib, package, Context -from lib.log import get_logger -from lib.exceptions import CortexException, UserRuntimeException, UserException -from lib.context import create_transformer_inputs_from_map +from cortex import consts +from cortex.lib import util, tf_lib, package, Context +from cortex.lib.log import get_logger +from cortex.lib.exceptions import CortexException, UserRuntimeException, UserException +from cortex.lib.context import create_transformer_inputs_from_map logger = get_logger() logger.propagate = False # prevent double logging (flask modifies root logger) diff --git a/pkg/workloads/tf_api/requirements.txt b/pkg/workloads/cortex/tf_api/requirements.txt similarity index 100% rename from pkg/workloads/tf_api/requirements.txt rename to pkg/workloads/cortex/tf_api/requirements.txt diff --git a/pkg/workloads/tf_train/__init__.py b/pkg/workloads/cortex/tf_train/__init__.py similarity index 100% rename from pkg/workloads/tf_train/__init__.py rename to pkg/workloads/cortex/tf_train/__init__.py diff --git a/pkg/workloads/tf_train/train.py b/pkg/workloads/cortex/tf_train/train.py similarity index 95% rename from pkg/workloads/tf_train/train.py rename to pkg/workloads/cortex/tf_train/train.py index f812a2fe9f..4bbd2bf1e0 100644 --- a/pkg/workloads/tf_train/train.py +++ b/pkg/workloads/cortex/tf_train/train.py @@ -18,11 +18,11 @@ import traceback import tensorflow as tf -from lib import util, package, Context -from lib.exceptions import CortexException, UserRuntimeException +from cortex.lib import util, package, Context +from cortex.lib.exceptions import CortexException, UserRuntimeException import train_util -from lib.log import get_logger +from cortex.lib.log import get_logger logger = get_logger() diff --git a/pkg/workloads/tf_train/train_util.py b/pkg/workloads/cortex/tf_train/train_util.py similarity index 98% rename from pkg/workloads/tf_train/train_util.py rename to pkg/workloads/cortex/tf_train/train_util.py index ed44b7c8fb..f64e569b6c 100644 --- a/pkg/workloads/tf_train/train_util.py +++ b/pkg/workloads/cortex/tf_train/train_util.py @@ -20,9 +20,9 @@ import math import tensorflow as tf -import consts -from lib import util, tf_lib -from lib.exceptions import UserRuntimeException +from cortex import consts +from cortex.lib import util, tf_lib +from cortex.lib.exceptions import UserRuntimeException def get_input_placeholder(model_name, ctx, training=True): From 69783552d1934548605822a91090743adf1d5447 Mon Sep 17 00:00:00 2001 From: vishal Date: Thu, 11 Jul 2019 14:56:29 +0000 Subject: [PATCH 2/3] Fix lint error --- pkg/workloads/cortex/spark_job/spark_util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/workloads/cortex/spark_job/spark_util.py b/pkg/workloads/cortex/spark_job/spark_util.py index 21ad8892ef..ac0db7ae19 100644 --- a/pkg/workloads/cortex/spark_job/spark_util.py +++ b/pkg/workloads/cortex/spark_job/spark_util.py @@ -21,7 +21,10 @@ import pyspark.sql.functions as F from cortex.lib import util -from cortex.lib.context import create_transformer_inputs_from_map, create_transformer_inputs_from_lists +from cortex.lib.context import ( + create_transformer_inputs_from_map, + create_transformer_inputs_from_lists, +) from cortex.lib.exceptions import CortexException, UserException, UserRuntimeException from cortex.lib.log import get_logger from cortex import consts From 667043d828fd8371e46c4739141c829a36a16b49 Mon Sep 17 00:00:00 2001 From: vishal Date: Thu, 11 Jul 2019 15:34:33 +0000 Subject: [PATCH 3/3] Update relative path to absolute path in init.py --- pkg/workloads/cortex/lib/storage/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/workloads/cortex/lib/storage/__init__.py b/pkg/workloads/cortex/lib/storage/__init__.py index d134bff6ca..1eef17d54a 100644 --- a/pkg/workloads/cortex/lib/storage/__init__.py +++ b/pkg/workloads/cortex/lib/storage/__init__.py @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .local import LocalStorage -from .s3 import S3 +from cortex.lib.storage.local import LocalStorage +from cortex.lib.storage.s3 import S3