Container as a Service (CaaS) (#2173)

Co-authored-by: Robert Lucian Chiriac <[email protected]>
Co-authored-by: David Eliahu <[email protected]>
Co-authored-by: Vishal Bollu <[email protected]>
Co-authored-by: vishal <[email protected]>

Author: 4 people

.circleci/config.yml

@@ -20,8 +20,9 @@ commands:
       - run:
           name: Install Go
           command: |
-            wget https://dl.google.com/go/go1.14.7.linux-amd64.tar.gz
-            sudo tar -C /usr/local -xzf go1.14.7.linux-amd64.tar.gz
+            sudo rm -rf /usr/local/go
+            wget https://dl.google.com/go/go1.15.12.linux-amd64.tar.gz
+            sudo tar -C /usr/local -xzf go1.15.12.linux-amd64.tar.gz
             rm -rf go*.tar.gz
             echo 'export PATH=$PATH:/usr/local/go/bin' >> $BASH_ENV
             echo 'export PATH=$PATH:~/go/bin' >> $BASH_ENV
@@ -75,18 +76,17 @@ commands:
 
 jobs:
   test:
-    docker:
-      - image: circleci/python:3.6
+    machine:
+      image: ubuntu-1604:202104-01  # machine executor necessary to run go integration tests
     steps:
       - checkout
-      - setup_remote_docker
       - install-go
       - run:
           name: Install Linting Tools
           command: |
             go get -u -v golang.org/x/lint/golint
             go get -u -v github.com/kyoh86/looppointer/cmd/looppointer
-            sudo pip install black aiohttp
+            pip3 install black aiohttp
       - run:
           name: Initialize Credentials
           command: |
@@ -111,9 +111,6 @@ jobs:
       - run:
           name: Go Tests
           command: make test-go
-      - run:
-          name: Python Tests
-          command: make test-python
 
   build-and-deploy:
     docker:
@@ -162,8 +159,8 @@ jobs:
             node_groups:
               - name: spot
                 instance_type: t3.medium
-                min_instances: 10
-                max_instances: 10
+                min_instances: 16
+                max_instances: 16
                 spot: true
               - name: cpu
                 instance_type: c5.xlarge

.github/ISSUE_TEMPLATE/bug-report.md

@@ -39,7 +39,7 @@ assignees: ''
 
 ### Stack traces
 
-(error output from `cortex logs <api name>`)
+(error output from CloudWatch Insights or from a random pod `cortex logs <api name>`)
 
 ```text
 <paste stack traces here>

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 ## Remote development
 
-We recommend that you run your development environment on an EC2 instance due to frequent docker registry pushing. We've had a good experience using [Mutagen](https://mutagen.io/documentation/introduction) to synchronize local / remote file systems.
+We recommend that you run your development environment on an EC2 instance due to frequent docker registry pushing. We've had a good experience using [Mutagen](https://mutagen.io/documentation/introduction) to synchronize local / remote filesystems.
 
 ## Prerequisites
 
@@ -169,7 +169,7 @@ node_groups:
 Add this to your bash profile (e.g. `~/.bash_profile`, `~/.profile` or `~/.bashrc`), replacing the placeholders accordingly:
 
 ```bash
-# set the default image for APIs
+# set the default image registry
 export CORTEX_DEV_DEFAULT_IMAGE_REGISTRY="<account_id>.dkr.ecr.<region>.amazonaws.com/cortexlabs"
 
 # redirect analytics and error reporting to our dev environment
@@ -209,7 +209,7 @@ Here is the typical full dev workflow which covers most cases:
 1. `make cluster-up` (creates a cluster using `dev/config/cluster.yaml`)
 2. `make devstart` (deletes the in-cluster operator, builds the CLI, and starts the operator locally; file changes will trigger the CLI and operator to re-build)
 3. Make your changes
-4. `make images-dev` (only necessary if API images or the manager are modified)
+4. `make images-dev` (only necessary if changes were made outside of the operator and CLI)
 5. Test your changes e.g. via `cortex deploy` (and repeat steps 3 and 4 as necessary)
 6. `make cluster-down` (deletes your cluster)
 
@@ -224,6 +224,4 @@ If you are only modifying the CLI, `make cli-watch` will build the CLI and re-bu
 
 If you are only modifying the operator, `make operator-local` will build and start the operator locally, and build/restart it when files are changed.
 
-If you are modifying code in the API images (i.e. any of the Python serving code), `make images-dev` may build more images than you need during testing. For example, if you are only testing using the `python-handler-cpu` image, you can run `./dev/registry.sh update-single python-handler-cpu`.
-
 See `Makefile` for additional dev commands.

Makefile

@@ -124,8 +124,7 @@ async-gateway-update:
 	@./dev/registry.sh update-single async-gateway
 	@kubectl delete pods -l cortex.dev/async=gateway --namespace=default
 
-# Docker images
-
+# docker images
 images-all:
 	@./dev/registry.sh update all
 images-all-skip-push:
@@ -136,15 +135,8 @@ images-dev:
 images-dev-skip-push:
 	@./dev/registry.sh update dev --skip-push
 
-images-api:
-	@./dev/registry.sh update api
-images-api-skip-push:
-	@./dev/registry.sh update api --skip-push
-
 images-manager-skip-push:
 	@./dev/registry.sh update-single manager --skip-push
-images-iris:
-	@./dev/registry.sh update-single python-handler-cpu
 
 registry-create:
 	@./dev/registry.sh create
@@ -170,15 +162,14 @@ format:
 # Tests #
 #########
 
-test:
-	@./build/test.sh
+# build test api images
+# make sure you login with your quay credentials
+build-test-api-images:
+	@./test/utils/build-all.sh quay.io/cortexlabs-test
 
-test-go:
+test:
 	@./build/test.sh go
 
-test-python:
-	@./build/test.sh python
-
 # run e2e tests on an existing cluster
 # read test/e2e/README.md for instructions first
 test-e2e:

build/build-image.sh

@@ -26,16 +26,4 @@ image=$1
 if [ "$image" == "inferentia" ]; then
   aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 790709498068.dkr.ecr.us-west-2.amazonaws.com
 fi
-
-build_args=""
-
-if [ "${image}" == "python-handler-gpu" ]; then
-  cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1")
-  cudnn=("7" "7" "8" "7" "8" "8" "8")
-  for i in ${!cudnn[@]}; do
-    build_args="${build_args} --build-arg CUDA_VERSION=${cuda[$i]} --build-arg CUDNN=${cudnn[$i]}"
-    docker build "$ROOT" -f $ROOT/images/$image/Dockerfile $build_args -t quay.io/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]} -t cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
-  done
-else
-  docker build "$ROOT" -f $ROOT/images/$image/Dockerfile $build_args -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION}
-fi
+docker build "$ROOT" -f $ROOT/images/$image/Dockerfile -t quay.io/cortexlabs/${image}:${CORTEX_VERSION} -t cortexlabs/${image}:${CORTEX_VERSION}

build/images.sh

@@ -19,24 +19,15 @@
 
 set -euo pipefail
 
-api_images=(
-  "python-handler-cpu"
-  "python-handler-gpu"
-  "tensorflow-handler"
-  "python-handler-inf"
-)
-
 dev_images=(
-  "downloader"
   "manager"
-  "request-monitor"
+  "proxy"
   "async-gateway"
   "enqueuer"
+  "dequeuer"
 )
 
 non_dev_images=(
-  "tensorflow-serving-cpu"
-  "tensorflow-serving-gpu"
   "cluster-autoscaler"
   "operator"
   "controller-manager"
@@ -53,16 +44,13 @@ non_dev_images=(
   "kube-rbac-proxy"
   "grafana"
   "event-exporter"
-  "tensorflow-serving-inf"
   "metrics-server"
   "inferentia"
-  "neuron-rtd"
   "nvidia"
   "kubexit"
 )
 
 all_images=(
-  "${api_images[@]}"
   "${dev_images[@]}"
   "${non_dev_images[@]}"
 )

build/lint.sh

@@ -84,6 +84,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.vscode/*" \
 ! -path "**/.idea/*" \
 ! -path "**/.history/*" \
+! -path "**/testbin/*" \
 ! -path "**/__pycache__/*" \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
@@ -118,6 +119,7 @@ if [ "$is_release_branch" = "true" ]; then
   ! -path "**/.vscode/*" \
   ! -path "**/.idea/*" \
   ! -path "**/.history/*" \
+  ! -path "**/testbin/*" \
   ! -path "**/__pycache__/*" \
   ! -path "**/.pytest_cache/*" \
   ! -path "**/*.egg-info/*" \
@@ -141,6 +143,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.idea/*" \
 ! -path "**/.history/*" \
 ! -path "**/.vscode/*" \
+! -path "**/testbin/*" \
 ! -path "**/__pycache__/*" \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
@@ -164,6 +167,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.idea/*" \
 ! -path "**/.history/*" \
 ! -path "**/.vscode/*" \
+! -path "**/testbin/*" \
 ! -path "**/__pycache__/*" \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
@@ -188,6 +192,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.vscode/*" \
 ! -path "**/.idea/*" \
 ! -path "**/.history/*" \
+! -path "**/testbin/*" \
 ! -path "**/__pycache__/*" \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \
@@ -210,6 +215,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "**/.idea/*" \
 ! -path "**/.history/*" \
 ! -path "**/.vscode/*" \
+! -path "**/testbin/*" \
 ! -path "**/__pycache__/*" \
 ! -path "**/.pytest_cache/*" \
 ! -path "**/*.egg-info/*" \

build/push-image.sh

@@ -23,13 +23,4 @@ host=$1
 image=$2
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-
-if [ "$image" == "python-handler-gpu" ]; then
-  cuda=("10.0" "10.1" "10.1" "10.2" "10.2" "11.0" "11.1")
-  cudnn=("7" "7" "8" "7" "8" "8" "8")
-  for i in ${!cudnn[@]}; do
-    docker push $host/cortexlabs/${image}:${CORTEX_VERSION}-cuda${cuda[$i]}-cudnn${cudnn[$i]}
-  done
-else
-  docker push $host/cortexlabs/${image}:${CORTEX_VERSION}
-fi
+docker push $host/cortexlabs/${image}:${CORTEX_VERSION}

build/test.sh

@@ -79,11 +79,6 @@ function run_go_tests() {
   )
 }
 
-function run_python_tests() {
-  docker build $ROOT -f $ROOT/images/test/Dockerfile -t cortexlabs/test
-  docker run cortexlabs/test
-}
-
 function run_e2e_tests() {
   if [ "$create_cluster" = "yes" ]; then
     pytest $ROOT/test/e2e/tests --config "$sub_cmd"
@@ -94,11 +89,6 @@ function run_e2e_tests() {
 
 if [ "$cmd" = "go" ]; then
   run_go_tests
-elif [ "$cmd" = "python" ]; then
-  run_python_tests
 elif [ "$cmd" = "e2e" ]; then
   run_e2e_tests
-else
-  run_go_tests
-  run_python_tests
 fi

cli/cluster/logs.go

@@ -35,12 +35,40 @@ import (
 	"github.com/gorilla/websocket"
 )
 
+func GetLogs(operatorConfig OperatorConfig, apiName string) (schema.LogResponse, error) {
+	httpRes, err := HTTPGet(operatorConfig, "/logs/"+apiName)
+	if err != nil {
+		return schema.LogResponse{}, err
+	}
+
+	var logResponse schema.LogResponse
+	if err = json.Unmarshal(httpRes, &logResponse); err != nil {
+		return schema.LogResponse{}, errors.Wrap(err, "/logs/"+apiName, string(httpRes))
+	}
+
+	return logResponse, nil
+}
+
+func GetJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) (schema.LogResponse, error) {
+	httpRes, err := HTTPGet(operatorConfig, "/logs/"+apiName, map[string]string{"jobID": jobID})
+	if err != nil {
+		return schema.LogResponse{}, err
+	}
+
+	var logResponse schema.LogResponse
+	if err = json.Unmarshal(httpRes, &logResponse); err != nil {
+		return schema.LogResponse{}, errors.Wrap(err, "/logs/"+apiName, string(httpRes))
+	}
+
+	return logResponse, nil
+}
+
 func StreamLogs(operatorConfig OperatorConfig, apiName string) error {
-	return streamLogs(operatorConfig, "/logs/"+apiName)
+	return streamLogs(operatorConfig, "/streamlogs/"+apiName)
 }
 
 func StreamJobLogs(operatorConfig OperatorConfig, apiName string, jobID string) error {
-	return streamLogs(operatorConfig, "/logs/"+apiName, map[string]string{"jobID": jobID})
+	return streamLogs(operatorConfig, "/streamlogs/"+apiName, map[string]string{"jobID": jobID})
 }
 
 func streamLogs(operatorConfig OperatorConfig, path string, qParams ...map[string]string) error {