Add cloudwatch statsd daemonset

Makefile

@@ -146,6 +146,7 @@ ci-build-images:
 	@./build/build-image.sh images/onnx-serve-gpu onnx-serve-gpu
 	@./build/build-image.sh images/operator operator
 	@./build/build-image.sh images/fluentd fluentd
+	@./build/build-image.sh images/statsd statsd
 	@./build/build-image.sh images/cluster-autoscaler cluster-autoscaler
 	@./build/build-image.sh images/nvidia nvidia
 	@./build/build-image.sh images/metrics-server metrics-server
@@ -164,6 +165,7 @@ ci-push-images:
 	@./build/push-image.sh onnx-serve-gpu
 	@./build/push-image.sh operator
 	@./build/push-image.sh fluentd
+	@./build/push-image.sh statsd
 	@./build/push-image.sh cluster-autoscaler
 	@./build/push-image.sh nvidia
 	@./build/push-image.sh metrics-server

cortex.sh

@@ -168,6 +168,7 @@ export CORTEX_NAMESPACE="${CORTEX_NAMESPACE:-cortex}"
 
 export CORTEX_IMAGE_MANAGER="${CORTEX_IMAGE_MANAGER:-cortexlabs/manager:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_FLUENTD="${CORTEX_IMAGE_FLUENTD:-cortexlabs/fluentd:$CORTEX_VERSION_STABLE}"
+export CORTEX_IMAGE_STATSD="${CORTEX_IMAGE_STATSD:-cortexlabs/statsd:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_OPERATOR="${CORTEX_IMAGE_OPERATOR:-cortexlabs/operator:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_TF_SERVE="${CORTEX_IMAGE_TF_SERVE:-cortexlabs/tf-serve:$CORTEX_VERSION_STABLE}"
 export CORTEX_IMAGE_TF_API="${CORTEX_IMAGE_TF_API:-cortexlabs/tf-api:$CORTEX_VERSION_STABLE}"
@@ -227,6 +228,7 @@ function install_cortex() {
     -e CORTEX_LOG_GROUP=$CORTEX_LOG_GROUP \
     -e CORTEX_BUCKET=$CORTEX_BUCKET \
     -e CORTEX_IMAGE_FLUENTD=$CORTEX_IMAGE_FLUENTD \
+    -e CORTEX_IMAGE_STATSD=$CORTEX_IMAGE_STATSD \
     -e CORTEX_IMAGE_OPERATOR=$CORTEX_IMAGE_OPERATOR \
     -e CORTEX_IMAGE_TF_SERVE=$CORTEX_IMAGE_TF_SERVE \
     -e CORTEX_IMAGE_TF_API=$CORTEX_IMAGE_TF_API \

dev/registry.sh

@@ -37,6 +37,7 @@ function ecr_login() {
 function create_registry() {
   aws ecr create-repository --repository-name=cortexlabs/manager --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/fluentd --region=$REGISTRY_REGION || true
+  aws ecr create-repository --repository-name=cortexlabs/statsd --region=$REGISTRY_REGION || true  
   aws ecr create-repository --repository-name=cortexlabs/istio-citadel --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/istio-pilot --region=$REGISTRY_REGION || true
   aws ecr create-repository --repository-name=cortexlabs/istio-galley --region=$REGISTRY_REGION || true
@@ -128,6 +129,7 @@ elif [ "$cmd" = "update" ]; then
     build_and_push $ROOT/images/tf-serve-gpu tf-serve-gpu latest
 
     build_and_push $ROOT/images/fluentd fluentd latest
+    build_and_push $ROOT/images/statsd statsd latest
     build_and_push $ROOT/images/cluster-autoscaler cluster-autoscaler latest
     build_and_push $ROOT/images/nvidia nvidia latest
     build_and_push $ROOT/images/metrics-server metrics-server latest
@@ -140,7 +142,6 @@ elif [ "$cmd" = "update" ]; then
   build_and_push $ROOT/images/downloader downloader latest
   build_and_push $ROOT/images/tf-api tf-api latest
   build_and_push $ROOT/images/onnx-serve onnx-serve latest
-  build_and_push $ROOT/images/onnx-serve-gpu onnx-serve-gpu latest
 
   cleanup
 fi

docs/cluster/config.md

@@ -45,6 +45,7 @@ export CORTEX_NAMESPACE="cortex"
 # Image paths
 export CORTEX_IMAGE_MANAGER="cortexlabs/manager:master"
 export CORTEX_IMAGE_FLUENTD="cortexlabs/fluentd:master"
+export CORTEX_IMAGE_STATSD="cortexlabs/statsd:master"
 export CORTEX_IMAGE_OPERATOR="cortexlabs/operator:master"
 export CORTEX_IMAGE_TF_SERVE="cortexlabs/tf-serve:master"
 export CORTEX_IMAGE_TF_API="cortexlabs/tf-api:master"

docs/development.md

@@ -63,6 +63,7 @@ export CORTEX_NAMESPACE="cortex"
 
 export CORTEX_IMAGE_MANAGER="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/manager:latest"
 export CORTEX_IMAGE_FLUENTD="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/fluentd:latest"
+export CORTEX_IMAGE_STATSD="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/statsd:latest"
 export CORTEX_IMAGE_ONNX_SERVE="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/onnx-serve:latest"
 export CORTEX_IMAGE_ONNX_SERVE_GPU="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/onnx-serve-gpu:latest"
 export CORTEX_IMAGE_OPERATOR="XXXXXXXX.dkr.ecr.us-west-2.amazonaws.com/cortexlabs/operator:latest"

images/statsd/Dockerfile

@@ -0,0 +1 @@
+FROM amazon/cloudwatch-agent:1.226589.0

manager/manifests/statsd.yaml

@@ -53,7 +53,7 @@ spec:
     spec:
       containers:
         - name: cloudwatch-agent
-          image: amazon/cloudwatch-agent:latest
+          image: $CORTEX_IMAGE_STATSD
           imagePullPolicy: Always
           ports:
             # containerPort should be consistent with the listen port defined in configmap
@@ -65,7 +65,7 @@ spec:
               cpu:  200m
               memory: 100Mi
             requests:
-              cpu: 200m
+              cpu: 100m
               memory: 100Mi
           # Please don't change the env
           env:

pkg/operator/workloads/api_workload.go

@@ -330,14 +330,18 @@ func tfAPISpec(
 							"--cache-dir=" + consts.ContextCacheDir,
 							"--project-dir=" + path.Join(consts.EmptyDirMountPath, "project"),
 						},
-						Env: append(k8s.AWSCredentials(), kcore.EnvVar{
-							Name: "HOST_IP",
-							ValueFrom: &kcore.EnvVarSource{
-								FieldRef: &kcore.ObjectFieldSelector{
-									FieldPath: "status.hostIP",
+						Env: append(
+							k8s.AWSCredentials(),
+							kcore.EnvVar{
+								Name: "HOST_IP",
+								ValueFrom: &kcore.EnvVarSource{
+									FieldRef: &kcore.ObjectFieldSelector{
+										FieldPath: "status.hostIP",
+									},
 								},
 							},
-						}), VolumeMounts: k8s.DefaultVolumeMounts(),
+						),
+						VolumeMounts: k8s.DefaultVolumeMounts(),
 						ReadinessProbe: &kcore.Probe{
 							InitialDelaySeconds: 5,
 							TimeoutSeconds:      5,
@@ -494,14 +498,17 @@ func onnxAPISpec(
 							"--cache-dir=" + consts.ContextCacheDir,
 							"--project-dir=" + path.Join(consts.EmptyDirMountPath, "project"),
 						},
-						Env: append(k8s.AWSCredentials(), kcore.EnvVar{
-							Name: "HOST_IP",
-							ValueFrom: &kcore.EnvVarSource{
-								FieldRef: &kcore.ObjectFieldSelector{
-									FieldPath: "status.hostIP",
+						Env: append(
+							k8s.AWSCredentials(),
+							kcore.EnvVar{
+								Name: "HOST_IP",
+								ValueFrom: &kcore.EnvVarSource{
+									FieldRef: &kcore.ObjectFieldSelector{
+										FieldPath: "status.hostIP",
+									},
 								},
 							},
-						}),
+						),
 						VolumeMounts: k8s.DefaultVolumeMounts(),
 						ReadinessProbe: &kcore.Probe{
 							InitialDelaySeconds: 5,

pkg/operator/workloads/metrics.go

@@ -245,17 +245,23 @@ func getAPIDimensions(appName string, api *context.API) []*cloudwatch.Dimension
 }
 
 func getAPIDimensionsCounter(appName string, api *context.API) []*cloudwatch.Dimension {
-	return append(getAPIDimensions(appName, api), &cloudwatch.Dimension{
-		Name:  aws.String("metric_type"),
-		Value: aws.String("counter"),
-	})
+	return append(
+		getAPIDimensions(appName, api),
+		&cloudwatch.Dimension{
+			Name:  aws.String("metric_type"),
+			Value: aws.String("counter"),
+		},
+	)
 }
 
 func getAPIDimensionsHistogram(appName string, api *context.API) []*cloudwatch.Dimension {
-	return append(getAPIDimensions(appName, api), &cloudwatch.Dimension{
-		Name:  aws.String("metric_type"),
-		Value: aws.String("histogram"),
-	})
+	return append(
+		getAPIDimensions(appName, api),
+		&cloudwatch.Dimension{
+			Name:  aws.String("metric_type"),
+			Value: aws.String("histogram"),
+		},
+	)
 }
 
 func getRegressionMetricDef(appName string, api *context.API, period int64) []*cloudwatch.MetricDataQuery {