diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index 375f1ca0d8..7d2a24b014 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -17,6 +17,7 @@ limitations under the License. package cmd import ( + "encoding/base64" "fmt" "os" "path/filepath" @@ -24,21 +25,25 @@ import ( "strings" "time" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/s3" "github.com/cortexlabs/cortex/cli/cluster" "github.com/cortexlabs/cortex/cli/types/cliconfig" "github.com/cortexlabs/cortex/cli/types/flags" "github.com/cortexlabs/cortex/pkg/consts" - "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/health" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" "github.com/cortexlabs/cortex/pkg/lib/console" "github.com/cortexlabs/cortex/pkg/lib/docker" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/exit" "github.com/cortexlabs/cortex/pkg/lib/files" libjson "github.com/cortexlabs/cortex/pkg/lib/json" + "github.com/cortexlabs/cortex/pkg/lib/k8s" libmath "github.com/cortexlabs/cortex/pkg/lib/math" "github.com/cortexlabs/cortex/pkg/lib/pointer" "github.com/cortexlabs/cortex/pkg/lib/prompt" @@ -51,6 +56,10 @@ import ( "github.com/cortexlabs/cortex/pkg/types/clusterstate" "github.com/cortexlabs/yaml" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/aws-iam-authenticator/pkg/token" ) var ( @@ -101,11 +110,21 @@ func clusterInit() { addClusterNameFlag(_clusterExportCmd) addClusterRegionFlag(_clusterExportCmd) _clusterCmd.AddCommand(_clusterExportCmd) + + _clusterHealthCmd.Flags().SortFlags = false + addClusterConfigFlag(_clusterHealthCmd) + addClusterNameFlag(_clusterHealthCmd) + addClusterRegionFlag(_clusterHealthCmd) + _clusterHealthCmd.Flags().VarP(&_flagOutput, "output", "o", fmt.Sprintf("output format: one of %s", strings.Join(flags.OutputTypeStringsExcluding(flags.YAMLOutputType), "|"))) + _clusterCmd.AddCommand(_clusterHealthCmd) } func addClusterConfigFlag(cmd *cobra.Command) { cmd.Flags().StringVarP(&_flagClusterConfig, "config", "c", "", "path to a cluster configuration file") - cmd.Flags().SetAnnotation("config", cobra.BashCompFilenameExt, _configFileExts) + err := cmd.Flags().SetAnnotation("config", cobra.BashCompFilenameExt, _configFileExts) + if err != nil { + exit.Error(err) // should never happen + } } func addClusterNameFlag(cmd *cobra.Command) { @@ -631,8 +650,8 @@ var _clusterDownCmd = &cobra.Command{ } // best-effort deletion of cached config - cachedClusterConfigPath := cachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region) - os.Remove(cachedClusterConfigPath) + cachedClusterConfigPath := getCachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region) + _ = os.Remove(cachedClusterConfigPath) if len(errorsList) > 0 { exit.Error(errors.ListOfErrors(ErrClusterDown, false, errorsList...)) @@ -743,7 +762,83 @@ var _clusterExportCmd = &cobra.Command{ }, } -func cmdInfo(awsClient *aws.Client, accessConfig *clusterconfig.AccessConfig, stacks clusterstate.ClusterStacks, printConfig bool, outputType flags.OutputType, disallowPrompt bool) { +var _clusterHealthCmd = &cobra.Command{ + Use: "health", + Short: "inspect the health of components in the cluster", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + accessConfig, err := getClusterAccessConfigWithCache(true) + if err != nil { + exit.Error(err) + } + + awsClient, err := awslib.NewForRegion(accessConfig.Region) + if err != nil { + exit.Error(err) + } + + restConfig, err := getClusterRESTConfig(awsClient, accessConfig.ClusterName) + if err != nil { + exit.Error(err) + } + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + exit.Error(err) + } + + k8sClient, err := k8s.New("default", false, restConfig, scheme) + if err != nil { + exit.Error(err) + } + + clusterHealth, err := health.Check(awsClient, k8sClient, accessConfig.ClusterName) + if err != nil { + exit.Error(err) + } + + clusterWarnings, err := health.GetWarnings(k8sClient) + if err != nil { + exit.Error(err) + } + + if _flagOutput == flags.JSONOutputType { + fmt.Println(clusterHealth) + return + } + + healthTable := table.Table{ + Headers: []table.Header{ + {Title: ""}, + {Title: "live"}, + {Title: "warning", Hidden: !clusterWarnings.HasWarnings()}, + }, + Rows: [][]interface{}{ + {"operator", console.BoolColor(clusterHealth.Operator), ""}, + {"prometheus", console.BoolColor(clusterHealth.Prometheus), clusterWarnings.Prometheus}, + {"autoscaler", console.BoolColor(clusterHealth.Autoscaler), ""}, + {"activator", console.BoolColor(clusterHealth.Activator), ""}, + {"grafana", console.BoolColor(clusterHealth.Grafana), ""}, + {"controller manager", console.BoolColor(clusterHealth.ControllerManager), ""}, + {"apis gateway", console.BoolColor(clusterHealth.APIsGateway), ""}, + {"operator gateway", console.BoolColor(clusterHealth.APIsGateway), ""}, + {"cluster autoscaler", console.BoolColor(clusterHealth.ClusterAutoscaler), ""}, + {"operator load balancer", console.BoolColor(clusterHealth.OperatorLoadBalancer), ""}, + {"apis load balancer", console.BoolColor(clusterHealth.APIsLoadBalancer), ""}, + {"fluent bit", console.BoolColor(clusterHealth.FluentBit), ""}, + {"node exporter", console.BoolColor(clusterHealth.NodeExporter), ""}, + {"dcgm exporter", console.BoolColor(clusterHealth.DCGMExporter), ""}, + {"statsd exporter", console.BoolColor(clusterHealth.StatsDExporter), ""}, + {"event exporter", console.BoolColor(clusterHealth.EventExporter), ""}, + {"kube state metrics", console.BoolColor(clusterHealth.KubeStateMetrics), ""}, + }, + } + + fmt.Println(healthTable.MustFormat()) + }, +} + +func cmdInfo(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, stacks clusterstate.ClusterStacks, printConfig bool, outputType flags.OutputType, disallowPrompt bool) { clusterConfig := refreshCachedClusterConfig(awsClient, accessConfig, outputType == flags.PrettyOutputType) operatorLoadBalancer, err := getLoadBalancer(accessConfig.ClusterName, OperatorLoadBalancer, awsClient) @@ -846,14 +941,14 @@ func getInfoOperatorResponse(operatorEndpoint string) (*schema.InfoResponse, err } func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterconfig.Config) { - eksPrice := aws.EKSPrices[clusterConfig.Region] - operatorInstancePrice := aws.InstanceMetadatas[clusterConfig.Region]["t3.medium"].Price - operatorEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 - prometheusInstancePrice := aws.InstanceMetadatas[clusterConfig.Region][clusterConfig.PrometheusInstanceType].Price - prometheusEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 - metricsEBSPrice := aws.EBSMetadatas[clusterConfig.Region]["gp2"].PriceGB * (40 + 2) / 30 / 24 - nlbPrice := aws.NLBMetadatas[clusterConfig.Region].Price - natUnitPrice := aws.NATMetadatas[clusterConfig.Region].Price + eksPrice := awslib.EKSPrices[clusterConfig.Region] + operatorInstancePrice := awslib.InstanceMetadatas[clusterConfig.Region]["t3.medium"].Price + operatorEBSPrice := awslib.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 + prometheusInstancePrice := awslib.InstanceMetadatas[clusterConfig.Region][clusterConfig.PrometheusInstanceType].Price + prometheusEBSPrice := awslib.EBSMetadatas[clusterConfig.Region]["gp3"].PriceGB * 20 / 30 / 24 + metricsEBSPrice := awslib.EBSMetadatas[clusterConfig.Region]["gp2"].PriceGB * (40 + 2) / 30 / 24 + nlbPrice := awslib.NLBMetadatas[clusterConfig.Region].Price + natUnitPrice := awslib.NATMetadatas[clusterConfig.Region].Price headers := []table.Header{ {Title: "aws resource"}, @@ -874,13 +969,13 @@ func printInfoPricing(infoResponse *schema.InfoResponse, clusterConfig clusterco nodesInfo := infoResponse.GetNodesWithNodeGroupName(ngNamePrefix + ng.Name) numInstances := len(nodesInfo) - ebsPrice := aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24 + ebsPrice := awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceGB * float64(ng.InstanceVolumeSize) / 30 / 24 if ng.InstanceVolumeType == clusterconfig.IO1VolumeType && ng.InstanceVolumeIOPS != nil { - ebsPrice += aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24 + ebsPrice += awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS * float64(*ng.InstanceVolumeIOPS) / 30 / 24 } if ng.InstanceVolumeType == clusterconfig.GP3VolumeType && ng.InstanceVolumeIOPS != nil && ng.InstanceVolumeThroughput != nil { - ebsPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24) - ebsPrice += libmath.MaxFloat64(0, (aws.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24) + ebsPrice += libmath.MaxFloat64(0, (awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceIOPS-3000)*float64(*ng.InstanceVolumeIOPS)/30/24) + ebsPrice += libmath.MaxFloat64(0, (awslib.EBSMetadatas[clusterConfig.Region][ng.InstanceVolumeType.String()].PriceThroughput-125)*float64(*ng.InstanceVolumeThroughput)/30/24) } totalEBSPrice := ebsPrice * float64(numInstances) @@ -1040,7 +1135,7 @@ func updateCLIEnv(envName string, operatorEndpoint string, disallowPrompt bool, return nil } -func cmdDebug(awsClient *aws.Client, accessConfig *clusterconfig.AccessConfig) { +func cmdDebug(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig) { // note: if modifying this string, also change it in files.IgnoreCortexDebug() debugFileName := fmt.Sprintf("cortex-debug-%s.tgz", time.Now().UTC().Format("2006-01-02-15-04-05")) @@ -1064,9 +1159,9 @@ func cmdDebug(awsClient *aws.Client, accessConfig *clusterconfig.AccessConfig) { return } -func refreshCachedClusterConfig(awsClient *aws.Client, accessConfig *clusterconfig.AccessConfig, printToStdout bool) clusterconfig.Config { +func refreshCachedClusterConfig(awsClient *awslib.Client, accessConfig *clusterconfig.AccessConfig, printToStdout bool) clusterconfig.Config { // add empty file if cached cluster doesn't exist so that the file output by manager container maintains current user permissions - cachedClusterConfigPath := cachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region) + cachedClusterConfigPath := getCachedClusterConfigPath(accessConfig.ClusterName, accessConfig.Region) containerConfigPath := fmt.Sprintf("/out/%s", filepath.Base(cachedClusterConfigPath)) copyFromPaths := []dockerCopyFromPath{ @@ -1095,7 +1190,7 @@ func refreshCachedClusterConfig(awsClient *aws.Client, accessConfig *clusterconf return *refreshedClusterConfig } -func createS3BucketIfNotFound(awsClient *aws.Client, bucket string, tags map[string]string) error { +func createS3BucketIfNotFound(awsClient *awslib.Client, bucket string, tags map[string]string) error { bucketFound, err := awsClient.DoesBucketExist(bucket) if err != nil { return err @@ -1123,7 +1218,7 @@ func createS3BucketIfNotFound(awsClient *aws.Client, bucket string, tags map[str fmt.Println(" ✓") return nil } - if !aws.IsNoSuchBucketErr(err) { + if !awslib.IsNoSuchBucketErr(err) { break } time.Sleep(1 * time.Second) @@ -1133,7 +1228,7 @@ func createS3BucketIfNotFound(awsClient *aws.Client, bucket string, tags map[str return err } -func setLifecycleRulesOnClusterUp(awsClient *aws.Client, bucket, newClusterUID string) error { +func setLifecycleRulesOnClusterUp(awsClient *awslib.Client, bucket, newClusterUID string) error { err := awsClient.DeleteLifecycleRules(bucket) if err != nil { return err @@ -1177,7 +1272,7 @@ func setLifecycleRulesOnClusterUp(awsClient *aws.Client, bucket, newClusterUID s return awsClient.SetLifecycleRules(bucket, rules) } -func setLifecycleRulesOnClusterDown(awsClient *aws.Client, bucket string) error { +func setLifecycleRulesOnClusterDown(awsClient *awslib.Client, bucket string) error { err := awsClient.DeleteLifecycleRules(bucket) if err != nil { return err @@ -1198,7 +1293,7 @@ func setLifecycleRulesOnClusterDown(awsClient *aws.Client, bucket string) error }) } -func createLogGroupIfNotFound(awsClient *aws.Client, logGroup string, tags map[string]string) error { +func createLogGroupIfNotFound(awsClient *awslib.Client, logGroup string, tags map[string]string) error { logGroupFound, err := awsClient.DoesLogGroupExist(logGroup) if err != nil { return err @@ -1240,7 +1335,7 @@ func (lb LoadBalancer) String() string { } // Will return error if the load balancer can't be found -func getLoadBalancer(clusterName string, whichLB LoadBalancer, awsClient *aws.Client) (*elbv2.LoadBalancer, error) { +func getLoadBalancer(clusterName string, whichLB LoadBalancer, awsClient *awslib.Client) (*elbv2.LoadBalancer, error) { loadBalancer, err := awsClient.FindLoadBalancer(map[string]string{ clusterconfig.ClusterNameTag: clusterName, "cortex.dev/load-balancer": whichLB.String(), @@ -1256,7 +1351,7 @@ func getLoadBalancer(clusterName string, whichLB LoadBalancer, awsClient *aws.Cl return loadBalancer, nil } -func listPVCVolumesForCluster(awsClient *aws.Client, clusterName string) ([]ec2.Volume, error) { +func listPVCVolumesForCluster(awsClient *awslib.Client, clusterName string) ([]ec2.Volume, error) { return awsClient.ListVolumes(ec2.Tag{ Key: pointer.String(fmt.Sprintf("kubernetes.io/cluster/%s", clusterName)), Value: nil, // any value should be ok as long as the key is present @@ -1266,3 +1361,41 @@ func listPVCVolumesForCluster(awsClient *aws.Client, clusterName string) ([]ec2. func filterEKSCTLOutput(out string) string { return strings.Join(s.RemoveDuplicates(strings.Split(out, "\n"), _eksctlPrefixRegex), "\n") } + +func getClusterRESTConfig(awsClient *awslib.Client, clusterName string) (*rest.Config, error) { + clusterOutput, err := awsClient.EKS().DescribeCluster( + &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + }, + ) + if err != nil { + return nil, err + } + + gen, err := token.NewGenerator(true, false) + if err != nil { + return nil, err + } + + opts := &token.GetTokenOptions{ + ClusterID: aws.StringValue(clusterOutput.Cluster.Name), + } + + tok, err := gen.GetWithOptions(opts) + if err != nil { + return nil, err + } + + ca, err := base64.StdEncoding.DecodeString(aws.StringValue(clusterOutput.Cluster.CertificateAuthority.Data)) + if err != nil { + return nil, err + } + + return &rest.Config{ + Host: aws.StringValue(clusterOutput.Cluster.Endpoint), + BearerToken: tok.Token, + TLSClientConfig: rest.TLSClientConfig{ + CAData: ca, + }, + }, nil +} diff --git a/cli/cmd/lib_cluster_config.go b/cli/cmd/lib_cluster_config.go index 1da6de2f7a..b6c652fc8b 100644 --- a/cli/cmd/lib_cluster_config.go +++ b/cli/cmd/lib_cluster_config.go @@ -39,7 +39,7 @@ import ( var _cachedClusterConfigRegex = regexp.MustCompile(`^cluster_\S+\.yaml$`) -func cachedClusterConfigPath(clusterName string, region string) string { +func getCachedClusterConfigPath(clusterName string, region string) string { return filepath.Join(_localDir, fmt.Sprintf("cluster_%s_%s.yaml", clusterName, region)) } diff --git a/cli/cmd/lib_manager.go b/cli/cmd/lib_manager.go index 656ab3df2f..edacce2111 100644 --- a/cli/cmd/lib_manager.go +++ b/cli/cmd/lib_manager.go @@ -81,7 +81,7 @@ func runManager(containerConfig *container.Config, addNewLineAfterPull bool, cop } removeContainer := func() { - dockerClient.ContainerRemove(context.Background(), containerInfo.ID, dockertypes.ContainerRemoveOptions{ + _ = dockerClient.ContainerRemove(context.Background(), containerInfo.ID, dockertypes.ContainerRemoveOptions{ RemoveVolumes: true, Force: true, }) @@ -166,7 +166,7 @@ func runManagerWithClusterConfig(entrypoint string, clusterConfig *clusterconfig return "", nil, errors.WithStack(err) } - cachedClusterConfigPath := cachedClusterConfigPath(clusterConfig.ClusterName, clusterConfig.Region) + cachedClusterConfigPath := getCachedClusterConfigPath(clusterConfig.ClusterName, clusterConfig.Region) if err := files.WriteFile(clusterConfigBytes, cachedClusterConfigPath); err != nil { return "", nil, err } diff --git a/go.mod b/go.mod index ba5c190c9c..6acb918587 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,9 @@ require ( k8s.io/client-go v0.20.8 k8s.io/klog/v2 v2.9.0 // indirect k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d // indirect + k8s.io/metrics v0.20.8 k8s.io/utils v0.0.0-20210629042839-4a2b36d8d73f // indirect + sigs.k8s.io/aws-iam-authenticator v0.5.3 sigs.k8s.io/controller-runtime v0.8.3 sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect ) diff --git a/go.sum b/go.sum index 1b0384d8fd..3517621d1e 100644 --- a/go.sum +++ b/go.sum @@ -128,6 +128,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/amazon-vpc-cni-k8s v1.8.0 h1:YVteZl3WkKoe554gkmFVUvR9/CliamTXRSsJaE1neno= github.com/aws/amazon-vpc-cni-k8s v1.8.0/go.mod h1:p7GsqsIcTy/IN/zOK2k2GKg1nd1Z/1t0ydTTVY80zV8= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.37.1/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.37.23/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.70 h1:EGHVUQzHIxQDF9LwQU22yE9bJd1HuBAWpJYSEnxnnhc= github.com/aws/aws-sdk-go v1.38.70/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -300,6 +301,7 @@ github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/ github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -336,6 +338,7 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -461,6 +464,8 @@ github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g= +github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -524,6 +529,7 @@ github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -635,6 +641,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -738,6 +745,7 @@ github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6U github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -766,6 +774,7 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -776,6 +785,7 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -827,12 +837,14 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= @@ -847,6 +859,7 @@ github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= @@ -856,6 +869,7 @@ github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJ github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -868,6 +882,7 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -933,6 +948,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1009,6 +1025,7 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3C go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.hein.dev/go-version v0.1.0/go.mod h1:WOEm7DWMroRe5GdUgHMvx+Pji5WWIpMuXmK/3foylXs= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -1064,7 +1081,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= @@ -1212,6 +1231,7 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812172437-4e8604ab3aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1306,6 +1326,7 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1321,6 +1342,7 @@ golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190812233024-afc3694995b6/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1378,6 +1400,9 @@ gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3m gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1563,6 +1588,7 @@ istio.io/client-go v1.10.2-0.20210617171818-3dcf18fc084e h1:MABqeyEb9EbOUGOvnyxR istio.io/client-go v1.10.2-0.20210617171818-3dcf18fc084e/go.mod h1:/lKE20CQpKX9fWglhrT4X7bQpx4OlFYrsbxD9pUMItA= istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f h1:9710FpGLvIJ1GGEbpuTh1smVBv+r8cJfR3G82ouSxIQ= istio.io/gogo-genproto v0.0.0-20210113155706-4daf5697332f/go.mod h1:6BwTZRNbWS570wHX/uR1Wqk5e0157TofTAUMzT7N4+s= +k8s.io/api v0.16.8/go.mod h1:a8EOdYHO8en+YHhPBLiW5q+3RfHTr7wxTqqp7emJ7PM= k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= @@ -1574,6 +1600,7 @@ k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKi k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apiextensions-apiserver v0.20.8 h1:ZHXuqNKUtvOq36q00jf9bRAQ/e+/ryQ0Weg9B1fPB8U= k8s.io/apiextensions-apiserver v0.20.8/go.mod h1:d7JXi5muyuBX1DrzBZANxJEkgqi9B4wFMM6Lg1Bnz68= +k8s.io/apimachinery v0.16.8/go.mod h1:Xk2vD2TRRpuWYLQNM6lT9R7DSFZUYG03SarNkbGrnKE= k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= @@ -1586,6 +1613,7 @@ k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/apiserver v0.20.8/go.mod h1:uOh7ih6qLY+5jOC5/RauUPEeqQpMS0DXbjeESxlbeh4= +k8s.io/client-go v0.16.8/go.mod h1:WmPuN0yJTKHXoklExKxzo3jSXmr3EnN+65uaTb5VuNs= k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= @@ -1593,9 +1621,11 @@ k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= k8s.io/client-go v0.20.8 h1:ewvQIQDqUkQVajs06zzKErH/qpYcHaMvz+P7AF7nsWs= k8s.io/client-go v0.20.8/go.mod h1:ufY6eLPP3u1Xjc3YjvzyXbYwtGVKMNyeH3m7oA/2s/w= +k8s.io/code-generator v0.16.8/go.mod h1:wFdrXdVi/UC+xIfLi+4l9elsTT/uEF61IfcN2wOLULQ= k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.8/go.mod h1:i6FmG+QxaLxvJsezvZp0q/gAEzzOz3U53KFibghWToU= +k8s.io/component-base v0.16.8/go.mod h1:Q8UWOWShpP3MZZny4n/15gOncfaaVtc9SbCdkM5MhUE= k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= @@ -1609,6 +1639,7 @@ k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -1621,17 +1652,27 @@ k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d h1:lUK8GPtuJy8ClWZhuvKoaLdKGPLq9H1PxWp7VPBZBkU= k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/metrics v0.20.8 h1:BQ+WmcYnYzHFnsNstoJUGTa9ILjjhLPE7orF9uKL5j4= +k8s.io/metrics v0.20.8/go.mod h1:Cj4wgRhrw2ifUXth/r87snepxhOTG1rmaCDe6DN/xCA= +k8s.io/sample-controller v0.16.8/go.mod h1:aXlORS1ekU77qhGybB5t3JORDurzDpWgvMYxmCsiuos= +k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210629042839-4a2b36d8d73f h1:6Cyc8f2OS555SrragQyv4rQ5G7F2haZ6KY2oxO1wzlE= k8s.io/utils v0.0.0-20210629042839-4a2b36d8d73f/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -1639,9 +1680,13 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/aws-iam-authenticator v0.5.3 h1:EyqQ/uxzbe2mDETZZmuMnv0xHITnyLhZfPlGb6Mma20= +sigs.k8s.io/aws-iam-authenticator v0.5.3/go.mod h1:DIq7gy0lvnyaG88AgFyJzUVeix+ia5msHEp4RL0102I= sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/pkg/health/health.go b/pkg/health/health.go new file mode 100644 index 0000000000..cd123c13b4 --- /dev/null +++ b/pkg/health/health.go @@ -0,0 +1,332 @@ +/* +Copyright 2021 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package health + +import ( + "context" + "fmt" + "reflect" + + "github.com/aws/aws-sdk-go/service/elbv2" + awslib "github.com/cortexlabs/cortex/pkg/lib/aws" + "github.com/cortexlabs/cortex/pkg/lib/errors" + "github.com/cortexlabs/cortex/pkg/lib/json" + "github.com/cortexlabs/cortex/pkg/lib/k8s" + "github.com/cortexlabs/cortex/pkg/lib/parallel" + "github.com/cortexlabs/cortex/pkg/types/clusterconfig" + kapps "k8s.io/api/apps/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + kresource "k8s.io/apimachinery/pkg/api/resource" + kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" + kmetrics "k8s.io/metrics/pkg/client/clientset/versioned" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ClusterHealth represents the healthiness of each component of a cluster +type ClusterHealth struct { + Operator bool `json:"operator"` + ControllerManager bool `json:"controller_manager"` + Prometheus bool `json:"prometheus"` + Autoscaler bool `json:"autoscaler"` + Activator bool `json:"activator"` + Grafana bool `json:"grafana"` + OperatorGateway bool `json:"operator_gateway"` + APIsGateway bool `json:"apis_gateway"` + ClusterAutoscaler bool `json:"cluster_autoscaler"` + OperatorLoadBalancer bool `json:"operator_load_balancer"` + APIsLoadBalancer bool `json:"apis_load_balancer"` + FluentBit bool `json:"fluent_bit"` + NodeExporter bool `json:"node_exporter"` + DCGMExporter bool `json:"dcgm_exporter"` + StatsDExporter bool `json:"statsd_exporter"` + EventExporter bool `json:"event_exporter"` + KubeStateMetrics bool `json:"kube_state_metrics"` +} + +func (c ClusterHealth) String() string { + bytes, err := json.Marshal(c) + if err != nil { + panic(err) + } + + return string(bytes) +} + +type ClusterWarnings struct { + Prometheus string +} + +// HasWarnings checks if ClusterWarnings has any warnings in its' fields +func (w ClusterWarnings) HasWarnings() bool { + v := reflect.ValueOf(w) + for i := 0; i < v.NumField(); i++ { + if v.Field(i).String() != "" { + return true + } + } + return false +} + +// Check checks for the health of the different components of a cluster +func Check(awsClient *awslib.Client, k8sClient *k8s.Client, clusterName string) (ClusterHealth, error) { + var ( + operatorHealth bool + controllerManagerHealth bool + prometheusHealth bool + autoscalerHealth bool + activatorHealth bool + grafanaHealth bool + operatorGatewayHealth bool + apisGatewayHealth bool + clusterAutoscalerHealth bool + operatorLoadBalancerHealth bool + apisLoadBalancerHealth bool + fluentBitHealth bool + nodeExporterHealth bool + dcgmExporterHealth bool + statsdExporterHealth bool + eventExporterHealth bool + kubeStateMetricsHealth bool + ) + + if err := parallel.RunFirstErr( + func() error { + var err error + operatorHealth, err = getDeploymentReadiness(k8sClient, "operator", "default") + return err + }, + func() error { + var err error + controllerManagerHealth, err = getDeploymentReadiness(k8sClient, "operator-controller-manager", "default") + return err + }, + func() error { + var err error + prometheusHealth, err = getStatefulSetReadiness(k8sClient, "prometheus-prometheus", "default") + return err + }, + func() error { + var err error + autoscalerHealth, err = getDeploymentReadiness(k8sClient, "autoscaler", "default") + return err + }, + func() error { + var err error + activatorHealth, err = getDeploymentReadiness(k8sClient, "activator", "default") + return err + }, + func() error { + var err error + grafanaHealth, err = getStatefulSetReadiness(k8sClient, "grafana", "default") + return err + }, + func() error { + var err error + operatorGatewayHealth, err = getDeploymentReadiness(k8sClient, "ingressgateway-operator", "istio-system") + return err + }, + func() error { + var err error + apisGatewayHealth, err = getDeploymentReadiness(k8sClient, "ingressgateway-apis", "istio-system") + return err + }, + func() error { + var err error + clusterAutoscalerHealth, err = getDeploymentReadiness(k8sClient, "cluster-autoscaler", "kube-system") + return err + }, + func() error { + var err error + operatorLoadBalancerHealth, err = getLoadBalancerHealth(awsClient, clusterName, "operator") + return err + }, + func() error { + var err error + apisLoadBalancerHealth, err = getLoadBalancerHealth(awsClient, clusterName, "api") + return err + }, + func() error { + var err error + fluentBitHealth, err = getDaemonSetReadiness(k8sClient, "fluent-bit", "default") + return err + }, + func() error { + var err error + dcgmExporterHealth, err = getDaemonSetReadiness(k8sClient, "dcgm-exporter", "default") + return err + }, + func() error { + var err error + nodeExporterHealth, err = getDaemonSetReadiness(k8sClient, "node-exporter", "default") + return err + }, + func() error { + var err error + statsdExporterHealth, err = getDeploymentReadiness(k8sClient, "prometheus-statsd-exporter", "default") + return err + }, + func() error { + var err error + eventExporterHealth, err = getDeploymentReadiness(k8sClient, "event-exporter", "default") + return err + }, + func() error { + var err error + kubeStateMetricsHealth, err = getDeploymentReadiness(k8sClient, "kube-state-metrics", "default") + return err + }, + ); err != nil { + return ClusterHealth{}, err + } + + return ClusterHealth{ + Operator: operatorHealth, + ControllerManager: controllerManagerHealth, + Prometheus: prometheusHealth, + Autoscaler: autoscalerHealth, + Activator: activatorHealth, + Grafana: grafanaHealth, + OperatorGateway: operatorGatewayHealth, + APIsGateway: apisGatewayHealth, + ClusterAutoscaler: clusterAutoscalerHealth, + OperatorLoadBalancer: operatorLoadBalancerHealth, + APIsLoadBalancer: apisLoadBalancerHealth, + FluentBit: fluentBitHealth, + NodeExporter: nodeExporterHealth, + DCGMExporter: dcgmExporterHealth, + StatsDExporter: statsdExporterHealth, + EventExporter: eventExporterHealth, + KubeStateMetrics: kubeStateMetricsHealth, + }, nil +} + +func GetWarnings(k8sClient *k8s.Client) (ClusterWarnings, error) { + var ( + prometheusMemorySaturationWarn string + ) + + saturation, err := getPodMemorySaturation(k8sClient, "prometheus-prometheus-0", "default") + if err != nil { + return ClusterWarnings{}, err + } + + if saturation >= 0.7 { + prometheusMemorySaturationWarn = fmt.Sprintf("memory usage is critically high (%.1f%%)", saturation*100) + } + + return ClusterWarnings{ + Prometheus: prometheusMemorySaturationWarn, + }, nil +} + +func getDeploymentReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) { + ctx := context.Background() + var deployment kapps.Deployment + if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{ + Namespace: namespace, + Name: name, + }, &deployment); err != nil { + if kerrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return deployment.Status.ReadyReplicas > 0, nil +} + +func getStatefulSetReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) { + ctx := context.Background() + var statefulSet kapps.StatefulSet + if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{ + Namespace: namespace, + Name: name, + }, &statefulSet); err != nil { + if kerrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return statefulSet.Status.ReadyReplicas > 0, nil +} + +func getDaemonSetReadiness(k8sClient *k8s.Client, name, namespace string) (bool, error) { + ctx := context.Background() + var daemonSet kapps.DaemonSet + if err := k8sClient.Get(ctx, ctrlclient.ObjectKey{ + Namespace: namespace, + Name: name, + }, &daemonSet); err != nil { + if kerrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return daemonSet.Status.NumberReady == daemonSet.Status.CurrentNumberScheduled, nil +} + +func getLoadBalancerHealth(awsClient *awslib.Client, clusterName string, loadBalancerName string) (bool, error) { + loadBalancer, err := awsClient.FindLoadBalancer(map[string]string{ + clusterconfig.ClusterNameTag: clusterName, + "cortex.dev/load-balancer": loadBalancerName, + }) + if err != nil { + return false, errors.Wrap(err, fmt.Sprintf("unable to locate %s load balancer", loadBalancerName)) + } + + if loadBalancer.State == nil || loadBalancer.State.Code == nil { + return false, nil + } + + return *loadBalancer.State.Code == elbv2.LoadBalancerStateEnumActive, nil +} + +func getPodMemorySaturation(k8sClient *k8s.Client, podName, namespace string) (float64, error) { + ctx := context.Background() + pod, err := k8sClient.GetPod(podName) + if err != nil { + return 0, err + } + + metricsClient, err := kmetrics.NewForConfig(k8sClient.RestConfig) + if err != nil { + return 0, err + } + podMetrics, err := metricsClient.MetricsV1beta1().PodMetricses(namespace).Get(ctx, podName, kmeta.GetOptions{}) + if err != nil { + return 0, err + } + + var totalMememoryUsage kresource.Quantity + for _, container := range podMetrics.Containers { + memory := container.Usage.Memory() + if memory != nil { + totalMememoryUsage.Add(*container.Usage.Memory()) + } + } + + node, err := k8sClient.ClientSet().CoreV1().Nodes().Get(ctx, pod.Spec.NodeName, kmeta.GetOptions{}) + if err != nil { + return 0, err + } + + nodeMemory := node.Status.Allocatable.Memory() + + memRatio := totalMememoryUsage.AsApproximateFloat64() / nodeMemory.AsApproximateFloat64() + + return memRatio, nil +} diff --git a/pkg/lib/console/format.go b/pkg/lib/console/format.go index d8a318172c..5ca478c8b7 100644 --- a/pkg/lib/console/format.go +++ b/pkg/lib/console/format.go @@ -22,6 +22,15 @@ import ( var _bold = color.New(color.Bold).SprintFunc() +// Bold returns a string formatted in bold func Bold(a ...interface{}) string { return _bold(a...) } + +// BoolColor converts a boolean into a colored string (green: true, red: false) +func BoolColor(b bool) string { + if b { + return color.GreenString("%t", b) + } + return color.RedString("%t", b) +}