diff --git a/hack/cl2/README.md b/hack/cl2/README.md new file mode 100644 index 00000000..3585bb11 --- /dev/null +++ b/hack/cl2/README.md @@ -0,0 +1,24 @@ +This folder contains a list of [clusterloader2](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) configuration manifests for large scale testing. + +## Prerequisites + +Follow [clusterloader2 GETTING STARTED](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) to clone the `perf-tests` repository. + +## Execute Tests + +Run `clusterloader2` under `perf-tests/clusterloader2` directory: +```bash +go run cmd/clusterloader.go --testconfig= --provider=local --kubeconfig= --v=2 --enable-exec-service=false +``` +We need to set `--enable-exec-service=false` to prevent creating agnostic deployment on the hub cluster as hub +cluster does not allow pod creation. + +## Cleanup Resources After Tests + +By default, clusterloader2 automatically deletes all the generated namespaces after the tests. +In addition, we also want to delete all the CRPs created. To facilitate cleanup process, run the simple script +provided in this folder: +``` +export KUBECONFIG= +./cleanup.sh +``` \ No newline at end of file diff --git a/hack/cl2/atm_scale_test_config.yaml b/hack/cl2/atm_scale_test_config.yaml new file mode 100644 index 00000000..20a059dd --- /dev/null +++ b/hack/cl2/atm_scale_test_config.yaml @@ -0,0 +1,151 @@ +name: atm_scale_test + +{{$duration := "40m"}} +{{$namespaceCount := 3}} +{{$count := 20}} + +namespace: + number: {{$namespaceCount}} + prefix: atm-test-ns + +tuningSets: +- name: Uniform10qps + qpsLoad: + qps: 10 +- name: CleanupQps + qpsLoad: + qps: 0.2 # 1 deletion every 5 seconds + +steps: +- name: Create nginx Deployments, Services, and ServiceExports + phases: + - namespaceRange: + min: 1 + max: {{$namespaceCount}} + replicasPerNamespace: {{$count}} + tuningSet: Uniform10qps + objectBundle: + - basename: nginx + objectTemplatePath: "manifests/test-nginx-deploy.yaml" + - basename: nginx-svc + objectTemplatePath: "manifests/test-nginx-svc.yaml" + - basename: nginx-svc + objectTemplatePath: "manifests/test-nginx-svc-export.yaml" +- name: Create resourceOverrides + phases: + - namespaceRange: + min: 1 + max: {{$namespaceCount}} + replicasPerNamespace: {{$count}} + tuningSet: Uniform10qps + objectBundle: + - basename: nginx-ro + objectTemplatePath: "manifests/test-nginx-ro.yaml" +- name: Create CRPs + phases: + - replicasPerNamespace: {{$namespaceCount}} + tuningSet: Uniform10qps + objectBundle: + - basename: test-crp + objectTemplatePath: "manifests/test-crp.yaml" +- name: Wait for CRPs to be Ready + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + objectGroup: placement.kubernetes-fleet.io + objectVersion: v1beta1 + objectResource: clusterresourceplacements + timeout: {{$duration}} + successfulConditions: + - ClusterResourcePlacementAvailable=True + failedConditions: + - ClusterResourcePlacementApplied=False + minDesiredObjectCount: {{$namespaceCount}} + maxFailedObjectCount: 0 +- name: Create TrafficManagerProfiles + phases: + - namespaceRange: + min: 1 + max: {{$namespaceCount}} + replicasPerNamespace: 1 + tuningSet: Uniform10qps + objectBundle: + - basename: test-tmp + objectTemplatePath: "manifests/test-tmp.yaml" +- name: Wait for TrafficManagerProfiles to be Programmed + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + namespaceRange: + min: 1 + max: {{$namespaceCount}} + objectGroup: networking.fleet.azure.com + objectVersion: v1beta1 + objectResource: trafficmanagerprofiles + timeout: {{$duration}} + successfulConditions: + - Programmed=True + failedConditions: + - Programmed=False + minDesiredObjectCount: {{$namespaceCount}} + maxFailedObjectCount: 0 +- name: Create TrafficManagerBackends + phases: + - namespaceRange: + min: 1 + max: {{$namespaceCount}} + replicasPerNamespace: {{$count}} + tuningSet: Uniform10qps + objectBundle: + - basename: test-tmb + objectTemplatePath: "manifests/test-tmb.yaml" + templateFillMap: + ProfileName: test-tmp-0 +- name: Wait for TrafficManagerBackends to be Accepted + measurements: + - Identifier: WaitForGenericK8sObjects + Method: WaitForGenericK8sObjects + Params: + namespaceRange: + min: 1 + max: {{$namespaceCount}} + objectGroup: networking.fleet.azure.com + objectVersion: v1beta1 + objectResource: trafficmanagerbackends + timeout: {{$duration}} + successfulConditions: + - Accepted=True + failedConditions: + - Accepted=False + minDesiredObjectCount: {{MultiplyInt $count $namespaceCount}} + maxFailedObjectCount: 0 +# By default, the created namespaces together with all resources created within the namespaces +# will be deleted at the end of the test automatically. +# We add this step because the namespace cleanup happens all at once, +# and it may cause throttling when deleting the ATM endpoints. +# We add this step to delete the TrafficManagerBackends and TrafficManagerProfiles +# (and thus ATM endpoints and profiles) slowly, (1 per 5 seconds), to avoid being throttled, +- name: Delete TrafficManagerBackends + phases: + - namespaceRange: + min: 1 + max: {{$namespaceCount}} + replicasPerNamespace: 0 + tuningSet: CleanupQps + objectBundle: + - basename: test-tmb + objectTemplatePath: "manifests/test-tmb.yaml" + templateFillMap: + ProfileName: test-tmp-0 +- name: Delete TrafficManagerProfiles + phases: + - namespaceRange: + min: 1 + max: {{$namespaceCount}} + replicasPerNamespace: 0 + tuningSet: CleanupQps + objectBundle: + - basename: test-tmp + objectTemplatePath: "manifests/test-tmp.yaml" diff --git a/hack/cl2/cleanup.sh b/hack/cl2/cleanup.sh new file mode 100755 index 00000000..eea5120b --- /dev/null +++ b/hack/cl2/cleanup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +: "${KUBECONFIG:?Environment variable KUBECONFIG must be set}" + +echo "Deleting all CRPs generated during the load test from the hub cluster..." +kubectl delete crp -l test=cl2-test diff --git a/hack/cl2/manifests/test-crp.yaml b/hack/cl2/manifests/test-crp.yaml new file mode 100644 index 00000000..35c5a4a3 --- /dev/null +++ b/hack/cl2/manifests/test-crp.yaml @@ -0,0 +1,19 @@ +apiVersion: placement.kubernetes-fleet.io/v1beta1 +kind: ClusterResourcePlacement +metadata: + name: {{.Name}} + labels: + test: cl2-test +spec: + resourceSelectors: + - group: "" + kind: Namespace + name: atm-test-ns-{{AddInt .Index 1}} + version: v1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + maxSurge: 25% + unavailablePeriodSeconds: 60 + revisionHistoryLimit: 15 diff --git a/hack/cl2/manifests/test-nginx-deploy.yaml b/hack/cl2/manifests/test-nginx-deploy.yaml new file mode 100644 index 00000000..746d9184 --- /dev/null +++ b/hack/cl2/manifests/test-nginx-deploy.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.Name}} +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + index: "{{.Index}}" + template: + metadata: + labels: + app: nginx + index: "{{.Index}}" + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/hack/cl2/manifests/test-nginx-ro.yaml b/hack/cl2/manifests/test-nginx-ro.yaml new file mode 100644 index 00000000..dcc0aeda --- /dev/null +++ b/hack/cl2/manifests/test-nginx-ro.yaml @@ -0,0 +1,19 @@ +apiVersion: placement.kubernetes-fleet.io/v1alpha1 +kind: ResourceOverride +metadata: + name: {{.Name}} +spec: + resourceSelectors: + - group: "" + kind: Service + version: v1 + name: nginx-svc-{{.Index}} + policy: + overrideRules: + - clusterSelector: + clusterSelectorTerms: [] + jsonPatchOverrides: + - op: add + path: /metadata/annotations + value: + {"service.beta.kubernetes.io/azure-dns-label-name":"test-${MEMBER-CLUSTER-NAME}-{{.Namespace}}-nginx-{{.Index}}"} diff --git a/hack/cl2/manifests/test-nginx-svc-export.yaml b/hack/cl2/manifests/test-nginx-svc-export.yaml new file mode 100644 index 00000000..77aca817 --- /dev/null +++ b/hack/cl2/manifests/test-nginx-svc-export.yaml @@ -0,0 +1,4 @@ +apiVersion: networking.fleet.azure.com/v1alpha1 +kind: ServiceExport +metadata: + name: {{.Name}} diff --git a/hack/cl2/manifests/test-nginx-svc.yaml b/hack/cl2/manifests/test-nginx-svc.yaml new file mode 100644 index 00000000..976e2ae3 --- /dev/null +++ b/hack/cl2/manifests/test-nginx-svc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.Name}} +spec: + selector: + app: nginx + index: "{{.Index}}" + ports: + - protocol: TCP + port: 80 + targetPort: 80 + type: LoadBalancer diff --git a/hack/cl2/manifests/test-tmb.yaml b/hack/cl2/manifests/test-tmb.yaml new file mode 100644 index 00000000..e3b7007f --- /dev/null +++ b/hack/cl2/manifests/test-tmb.yaml @@ -0,0 +1,10 @@ +apiVersion: networking.fleet.azure.com/v1beta1 +kind: TrafficManagerBackend +metadata: + name: {{.Name}} +spec: + profile: + name: {{.ProfileName}} + backend: + name: nginx-svc-{{.Index}} + weight: 100 diff --git a/hack/cl2/manifests/test-tmp.yaml b/hack/cl2/manifests/test-tmp.yaml new file mode 100644 index 00000000..0f7db107 --- /dev/null +++ b/hack/cl2/manifests/test-tmp.yaml @@ -0,0 +1,8 @@ +apiVersion: networking.fleet.azure.com/v1beta1 +kind: TrafficManagerProfile +metadata: + name: {{.Name}} +spec: + resourceGroup: "atm-scale-test-rg" + monitorConfig: + port: 80