test: add clusterloader tests for atm integration

hack/cl2/README.md

@@ -0,0 +1,24 @@
+This folder contains a list of [clusterloader2](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) configuration manifests for large scale testing.
+
+## Prerequisites
+
+Follow [clusterloader2 GETTING STARTED](https://github.com/kubernetes/perf-tests/blob/master/clusterloader2/docs/GETTING_STARTED.md) to clone the `perf-tests` repository.
+
+## Execute Tests
+
+Run `clusterloader2` under `perf-tests/clusterloader2` directory:
+```bash
+go run cmd/clusterloader.go --testconfig=<path to the test yaml> --provider=local --kubeconfig=<path to the hub cluster kubeconfig> --v=2 --enable-exec-service=false
+```
+We need to set `--enable-exec-service=false` to prevent creating agnostic deployment on the hub cluster as hub
+cluster does not allow pod creation.
+
+## Cleanup Resources After Tests
+
+By default, clusterloader2 automatically deletes all the generated namespaces after the tests.
+In addition, we also want to delete all the CRPs created. To facilitate cleanup process, run the simple script
+provided in this folder:
+```
+export KUBECONFIG=<your path to the kubeconfig file>
+./cleanup.sh
+```

hack/cl2/atm_scale_test_config.yaml

@@ -0,0 +1,151 @@
+name: atm_scale_test
+
+{{$duration := "40m"}}
+{{$namespaceCount := 3}}
+{{$count := 20}}
+
+namespace:
+  number: {{$namespaceCount}}
+  prefix: atm-test-ns
+
+tuningSets:
+- name: Uniform10qps
+  qpsLoad:
+    qps: 10
+- name: CleanupQps
+  qpsLoad:
+    qps: 0.2 # 1 deletion every 5 seconds
+
+steps:
+- name: Create nginx Deployments, Services, and ServiceExports
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaceCount}}
+    replicasPerNamespace: {{$count}}
+    tuningSet: Uniform10qps
+    objectBundle:
+    - basename: nginx
+      objectTemplatePath: "manifests/test-nginx-deploy.yaml"
+    - basename: nginx-svc
+      objectTemplatePath: "manifests/test-nginx-svc.yaml"
+    - basename: nginx-svc
+      objectTemplatePath: "manifests/test-nginx-svc-export.yaml"
+- name: Create resourceOverrides
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaceCount}}
+    replicasPerNamespace: {{$count}}
+    tuningSet: Uniform10qps
+    objectBundle:
+    - basename: nginx-ro
+      objectTemplatePath: "manifests/test-nginx-ro.yaml"
+- name: Create CRPs
+  phases:
+  - replicasPerNamespace: {{$namespaceCount}}
+    tuningSet: Uniform10qps
+    objectBundle:
+    - basename: test-crp
+      objectTemplatePath: "manifests/test-crp.yaml"
+- name: Wait for CRPs to be Ready
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      objectGroup: placement.kubernetes-fleet.io
+      objectVersion: v1beta1
+      objectResource: clusterresourceplacements
+      timeout: {{$duration}}
+      successfulConditions: 
+      - ClusterResourcePlacementAvailable=True
+      failedConditions:
+      - ClusterResourcePlacementApplied=False
+      minDesiredObjectCount: {{$namespaceCount}}
+      maxFailedObjectCount: 0
+- name: Create TrafficManagerProfiles
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaceCount}}
+    replicasPerNamespace: 1
+    tuningSet: Uniform10qps
+    objectBundle:
+    - basename: test-tmp
+      objectTemplatePath: "manifests/test-tmp.yaml"
+- name: Wait for TrafficManagerProfiles to be Programmed
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      namespaceRange:
+        min: 1
+        max: {{$namespaceCount}}
+      objectGroup: networking.fleet.azure.com
+      objectVersion: v1beta1
+      objectResource: trafficmanagerprofiles
+      timeout: {{$duration}}
+      successfulConditions: 
+      - Programmed=True
+      failedConditions:
+      - Programmed=False
+      minDesiredObjectCount: {{$namespaceCount}}
+      maxFailedObjectCount: 0
+- name: Create TrafficManagerBackends
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaceCount}}
+    replicasPerNamespace: {{$count}}
+    tuningSet: Uniform10qps
+    objectBundle:
+    - basename: test-tmb
+      objectTemplatePath: "manifests/test-tmb.yaml"
+      templateFillMap:
+        ProfileName: test-tmp-0
+- name: Wait for TrafficManagerBackends to be Accepted
+  measurements:
+  - Identifier: WaitForGenericK8sObjects
+    Method: WaitForGenericK8sObjects
+    Params:
+      namespaceRange:
+        min: 1
+        max: {{$namespaceCount}}
+      objectGroup: networking.fleet.azure.com
+      objectVersion: v1beta1
+      objectResource: trafficmanagerbackends
+      timeout: {{$duration}}
+      successfulConditions: 
+      - Accepted=True
+      failedConditions:
+      - Accepted=False
+      minDesiredObjectCount: {{MultiplyInt $count $namespaceCount}}
+      maxFailedObjectCount: 0
+# By default, the created namespaces together with all resources created within the namespaces
+# will be deleted at the end of the test automatically. 
+# We add this step because the namespace cleanup happens all at once, 
+# and it may cause throttling when deleting the ATM endpoints.
+# We add this step to delete the TrafficManagerBackends and TrafficManagerProfiles 
+# (and thus ATM endpoints and profiles) slowly, (1 per 5 seconds), to avoid being throttled,
+- name: Delete TrafficManagerBackends
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaceCount}}
+    replicasPerNamespace: 0
+    tuningSet: CleanupQps
+    objectBundle:
+    - basename: test-tmb
+      objectTemplatePath: "manifests/test-tmb.yaml"
+      templateFillMap:
+        ProfileName: test-tmp-0
+- name: Delete TrafficManagerProfiles
+  phases:
+  - namespaceRange:
+      min: 1
+      max: {{$namespaceCount}}
+    replicasPerNamespace: 0
+    tuningSet: CleanupQps
+    objectBundle:
+    - basename: test-tmp
+      objectTemplatePath: "manifests/test-tmp.yaml"

hack/cl2/cleanup.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+: "${KUBECONFIG:?Environment variable KUBECONFIG must be set}"
+
+echo "Deleting all CRPs generated during the load test from the hub cluster..."
+kubectl delete crp -l test=cl2-test

hack/cl2/manifests/test-crp.yaml

@@ -0,0 +1,19 @@
+apiVersion: placement.kubernetes-fleet.io/v1beta1
+kind: ClusterResourcePlacement
+metadata:
+  name: {{.Name}}
+  labels:
+    test: cl2-test
+spec:
+  resourceSelectors:
+    - group: ""
+      kind: Namespace
+      name: atm-test-ns-{{AddInt .Index 1}}
+      version: v1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 25%
+      maxSurge: 25%
+      unavailablePeriodSeconds: 60
+  revisionHistoryLimit: 15

hack/cl2/manifests/test-nginx-deploy.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{.Name}}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx
+      index: "{{.Index}}"
+  template:
+    metadata:
+      labels:
+        app: nginx
+        index: "{{.Index}}"
+    spec:
+      containers:
+        - name: nginx
+          image: nginx
+          ports:
+            - containerPort: 80

hack/cl2/manifests/test-nginx-ro.yaml

@@ -0,0 +1,19 @@
+apiVersion: placement.kubernetes-fleet.io/v1alpha1
+kind: ResourceOverride
+metadata:
+  name: {{.Name}}
+spec:
+  resourceSelectors:
+    -  group: ""
+       kind: Service
+       version: v1
+       name: nginx-svc-{{.Index}}
+  policy:
+    overrideRules:
+      - clusterSelector:
+          clusterSelectorTerms: []
+        jsonPatchOverrides:
+          - op: add
+            path: /metadata/annotations
+            value:
+              {"service.beta.kubernetes.io/azure-dns-label-name":"test-${MEMBER-CLUSTER-NAME}-{{.Namespace}}-nginx-{{.Index}}"}

hack/cl2/manifests/test-nginx-svc-export.yaml

@@ -0,0 +1,4 @@
+apiVersion: networking.fleet.azure.com/v1alpha1
+kind: ServiceExport
+metadata:
+  name: {{.Name}}

hack/cl2/manifests/test-nginx-svc.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{.Name}}
+spec:
+  selector:
+    app: nginx
+    index: "{{.Index}}"
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+  type: LoadBalancer

hack/cl2/manifests/test-tmb.yaml

@@ -0,0 +1,10 @@
+apiVersion: networking.fleet.azure.com/v1beta1
+kind: TrafficManagerBackend
+metadata:
+  name: {{.Name}}
+spec:
+  profile:
+    name: {{.ProfileName}}
+  backend:
+    name: nginx-svc-{{.Index}}
+  weight: 100

hack/cl2/manifests/test-tmp.yaml

@@ -0,0 +1,8 @@
+apiVersion: networking.fleet.azure.com/v1beta1
+kind: TrafficManagerProfile
+metadata:
+  name: {{.Name}}
+spec:
+  resourceGroup: "atm-scale-test-rg"
+  monitorConfig:
+    port: 80