Enhance Windows host interface creation timeout

Introduce an exponential backoff retry mechanism for host interface
creation on Windows. Under high pod churn or node rollout stress,
Windows HNS can be slow to create the virtual network adapter, causing
the hardcoded 1-minute timeout to fail and leave OVS ports permanently
misconfigured.

Also add a stress E2E test to verify Antrea agent rollout recovery
under high pod density on Windows.

Signed-off-by: Shuyang Xin <shuyang.xin@broadcom.com>

Author: XinShuYang

ci/jenkins/test.sh

@@ -746,7 +746,7 @@ function run_e2e_windows {
     mkdir -p `pwd`/antrea-test-logs
 
     echo "====== Run test with e2e test ======"
-    go test -v antrea.io/antrea/v2/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=50m --prometheus
+    go test -v antrea.io/antrea/v2/test/e2e --logs-export-dir `pwd`/antrea-test-logs --provider remote -timeout=500m --prometheus
     if [[ "$?" != "0" ]]; then
         TEST_FAILURE=true
     fi

pkg/agent/cniserver/interface_configuration_windows.go

@@ -466,34 +466,65 @@ func (ic *ifConfigurator) addPostInterfaceCreateHook(containerID, endpointName s
 	go func() {
 		ifaceName := fmt.Sprintf("vEthernet (%s)", endpointName)
 		var err error
-		pollErr := wait.PollUntilContextTimeout(context.TODO(), 100*time.Millisecond, 60*time.Second, true,
-			func(ctx context.Context) (bool, error) {
-				containerAccess.lockContainer(containerID)
-				defer containerAccess.unlockContainer(containerID)
-				currentEP, ok := ic.getEndpoint(endpointName)
-				if !ok {
-					klog.InfoS("HNSEndpoint doesn't exist in cache, exit current goroutine", "HNSEndpoint", endpointName)
-					return true, nil
-				}
-				if currentEP.Id != expectedEP.Id {
-					klog.InfoS("Detected HNSEndpoint change, exit current goroutine", "HNSEndpoint", endpointName)
+		startTime := time.Now()
+
+		// Exponential backoff for host interface creation retries.
+		// We retry up to 3 times (steps) with an initial delay of 2 seconds,
+		// doubling the delay each time (20s -> 40s -> 80s -> 160s -> 320s) with a mild random jitter.
+		backoff := wait.Backoff{
+			Steps:    5,
+			Duration: 20 * time.Second,
+			Factor:   2.0,
+			Jitter:   0.1,
+		}
+
+		attempt := 0
+		backoffErr := wait.ExponentialBackoff(backoff, func() (bool, error) {
+			attempt++
+			klog.V(2).InfoS("Starting poll attempt for host interface", "interface", ifaceName, "attempt", attempt)
+
+			pollStart := time.Now()
+			pollErr := wait.PollUntilContextTimeout(context.TODO(), 100*time.Millisecond, 60*time.Second, true,
+				func(ctx context.Context) (bool, error) {
+					containerAccess.lockContainer(containerID)
+					defer containerAccess.unlockContainer(containerID)
+					currentEP, ok := ic.getEndpoint(endpointName)
+					if !ok {
+						klog.InfoS("HNSEndpoint doesn't exist in cache, exit current goroutine", "HNSEndpoint", endpointName)
+						return true, nil
+					}
+					if currentEP.Id != expectedEP.Id {
+						klog.InfoS("Detected HNSEndpoint change, exit current goroutine", "HNSEndpoint", endpointName)
+						return true, nil
+					}
+					if !hostInterfaceExistsFunc(ifaceName) {
+						klog.V(4).InfoS("Waiting for interface to be created", "interface", ifaceName)
+						return false, nil
+					}
+					if err = hook(); err != nil {
+						return false, err
+					}
 					return true, nil
-				}
-				if !hostInterfaceExistsFunc(ifaceName) {
-					klog.V(2).InfoS("Waiting for interface to be created", "interface", ifaceName)
-					return false, nil
-				}
-				if err = hook(); err != nil {
-					return false, err
-				}
+				})
+
+			pollDuration := time.Since(pollStart)
+			totalDuration := time.Since(startTime)
+
+			if pollErr == nil {
+				klog.InfoS("Successfully configured host interface", "interface", ifaceName, "attempt", attempt, "pollDuration", pollDuration.String(), "totalDuration", totalDuration.String())
 				return true, nil
-			})
+			}
+
+			klog.InfoS("Failed to wait for host interface creation, will retry with backoff", "interface", ifaceName, "attempt", attempt, "pollDuration", pollDuration.String(), "totalDuration", totalDuration.String(), "err", err, "pollErr", pollErr)
+			return false, nil
+		})
 
-		if pollErr != nil {
+		if backoffErr != nil {
+			totalDuration := time.Since(startTime)
 			if err != nil {
-				klog.ErrorS(err, "Failed to execute postInterfaceCreateHook", "interface", ifaceName)
+				klog.ErrorS(err, "Failed to execute postInterfaceCreateHook after all retries", "interface", ifaceName, "totalDuration", totalDuration.String())
 			} else {
-				klog.ErrorS(pollErr, "Failed to wait for host interface creation in 1min", "interface", ifaceName)
+				klog.ErrorS(backoffErr, "Failed to wait for host interface creation after all retries", "interface", ifaceName, "totalDuration", totalDuration.String())
 			}
 		}
 	}()

test/e2e/framework.go

@@ -1415,7 +1415,7 @@ func (data *TestData) waitForAntreaDaemonSetPods(timeout time.Duration) error {
 	})
 	if wait.Interrupted(err) {
 		_, stdout, _, _ := data.provider.RunCommandOnNode(controlPlaneNodeName(), fmt.Sprintf("kubectl -n %s describe pod", antreaNamespace))
-		return fmt.Errorf("antrea-agent DaemonSet not ready within %v; kubectl describe pod output: %v", defaultTimeout, stdout)
+		return fmt.Errorf("antrea-agent DaemonSet not ready within %v; kubectl describe pod output: %v", timeout, stdout)
 	} else if err != nil {
 		return err
 	}
@@ -1483,6 +1483,10 @@ func (data *TestData) CreateClient(kubeconfigPath string) error {
 	if err != nil {
 		return fmt.Errorf("error when building kube config: %v", err)
 	}
+	// Increase QPS and Burst to prevent client-side throttling during high-concurrency stress tests.
+	kubeConfig.QPS = 100
+	kubeConfig.Burst = 200
+
 	clientset, err := kubernetes.NewForConfig(kubeConfig)
 	if err != nil {
 		return fmt.Errorf("error when creating kubernetes client: %v", err)
@@ -1581,6 +1585,7 @@ type PodBuilder struct {
 	ResourceLimits     corev1.ResourceList
 	ReadinessProbe     *corev1.Probe
 	DnsConfig          *corev1.PodDNSConfig
+	RestartPolicy      corev1.RestartPolicy
 }
 
 func NewPodBuilder(name, ns, image string) *PodBuilder {
@@ -1718,6 +1723,11 @@ func (b *PodBuilder) WithReadinessProbe(probe *corev1.Probe) *PodBuilder {
 	return b
 }
 
+func (b *PodBuilder) WithRestartPolicy(policy corev1.RestartPolicy) *PodBuilder {
+	b.RestartPolicy = policy
+	return b
+}
+
 // WithCustomDNSConfig adds a custom DNS Configuration to the Pod spec.
 // It ensures that the DNSPolicy is set to 'None' and assigns the provided DNSConfig.
 func (b *PodBuilder) WithCustomDNSConfig(dnsConfig *corev1.PodDNSConfig) *PodBuilder {
@@ -1730,6 +1740,10 @@ func (b *PodBuilder) Create(data *TestData) error {
 	if containerName == "" {
 		containerName = getImageName(b.Image)
 	}
+	policy := b.RestartPolicy
+	if policy == "" {
+		policy = corev1.RestartPolicyNever
+	}
 	podSpec := corev1.PodSpec{
 		Containers: []corev1.Container{
 			{
@@ -1752,7 +1766,7 @@ func (b *PodBuilder) Create(data *TestData) error {
 			},
 		},
 		Volumes:            b.Volumes,
-		RestartPolicy:      corev1.RestartPolicyNever,
+		RestartPolicy:      policy,
 		HostNetwork:        b.HostNetwork,
 		ServiceAccountName: b.ServiceAccountName,
 		// Set it to 1s for immediate shutdown to reduce test run time and to avoid affecting subsequent tests.