Support nftables in Node's host network for AntreaProxy (#7545)

Signed-off-by: Hongliang Liu <[email protected]>

Author: hongliangl

.github/workflows/kind.yml

@@ -468,7 +468,7 @@ jobs:
         retention-days: 30
 
   test-e2e-hybrid-non-default:
-    name: E2e tests on a Kind cluster on Linux (hybrid) with non default values (proxyAll=true)
+    name: E2e tests on a Kind cluster on Linux (hybrid) with non default values (proxyAll=true, kube-proxy-mode=nftables)
     needs: [ build-antrea-coverage-image ]
     runs-on: [ ubuntu-latest ]
     steps:
@@ -504,6 +504,9 @@ jobs:
             --encap-mode hybrid \
             --coverage \
             --proxy-all \
+            --host-network-mode nftables \
+            --kube-proxy-mode nftables \
+            --feature-gates NFTablesHostNetworkMode=true \
             --skip mode-irrelevant
       - name: Tar coverage files
         run: tar -czf test-e2e-hybrid-non-default-coverage.tar.gz test-e2e-hybrid-non-default-coverage

build/charts/antrea/README.md

@@ -105,6 +105,7 @@ Kubernetes: `>= 1.23.0-0`
 | fqdnCacheMinTTL | int | `0` | fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included in datapath rules for as long as the application caches them. Ideally, this value should be set to the maximum caching duration across all applications. |
 | hostGateway | string | `"antrea-gw0"` | Name of the interface antrea-agent will create and use for host <-> Pod communication. |
 | hostNetworkAcceleration.enable | bool | `true` | Enable to accelerate Pod-to-Pod traffic in the Node's host network using nftables flowtable when traffic mode is noEncap or hybrid. |
+| hostNetworkMode | string | `"iptables"` | Determines how antrea-agent implements Node host network netfilter rules required by functionalities and features, using either "iptables" or "nftables". The default value is "iptables". If "nftables" is specified, the NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start. Currently, nftables support is limited to the following features:   - AntreaProxy (proxyAll) |
 | image | object | `{}` | Container image to use for Antrea components. DEPRECATED: use agentImage and controllerImage instead. |
 | ipsec.authenticationMode | string | `"psk"` | The authentication mode to use for IPsec. Must be one of "psk" or "cert". |
 | ipsec.csrSigner.autoApprove | bool | `true` | Enable auto approval of Antrea signer for IPsec certificates. |

build/charts/antrea/conf/antrea-agent.conf

@@ -83,6 +83,11 @@ featureGates:
 # remote BGP peers.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "BGPPolicy" "default" false) }}
 
+# Enable Antrea to use nftables instead of iptables to enforce netfilter rules in the Node's host network for the
+# functionalities and features that rely on netfilter. Currently, nftables support is limited to the following features:
+# - AntreaProxy (proxyAll)
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "NFTablesHostNetworkMode" "default" false) }}
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}
@@ -496,3 +501,11 @@ hostNetworkAcceleration:
   # noEncap or hybrid.
   enable: {{ .enable }}
 {{- end }}
+
+# HostNetworkMode determines how antrea-agent implements Node host network netfilter rules required by Antrea
+# functionalities and features, using either "iptables" or "nftables". The default value is "iptables". If "nftables"
+# is specified, the NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start.
+# If the above condition is met but nftables is not supported or unavailable on the Node, antrea-agent will also fail
+# to start. Currently, nftables support is limited to the following features:
+#   - AntreaProxy (proxyAll)
+hostNetworkMode: {{ .Values.hostNetworkMode | quote }}

build/charts/antrea/values.yaml

@@ -489,3 +489,10 @@ hostNetworkAcceleration:
   # -- Enable to accelerate Pod-to-Pod traffic in the Node's host network using nftables flowtable when traffic mode is
   # noEncap or hybrid.
   enable: true
+
+# -- Determines how antrea-agent implements Node host network netfilter rules required by functionalities and features,
+# using either "iptables" or "nftables". The default value is "iptables". If "nftables" is specified, the
+# NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start. Currently, nftables
+# support is limited to the following features:
+#   - AntreaProxy (proxyAll)
+hostNetworkMode: "iptables"

build/yamls/antrea-aks.yml

@@ -4296,6 +4296,11 @@ data:
     # remote BGP peers.
     #  BGPPolicy: false
 
+    # Enable Antrea to use nftables instead of iptables to enforce netfilter rules in the Node's host network for the
+    # functionalities and features that rely on netfilter. Currently, nftables support is limited to the following features:
+    # - AntreaProxy (proxyAll)
+    #  NFTablesHostNetworkMode: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -4661,6 +4666,14 @@ data:
       # Enable to accelerate Pod-to-Pod traffic in the Node's host network using nftables flowtable when traffic mode is
       # noEncap or hybrid.
       enable: true
+
+    # HostNetworkMode determines how antrea-agent implements Node host network netfilter rules required by Antrea
+    # functionalities and features, using either "iptables" or "nftables". The default value is "iptables". If "nftables"
+    # is specified, the NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start.
+    # If the above condition is met but nftables is not supported or unavailable on the Node, antrea-agent will also fail
+    # to start. Currently, nftables support is limited to the following features:
+    #   - AntreaProxy (proxyAll)
+    hostNetworkMode: "iptables"
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -5700,7 +5713,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 3102c632b42c98e161207364e65fa751a0b0831e87d878a3242c013c1b7d3c2b
+        checksum/config: 288c5f3c4ff78c08520f7ddd0ba8547774d0fe9724ed649e32fa6e4f5ab6830c
       labels:
         app: antrea
         component: antrea-agent
@@ -5948,7 +5961,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 3102c632b42c98e161207364e65fa751a0b0831e87d878a3242c013c1b7d3c2b
+        checksum/config: 288c5f3c4ff78c08520f7ddd0ba8547774d0fe9724ed649e32fa6e4f5ab6830c
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-eks.yml

@@ -4292,6 +4292,11 @@ data:
     # remote BGP peers.
     #  BGPPolicy: false
 
+    # Enable Antrea to use nftables instead of iptables to enforce netfilter rules in the Node's host network for the
+    # functionalities and features that rely on netfilter. Currently, nftables support is limited to the following features:
+    # - AntreaProxy (proxyAll)
+    #  NFTablesHostNetworkMode: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -4657,6 +4662,14 @@ data:
       # Enable to accelerate Pod-to-Pod traffic in the Node's host network using nftables flowtable when traffic mode is
       # noEncap or hybrid.
       enable: true
+
+    # HostNetworkMode determines how antrea-agent implements Node host network netfilter rules required by Antrea
+    # functionalities and features, using either "iptables" or "nftables". The default value is "iptables". If "nftables"
+    # is specified, the NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start.
+    # If the above condition is met but nftables is not supported or unavailable on the Node, antrea-agent will also fail
+    # to start. Currently, nftables support is limited to the following features:
+    #   - AntreaProxy (proxyAll)
+    hostNetworkMode: "iptables"
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -5696,7 +5709,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 3102c632b42c98e161207364e65fa751a0b0831e87d878a3242c013c1b7d3c2b
+        checksum/config: 288c5f3c4ff78c08520f7ddd0ba8547774d0fe9724ed649e32fa6e4f5ab6830c
       labels:
         app: antrea
         component: antrea-agent
@@ -5945,7 +5958,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 3102c632b42c98e161207364e65fa751a0b0831e87d878a3242c013c1b7d3c2b
+        checksum/config: 288c5f3c4ff78c08520f7ddd0ba8547774d0fe9724ed649e32fa6e4f5ab6830c
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-gke.yml

@@ -4292,6 +4292,11 @@ data:
     # remote BGP peers.
     #  BGPPolicy: false
 
+    # Enable Antrea to use nftables instead of iptables to enforce netfilter rules in the Node's host network for the
+    # functionalities and features that rely on netfilter. Currently, nftables support is limited to the following features:
+    # - AntreaProxy (proxyAll)
+    #  NFTablesHostNetworkMode: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -4657,6 +4662,14 @@ data:
       # Enable to accelerate Pod-to-Pod traffic in the Node's host network using nftables flowtable when traffic mode is
       # noEncap or hybrid.
       enable: true
+
+    # HostNetworkMode determines how antrea-agent implements Node host network netfilter rules required by Antrea
+    # functionalities and features, using either "iptables" or "nftables". The default value is "iptables". If "nftables"
+    # is specified, the NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start.
+    # If the above condition is met but nftables is not supported or unavailable on the Node, antrea-agent will also fail
+    # to start. Currently, nftables support is limited to the following features:
+    #   - AntreaProxy (proxyAll)
+    hostNetworkMode: "iptables"
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -5687,7 +5700,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: cbbcacc8999c945122b022df66a13638197a09dbd45eaa1120c1549c2a43ef3a
+        checksum/config: 5e5d2ddcb67315f41931c275be9cb3df79cb82bd82b86390408b9aae2ddc1701
       labels:
         app: antrea
         component: antrea-agent
@@ -5933,7 +5946,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: cbbcacc8999c945122b022df66a13638197a09dbd45eaa1120c1549c2a43ef3a
+        checksum/config: 5e5d2ddcb67315f41931c275be9cb3df79cb82bd82b86390408b9aae2ddc1701
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-ipsec.yml

@@ -4305,6 +4305,11 @@ data:
     # remote BGP peers.
     #  BGPPolicy: false
 
+    # Enable Antrea to use nftables instead of iptables to enforce netfilter rules in the Node's host network for the
+    # functionalities and features that rely on netfilter. Currently, nftables support is limited to the following features:
+    # - AntreaProxy (proxyAll)
+    #  NFTablesHostNetworkMode: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -4670,6 +4675,14 @@ data:
       # Enable to accelerate Pod-to-Pod traffic in the Node's host network using nftables flowtable when traffic mode is
       # noEncap or hybrid.
       enable: true
+
+    # HostNetworkMode determines how antrea-agent implements Node host network netfilter rules required by Antrea
+    # functionalities and features, using either "iptables" or "nftables". The default value is "iptables". If "nftables"
+    # is specified, the NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start.
+    # If the above condition is met but nftables is not supported or unavailable on the Node, antrea-agent will also fail
+    # to start. Currently, nftables support is limited to the following features:
+    #   - AntreaProxy (proxyAll)
+    hostNetworkMode: "iptables"
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -5700,7 +5713,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5ff1f5f4019866e71c486257fd6ad4fb867855ba3c7bd603bfce7c4d77f16116
+        checksum/config: 81878499e95213bf2ccdbc6511bf7f61a84126783eda2cff31f21eb8109315e0
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -5992,7 +6005,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5ff1f5f4019866e71c486257fd6ad4fb867855ba3c7bd603bfce7c4d77f16116
+        checksum/config: 81878499e95213bf2ccdbc6511bf7f61a84126783eda2cff31f21eb8109315e0
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea.yml

@@ -4292,6 +4292,11 @@ data:
     # remote BGP peers.
     #  BGPPolicy: false
 
+    # Enable Antrea to use nftables instead of iptables to enforce netfilter rules in the Node's host network for the
+    # functionalities and features that rely on netfilter. Currently, nftables support is limited to the following features:
+    # - AntreaProxy (proxyAll)
+    #  NFTablesHostNetworkMode: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -4657,6 +4662,14 @@ data:
       # Enable to accelerate Pod-to-Pod traffic in the Node's host network using nftables flowtable when traffic mode is
       # noEncap or hybrid.
       enable: true
+
+    # HostNetworkMode determines how antrea-agent implements Node host network netfilter rules required by Antrea
+    # functionalities and features, using either "iptables" or "nftables". The default value is "iptables". If "nftables"
+    # is specified, the NFTablesHostNetworkMode feature gate must be enabled; otherwise, antrea-agent will fail to start.
+    # If the above condition is met but nftables is not supported or unavailable on the Node, antrea-agent will also fail
+    # to start. Currently, nftables support is limited to the following features:
+    #   - AntreaProxy (proxyAll)
+    hostNetworkMode: "iptables"
   antrea-cni.conflist: |
     {
         "cniVersion":"0.3.0",
@@ -5687,7 +5700,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5e83f654fbb48b39b21ceeeb20d4d60ca1cf3de258acc258a2cc961eda9ac2a2
+        checksum/config: 4cddc6760e67614e8a0fd94cb2f88148342bff967dcaf74a4053ea424ea0d853
       labels:
         app: antrea
         component: antrea-agent
@@ -5933,7 +5946,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5e83f654fbb48b39b21ceeeb20d4d60ca1cf3de258acc258a2cc961eda9ac2a2
+        checksum/config: 4cddc6760e67614e8a0fd94cb2f88148342bff967dcaf74a4053ea424ea0d853
       labels:
         app: antrea
         component: antrea-controller

ci/kind/kind-setup.sh

@@ -61,7 +61,8 @@ where:
   --service-cidr: specify service clusterip cidr used in kind cluster, kind's default value will be used if empty.
   --encap-mode: inter-node pod traffic encap mode, default is encap.
   --no-proxy: disable Antrea proxy.
-  --no-kube-proxy: disable Kube proxy.
+  --no-kube-proxy: deprecated. This option is still supported for compatibility, but will be removed. Use '--kube-proxy-mode none' going forward to disable kube-proxy.
+  --kube-proxy-mode: specify the kube-proxy mode. Supported values are 'iptables', 'nftables', 'ipvs' and 'none' (to disable kube-proxy).
   --no-kube-node-ipam: disable NodeIPAM in kube-controller-manager.
   --antrea-cni: install Antrea CNI in Kind cluster; by default the cluster is created without a CNI installed.
   --prometheus: create RBAC resources for Prometheus, default is false.
@@ -719,10 +720,16 @@ while [[ $# -gt 0 ]]
       shift
       ;;
     --no-kube-proxy)
+      echo "WARNING: '--no-kube-proxy' is deprecated. Use '--kube-proxy-mode none' instead."
       add_option "--no-kube-proxy" "create"
       KUBE_PROXY_MODE="none"
       shift
       ;;
+    --kube-proxy-mode)
+      add_option "--kube-proxy-mode" "create"
+      KUBE_PROXY_MODE="$2"
+      shift 2
+      ;;
     --no-kube-node-ipam)
       add_option "--no-kube-node-ipam" "create"
       KUBE_NODE_IPAM=false