Skip to content

Commit 9560ed0

Browse files
antoninbasantoninbas
committed
Fixing e2e tests for IPv6 / dual-stack
Signed-off-by: Antonin Bas <[email protected]>
1 parent 575b909 commit 9560ed0

File tree

1 file changed

+62
-52
lines changed

1 file changed

+62
-52
lines changed

‎test/e2e/nodeportlocal_test.go‎

Lines changed: 62 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -159,8 +159,7 @@ func getPodIPForFamily(podIPs *PodIPs, ipFamily types.IPFamilyType) string {
159159
return ""
160160
}
161161

162-
func checkNPLRulesForPod(t *testing.T, data *TestData, r *require.Assertions, nplAnnotations []types.NPLAnnotation, antreaPod string, podIPs *PodIPs, present bool) {
163-
var rules []nplRuleData
162+
func nplAnnotationsToRules(nplAnnotations []types.NPLAnnotation, podIPs *PodIPs) (rulesIPv4, rulesIPv6 []nplRuleData) {
164163
for _, ann := range nplAnnotations {
165164
podIP := getPodIPForFamily(podIPs, ann.IPFamily)
166165
if podIP == "" {
@@ -173,29 +172,31 @@ func checkNPLRulesForPod(t *testing.T, data *TestData, r *require.Assertions, np
173172
podPort: ann.PodPort,
174173
protocol: ann.Protocol,
175174
}
176-
rules = append(rules, rule)
175+
if ann.IPFamily == types.IPFamilyIPv4 {
176+
rulesIPv4 = append(rulesIPv4, rule)
177+
} else {
178+
rulesIPv6 = append(rulesIPv6, rule)
179+
}
177180
}
178-
checkForNPLRuleInIPTables(t, data, r, antreaPod, rules, present)
179-
checkForNPLListeningSockets(t, data, r, antreaPod, rules, present)
181+
return
180182
}
181183

182-
func checkNPLRulesForWindowsPod(t *testing.T, data *TestData, r *require.Assertions, nplAnnotations []types.NPLAnnotation, antreaPod string, podIPs *PodIPs, nodeName string, present bool) {
183-
var rules []nplRuleData
184-
for _, ann := range nplAnnotations {
185-
podIP := getPodIPForFamily(podIPs, ann.IPFamily)
186-
if podIP == "" {
187-
continue
188-
}
189-
rule := nplRuleData{
190-
nodeIP: ann.NodeIP,
191-
nodePort: ann.NodePort,
192-
podIP: podIP,
193-
podPort: ann.PodPort,
194-
protocol: ann.Protocol,
195-
}
196-
rules = append(rules, rule)
184+
func checkNPLRulesForPod(t *testing.T, data *TestData, r *require.Assertions, nplAnnotations []types.NPLAnnotation, antreaPod string, podIPs *PodIPs, present bool) {
185+
rulesIPv4, rulesIPv6 := nplAnnotationsToRules(nplAnnotations, podIPs)
186+
if len(rulesIPv4) > 0 {
187+
checkForNPLRuleInIPTables(t, data, r, antreaPod, rulesIPv4, false, present)
188+
checkForNPLListeningSockets(t, data, r, antreaPod, rulesIPv4, false, present)
197189
}
198-
checkForNPLRuleInNetNat(t, data, r, antreaPod, nodeName, rules, present)
190+
if len(rulesIPv6) > 0 {
191+
checkForNPLRuleInIPTables(t, data, r, antreaPod, rulesIPv6, true, present)
192+
checkForNPLListeningSockets(t, data, r, antreaPod, rulesIPv6, true, present)
193+
}
194+
}
195+
196+
func checkNPLRulesForWindowsPod(t *testing.T, data *TestData, r *require.Assertions, nplAnnotations []types.NPLAnnotation, antreaPod string, podIPs *PodIPs, nodeName string, present bool) {
197+
rulesIPv4, rulesIPv6 := nplAnnotationsToRules(nplAnnotations, podIPs)
198+
r.Empty(rulesIPv6, "We only support IPv4 for Windows Nodes")
199+
checkForNPLRuleInNetNat(t, data, r, antreaPod, nodeName, rulesIPv4, present)
199200
}
200201

201202
func buildRuleForPod(rule nplRuleData) []string {
@@ -209,8 +210,12 @@ func protocolToString(p corev1.Protocol) string {
209210
return strings.ToLower(string(p))
210211
}
211212

212-
func checkForNPLRuleInIPTables(t *testing.T, data *TestData, r *require.Assertions, antreaPod string, rules []nplRuleData, present bool) {
213-
cmd := []string{"iptables", "-t", "nat", "-S"}
213+
func checkForNPLRuleInIPTables(t *testing.T, data *TestData, r *require.Assertions, antreaPod string, rules []nplRuleData, isIPv6 bool, present bool) {
214+
cmd := []string{"iptables"}
215+
if isIPv6 {
216+
cmd = []string{"ip6tables"}
217+
}
218+
cmd = append(cmd, "-t", "nat", "-S")
214219
t.Logf("Verifying iptables rules %v, present: %v", rules, present)
215220
const timeout = 60 * time.Second
216221
err := wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, false, func(ctx context.Context) (bool, error) {
@@ -297,23 +302,32 @@ func checkForNPLRuleInNetNat(t *testing.T, data *TestData, r *require.Assertions
297302
r.NoError(err, "Poll for NetNat rules check failed")
298303
}
299304

300-
func checkForNPLListeningSockets(t *testing.T, data *TestData, r *require.Assertions, antreaPod string, rules []nplRuleData, present bool) {
305+
func checkForNPLListeningSockets(t *testing.T, data *TestData, r *require.Assertions, antreaPod string, rules []nplRuleData, isIPv6 bool, present bool) {
301306
t.Logf("Verifying NPL listening sockets")
302307
const timeout = 30 * time.Second
308+
bindIP := "0.0.0.0"
309+
if isIPv6 {
310+
bindIP = "[::]"
311+
}
303312
err := wait.PollUntilContextTimeout(context.Background(), time.Second, timeout, false, func(ctx context.Context) (bool, error) {
304313
for _, rule := range rules {
305314
protocolOption := "--" + rule.protocol
306315
cmd := []string{"ss", "--listening", protocolOption, "-H", "-n"}
316+
if isIPv6 {
317+
cmd = append(cmd, "-6")
318+
} else {
319+
cmd = append(cmd, "-4")
320+
}
307321
stdout, _, err := data.RunCommandFromPod(antreaNamespace, antreaPod, agentContainerName, cmd)
308322
if err != nil {
309323
return false, fmt.Errorf("error when running 'ss': %v", err)
310324
}
311325

312326
t.Logf("Checking if NPL is listening on %s:%d", rule.protocol, rule.nodePort)
313-
regexString := fmt.Sprintf(`(?m)^LISTEN.*0\.0\.0\.0:%d`, rule.nodePort)
327+
regexString := fmt.Sprintf(`(?m)^LISTEN.*%s:%d`, regexp.QuoteMeta(bindIP), rule.nodePort)
314328
// UDP is a connectionless protocol and hence, lacks states similar to those of TCP (LISTEN).
315329
if rule.protocol == "udp" {
316-
regexString = fmt.Sprintf(`(?m)^UNCONN.*0\.0\.0\.0:%d`, rule.nodePort)
330+
regexString = fmt.Sprintf(`(?m)^UNCONN.*%s:%d`, regexp.QuoteMeta(bindIP), rule.nodePort)
317331
}
318332
found, err := regexp.MatchString(regexString, stdout)
319333
if err != nil {
@@ -332,8 +346,13 @@ func checkForNPLListeningSockets(t *testing.T, data *TestData, r *require.Assert
332346
r.NoError(err, "Check for NPL listening sockets failed")
333347
}
334348

335-
func deleteNPLRuleFromIPTables(t *testing.T, data *TestData, r *require.Assertions, antreaPod string, rule nplRuleData) {
336-
cmd := append([]string{"iptables", "-w", "10", "-t", "nat", "-D", "ANTREA-NODE-PORT-LOCAL"}, buildRuleForPod(rule)...)
349+
func deleteNPLRuleFromIPTables(t *testing.T, data *TestData, r *require.Assertions, antreaPod string, rule nplRuleData, isIPv6 bool) {
350+
cmd := []string{"iptables"}
351+
if isIPv6 {
352+
cmd = []string{"ip6tables"}
353+
}
354+
cmd = append(cmd, "-w", "10", "-t", "nat", "-D", "ANTREA-NODE-PORT-LOCAL")
355+
cmd = append(cmd, buildRuleForPod(rule)...)
337356
t.Logf("Deleting iptables rule for %v", rule)
338357
_, _, err := data.RunCommandFromPod(antreaNamespace, antreaPod, agentContainerName, cmd)
339358
r.NoError(err, "Error when deleting iptables rule")
@@ -654,7 +673,7 @@ func testNPLMultiplePodsAgentRestart(t *testing.T, data *TestData) {
654673
if len(clusterInfo.windowsNodes) > 1 {
655674
deleteNPLRuleFromNetNat(t, data, r, antreaPod, ruleToDelete)
656675
} else {
657-
deleteNPLRuleFromIPTables(t, data, r, antreaPod, ruleToDelete)
676+
deleteNPLRuleFromIPTables(t, data, r, antreaPod, ruleToDelete, nplAnnotations[0].IPFamily == types.IPFamilyIPv6)
658677
}
659678

660679
err = data.RestartAntreaAgentPods(defaultTimeout)
@@ -710,40 +729,31 @@ func testNPLChangePortRangeAgentRestart(t *testing.T, data *TestData) {
710729
err = data.podWaitForRunning(defaultTimeout, clientName, data.testNamespace)
711730
r.NoError(err, "Error when waiting for Pod %s to be running", clientName)
712731

713-
var rules []nplRuleData
732+
var allRulesIPv4, allRulesIPv6 []nplRuleData
714733
for _, testPodName := range testPods {
715734
nplAnnotations, testPodIPs := getNPLAnnotations(t, data, r, testPodName, nil)
716-
for i := range nplAnnotations {
717-
// Determine which Pod IP to use based on the annotation's IP family
718-
var podIP string
719-
if nplAnnotations[i].IPFamily == "IPv6" && testPodIPs.IPv6 != nil {
720-
podIP = testPodIPs.IPv6.String()
721-
} else if testPodIPs.IPv4 != nil {
722-
podIP = testPodIPs.IPv4.String()
723-
}
724-
if podIP == "" {
725-
continue
726-
}
727-
rule := nplRuleData{
728-
nodePort: nplAnnotations[i].NodePort,
729-
podIP: podIP,
730-
podPort: nplAnnotations[i].PodPort,
731-
protocol: nplAnnotations[i].Protocol,
732-
}
733-
rules = append(rules, rule)
734-
}
735+
rulesIPv4, rulesIPv6 := nplAnnotationsToRules(nplAnnotations, testPodIPs)
736+
allRulesIPv4 = append(allRulesIPv4, rulesIPv4...)
737+
allRulesIPv6 = append(allRulesIPv6, rulesIPv6...)
735738
}
736739
configureNPLForAgent(t, data, updatedStartPort, updatedEndPort)
737740

738741
antreaPod, err := data.getAntreaPodOnNode(serverNode)
739742
r.NoError(err, "Error when getting Antrea Agent Pod on Node '%s'", serverNode)
740743

741744
if clusterInfo.nodesOS[serverNode] == "windows" {
745+
r.Empty(allRulesIPv6, "We only support IPv4 for Windows Nodes")
742746
time.Sleep(10 * time.Second)
743-
checkForNPLRuleInNetNat(t, data, r, antreaPod, serverNode, rules, false)
747+
checkForNPLRuleInNetNat(t, data, r, antreaPod, serverNode, allRulesIPv4, false)
744748
} else {
745-
checkForNPLRuleInIPTables(t, data, r, antreaPod, rules, false)
746-
checkForNPLListeningSockets(t, data, r, antreaPod, rules, false)
749+
if len(allRulesIPv4) > 0 {
750+
checkForNPLRuleInIPTables(t, data, r, antreaPod, allRulesIPv4, false, false)
751+
checkForNPLListeningSockets(t, data, r, antreaPod, allRulesIPv4, false, false)
752+
}
753+
if len(allRulesIPv6) > 0 {
754+
checkForNPLRuleInIPTables(t, data, r, antreaPod, allRulesIPv6, true, false)
755+
checkForNPLListeningSockets(t, data, r, antreaPod, allRulesIPv6, true, false)
756+
}
747757
}
748758

749759
for _, testPodName := range testPods {

0 commit comments

Comments
 (0)