Skip to content

Commit 4a05fdd

Browse files
jimyagjimyag
authored
test(e2e): cover per-pod IP family selection (#6985)
Signed-off-by: jimyag <git@jimyag.com>
1 parent c4731b5 commit 4a05fdd

2 files changed

Lines changed: 271 additions & 0 deletions

File tree

‎test/e2e/kube-ovn/ipam/ipam.go‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,74 @@ import (
1717

1818
apiv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
1919
"github.com/kubeovn/kube-ovn/pkg/ipam"
20+
"github.com/kubeovn/kube-ovn/pkg/ovs"
2021
"github.com/kubeovn/kube-ovn/pkg/util"
2122
"github.com/kubeovn/kube-ovn/test/e2e/framework"
23+
"github.com/kubeovn/kube-ovn/test/e2e/framework/iproute"
2224
)
2325

2426
const ippoolUpdateTimeout = 2 * time.Minute
2527

28+
func expectPodIPFamily(pod *corev1.Pod, family string) {
29+
ginkgo.GinkgoHelper()
30+
31+
ipv4, ipv6 := util.SplitStringIP(pod.Annotations[util.IPAddressAnnotation])
32+
switch family {
33+
case apiv1.ProtocolIPv4:
34+
framework.ExpectNotEmpty(ipv4)
35+
framework.ExpectEmpty(ipv6)
36+
framework.ExpectConsistOf(util.PodIPs(*pod), []string{ipv4})
37+
case apiv1.ProtocolIPv6:
38+
framework.ExpectEmpty(ipv4)
39+
framework.ExpectNotEmpty(ipv6)
40+
framework.ExpectConsistOf(util.PodIPs(*pod), []string{ipv6})
41+
case apiv1.ProtocolDual:
42+
framework.ExpectNotEmpty(ipv4)
43+
framework.ExpectNotEmpty(ipv6)
44+
framework.ExpectConsistOf(util.PodIPs(*pod), []string{ipv4, ipv6})
45+
default:
46+
framework.Failf("unexpected IP family %q", family)
47+
}
48+
}
49+
50+
func expectDefaultIPCRFamily(f *framework.Framework, pod *corev1.Pod, subnetName, family string) {
51+
ginkgo.GinkgoHelper()
52+
53+
ipName := ovs.PodNameToPortName(pod.Name, pod.Namespace, util.OvnProvider)
54+
ipCR := f.IPClient().Get(ipName)
55+
framework.ExpectEqual(ipCR.Spec.Subnet, subnetName)
56+
framework.ExpectEqual(ipCR.Spec.PodName, pod.Name)
57+
framework.ExpectEqual(ipCR.Spec.Namespace, pod.Namespace)
58+
framework.ExpectEqual(ipCR.Spec.NodeName, pod.Spec.NodeName)
59+
framework.ExpectEqual(ipCR.Spec.IPAddress, pod.Annotations[util.IPAddressAnnotation])
60+
ipv4, ipv6 := util.SplitStringIP(pod.Annotations[util.IPAddressAnnotation])
61+
framework.ExpectEqual(ipCR.Spec.V4IPAddress, ipv4)
62+
framework.ExpectEqual(ipCR.Spec.V6IPAddress, ipv6)
63+
64+
switch family {
65+
case apiv1.ProtocolIPv4:
66+
framework.ExpectNotEmpty(ipCR.Spec.V4IPAddress)
67+
framework.ExpectEmpty(ipCR.Spec.V6IPAddress)
68+
case apiv1.ProtocolIPv6:
69+
framework.ExpectEmpty(ipCR.Spec.V4IPAddress)
70+
framework.ExpectNotEmpty(ipCR.Spec.V6IPAddress)
71+
case apiv1.ProtocolDual:
72+
framework.ExpectNotEmpty(ipCR.Spec.V4IPAddress)
73+
framework.ExpectNotEmpty(ipCR.Spec.V6IPAddress)
74+
}
75+
}
76+
77+
func expectEth0IPs(pod *corev1.Pod, expectedIPs []string) {
78+
ginkgo.GinkgoHelper()
79+
80+
links, err := iproute.AddressShow("eth0", func(cmd ...string) ([]byte, []byte, error) {
81+
return framework.KubectlExec(pod.Namespace, pod.Name, cmd...)
82+
})
83+
framework.ExpectNoError(err)
84+
framework.ExpectHaveLen(links, 1)
85+
framework.ExpectConsistOf(links[0].NonLinkLocalIPs(), expectedIPs)
86+
}
87+
2688
var _ = framework.Describe("[group:ipam]", func() {
2789
f := framework.NewDefaultFramework("ipam")
2890

@@ -114,6 +176,120 @@ var _ = framework.Describe("[group:ipam]", func() {
114176
framework.ExpectConsistOf(util.PodIPs(*pod), strings.Split(ip, ","))
115177
})
116178

179+
framework.ConformanceIt("should allocate requested IP family for pod on dual-stack subnet", func() {
180+
f.SkipVersionPriorTo(1, 17, "Per-pod IP family selection was introduced in v1.17")
181+
if !f.IsDual() {
182+
ginkgo.Skip("This test requires a dual-stack cluster")
183+
}
184+
185+
cmd := []string{"sleep", "infinity"}
186+
cases := []struct {
187+
name string
188+
annotations map[string]string
189+
family string
190+
}{
191+
{
192+
name: "default dual-stack allocation",
193+
family: apiv1.ProtocolDual,
194+
},
195+
{
196+
name: "IPv4-only allocation",
197+
annotations: map[string]string{util.IPFamilyAnnotation: strings.ToLower(apiv1.ProtocolIPv4)},
198+
family: apiv1.ProtocolIPv4,
199+
},
200+
{
201+
name: "IPv6-only allocation",
202+
annotations: map[string]string{util.IPFamilyAnnotation: strings.ToLower(apiv1.ProtocolIPv6)},
203+
family: apiv1.ProtocolIPv6,
204+
},
205+
}
206+
207+
for _, tt := range cases {
208+
ginkgo.By("Creating pod " + podName + " with " + tt.name)
209+
pod := framework.MakePrivilegedPod(namespaceName, podName, nil, tt.annotations, f.KubeOVNImage, cmd, nil)
210+
pod = podClient.CreateSync(pod)
211+
212+
ginkgo.By("Validating pod annotations")
213+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
214+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
215+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.GatewayAnnotation, subnet.Spec.Gateway)
216+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnet.Name)
217+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.RoutedAnnotation, "true")
218+
framework.ExpectMAC(pod.Annotations[util.MacAddressAnnotation])
219+
expectPodIPFamily(pod, tt.family)
220+
expectDefaultIPCRFamily(f, pod, subnetName, tt.family)
221+
expectEth0IPs(pod, util.PodIPs(*pod))
222+
223+
ginkgo.By("Deleting pod " + podName)
224+
podClient.DeleteSync(podName)
225+
f.IPClient().DeleteSync(ovs.PodNameToPortName(podName, namespaceName, util.OvnProvider))
226+
}
227+
})
228+
229+
framework.ConformanceIt("should allocate requested IP family from named dual-stack IPPool", func() {
230+
f.SkipVersionPriorTo(1, 17, "Per-pod IP family selection was introduced in v1.17")
231+
if !f.IsDual() {
232+
ginkgo.Skip("This test requires a dual-stack cluster")
233+
}
234+
235+
ginkgo.By("Creating IPPool " + ippoolName)
236+
poolIPs := strings.Split(framework.RandomIPs(cidr, ",", 2), ",")
237+
ippool := framework.MakeIPPool(ippoolName, subnetName, poolIPs, []string{namespaceName})
238+
ippool = ippoolClient.CreateSync(ippool)
239+
framework.ExpectNotEmpty(ippool.Status.V4AvailableIPRange)
240+
framework.ExpectNotEmpty(ippool.Status.V6AvailableIPRange)
241+
242+
cmd := []string{"sleep", "infinity"}
243+
for _, family := range []string{apiv1.ProtocolIPv6, apiv1.ProtocolIPv4} {
244+
ginkgo.By("Creating pod " + podName + " with " + family + " IP family from IPPool " + ippoolName)
245+
annotations := map[string]string{
246+
util.IPFamilyAnnotation: strings.ToLower(family),
247+
util.IPPoolAnnotation: ippoolName,
248+
}
249+
pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil)
250+
pod = podClient.CreateSync(pod)
251+
252+
ginkgo.By("Validating pod allocation")
253+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.AllocatedAnnotation, "true")
254+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.IPPoolAnnotation, ippoolName)
255+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.CidrAnnotation, subnet.Spec.CIDRBlock)
256+
framework.ExpectHaveKeyWithValue(pod.Annotations, util.LogicalSwitchAnnotation, subnet.Name)
257+
expectPodIPFamily(pod, family)
258+
expectDefaultIPCRFamily(f, pod, subnetName, family)
259+
framework.ExpectContainElement(poolIPs, pod.Annotations[util.IPAddressAnnotation])
260+
expectEth0IPs(pod, util.PodIPs(*pod))
261+
262+
ginkgo.By("Deleting pod " + podName)
263+
podClient.DeleteSync(podName)
264+
f.IPClient().DeleteSync(ovs.PodNameToPortName(podName, namespaceName, util.OvnProvider))
265+
}
266+
})
267+
268+
framework.ConformanceIt("should reject requested IP family that does not match single-stack subnet", func() {
269+
f.SkipVersionPriorTo(1, 17, "Per-pod IP family selection was introduced in v1.17")
270+
if f.IsDual() {
271+
ginkgo.Skip("This test requires a single-stack cluster")
272+
}
273+
274+
requestedFamily := apiv1.ProtocolIPv4
275+
if f.HasIPv4() {
276+
requestedFamily = apiv1.ProtocolIPv6
277+
}
278+
279+
ginkgo.By("Creating pod " + podName + " with mismatched " + requestedFamily + " IP family on " + subnet.Spec.Protocol + " subnet")
280+
annotations := map[string]string{util.IPFamilyAnnotation: strings.ToLower(requestedFamily)}
281+
pod := framework.MakePod(namespaceName, podName, nil, annotations, "", nil, nil)
282+
_ = podClient.Create(pod)
283+
284+
ginkgo.By("Waiting for pod " + podName + " to have event indicating IP family mismatch")
285+
events := f.EventClient().WaitToHaveEvent(util.KindPod, podName, corev1.EventTypeWarning, "AcquireAddressFailed", "kube-ovn-controller", "")
286+
framework.ExpectContainSubstring(events[0].Message, fmt.Sprintf("requested ip family %s does not match subnet %s protocol %s", requestedFamily, subnetName, subnet.Spec.Protocol))
287+
288+
ginkgo.By("Validating pod " + podName + " is not allocated")
289+
pod = podClient.GetPod(podName)
290+
framework.ExpectNotHaveKey(pod.Annotations, util.AllocatedAnnotation)
291+
})
292+
117293
framework.ConformanceIt("should allocate static ip for pod with comma separated ippool", func() {
118294
if f.IsDual() {
119295
ginkgo.Skip("Comma separated ippool is not supported for dual stack")

‎test/e2e/multus/e2e_test.go‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,101 @@ var _ = framework.SerialDescribe("[group:multus]", func() {
156156
}
157157
})
158158

159+
framework.ConformanceIt("should allocate requested IP family for attachment interface", func() {
160+
f.SkipVersionPriorTo(1, 17, "Per-pod IP family selection was introduced in v1.17")
161+
if !f.IsDual() {
162+
ginkgo.Skip("This test requires a dual-stack cluster")
163+
}
164+
165+
provider := fmt.Sprintf("%s.%s.%s", nadName, namespaceName, util.OvnProvider)
166+
167+
ginkgo.By("Creating network attachment definition " + nadName)
168+
nad := framework.MakeOVNNetworkAttachmentDefinition(nadName, namespaceName, provider, nil)
169+
nad = nadClient.Create(nad)
170+
framework.Logf("created network attachment definition config:\n%s", nad.Spec.Config)
171+
172+
ginkgo.By("Creating subnet " + subnetName)
173+
subnet = framework.MakeSubnet(subnetName, "", cidr, "", "", "", nil, nil, nil)
174+
subnet.Spec.Provider = provider
175+
subnet = subnetClient.CreateSync(subnet)
176+
177+
for _, family := range []string{apiv1.ProtocolIPv4, apiv1.ProtocolIPv6} {
178+
ginkgo.By("Creating pod " + podName + " with " + family + " attachment IP family")
179+
annotations := map[string]string{
180+
nadv1.NetworkAttachmentAnnot: fmt.Sprintf("%s/%s", nad.Namespace, nad.Name),
181+
fmt.Sprintf(util.IPFamilyAnnotationTemplate, provider): strings.ToLower(family),
182+
}
183+
cmd := []string{"sleep", "infinity"}
184+
pod := framework.MakePrivilegedPod(namespaceName, podName, nil, annotations, f.KubeOVNImage, cmd, nil)
185+
pod = podClient.CreateSync(pod)
186+
187+
ginkgo.By("Validating pod annotations")
188+
framework.ExpectHaveKey(pod.Annotations, nadv1.NetworkStatusAnnot)
189+
framework.Logf("pod network status:\n%s", pod.Annotations[nadv1.NetworkStatusAnnot])
190+
ip := pod.Annotations[fmt.Sprintf(util.IPAddressAnnotationTemplate, provider)]
191+
cidr := pod.Annotations[fmt.Sprintf(util.CidrAnnotationTemplate, provider)]
192+
gateway := pod.Annotations[fmt.Sprintf(util.GatewayAnnotationTemplate, provider)]
193+
mac := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, provider)]
194+
framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.AllocatedAnnotationTemplate, provider), "true")
195+
framework.ExpectHaveKeyWithValue(pod.Annotations, fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, provider), subnet.Name)
196+
framework.ExpectIPInCIDR(ip, cidr)
197+
framework.ExpectMAC(mac)
198+
199+
ipv4, ipv6 := util.SplitStringIP(ip)
200+
gatewayV4, gatewayV6 := util.SplitStringIP(gateway)
201+
switch family {
202+
case apiv1.ProtocolIPv4:
203+
framework.ExpectNotEmpty(ipv4)
204+
framework.ExpectEmpty(ipv6)
205+
framework.ExpectNotEmpty(gatewayV4)
206+
case apiv1.ProtocolIPv6:
207+
framework.ExpectEmpty(ipv4)
208+
framework.ExpectNotEmpty(ipv6)
209+
framework.ExpectNotEmpty(gatewayV6)
210+
}
211+
212+
ipName := ovs.PodNameToPortName(podName, namespaceName, provider)
213+
ginkgo.By("Validating IP resource " + ipName)
214+
ipCR := ipClient.Get(ipName)
215+
framework.ExpectEqual(ipCR.Spec.Subnet, subnetName)
216+
framework.ExpectEqual(ipCR.Spec.PodName, podName)
217+
framework.ExpectEqual(ipCR.Spec.Namespace, namespaceName)
218+
framework.ExpectEqual(ipCR.Spec.NodeName, pod.Spec.NodeName)
219+
framework.ExpectEqual(ipCR.Spec.IPAddress, ip)
220+
framework.ExpectEqual(ipCR.Spec.MacAddress, mac)
221+
framework.ExpectEqual(ipCR.Spec.V4IPAddress, ipv4)
222+
framework.ExpectEqual(ipCR.Spec.V6IPAddress, ipv6)
223+
224+
ginkgo.By("Getting attachment interface name")
225+
statuses, err := nadutils.GetNetworkStatus(pod)
226+
framework.ExpectNoError(err)
227+
var ifaceName string
228+
nadKey := cache.MetaObjectToName(nad).String()
229+
for _, status := range statuses {
230+
if status.Name == nadKey {
231+
framework.ExpectConsistOf(status.IPs, strings.Split(ip, ","))
232+
framework.ExpectEqual(status.Mac, mac)
233+
ifaceName = status.Interface
234+
break
235+
}
236+
}
237+
framework.ExpectNotEmpty(ifaceName)
238+
239+
ginkgo.By("Validating attachment interface IPs")
240+
links, err := iproute.AddressShow(ifaceName, func(cmd ...string) ([]byte, []byte, error) {
241+
return framework.KubectlExec(pod.Namespace, pod.Name, cmd...)
242+
})
243+
framework.ExpectNoError(err)
244+
framework.ExpectHaveLen(links, 1)
245+
framework.ExpectEqual(links[0].Address, mac)
246+
framework.ExpectConsistOf(links[0].NonLinkLocalIPs(), strings.Split(ip, ","))
247+
248+
ginkgo.By("Deleting pod " + podName)
249+
podClient.DeleteSync(podName)
250+
ipClient.DeleteSync(ipName)
251+
}
252+
})
253+
159254
framework.ConformanceIt("should reject ovn attachment provider without matching subnet", func() {
160255
f.SkipVersionPriorTo(1, 15, "multiple network attachment validation requires v1.15+")
161256

0 commit comments

Comments
 (0)