Skip to content

Commit cf53d48

Browse files
alebedev87claude
alebedev87
and
claude
committed
NE-2422: Add Classic LB test and refactor dual-stack ingress tests
Add a second test case that verifies IPv4 connectivity through a Classic LB ingress controller on a dual-stack cluster. Extract common setup logic (backend service/pod creation, edge route creation, route admission waiting) into shared helper functions. Use distinct domains (nlb/clb) for each test's ingress controller shard. Use 2 replicas for the NLB shard to allocate addresses in multiple AZs, mitigating instability of IPv6 hairpin traffic. Reduce curl verbosity by only logging the successful response. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent bbeb835 commit cf53d48

2 files changed

Lines changed: 208 additions & 127 deletions

File tree

‎test/extended/router/dualstack.go‎

Lines changed: 198 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -37,33 +37,24 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:AWSDualStackInstall][Featu
3737
ctx := context.Background()
3838

3939
g.By("Checking that the Infrastructure CR has a DualStack IPFamily")
40-
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{})
41-
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get infrastructure CR")
42-
43-
if infra.Status.PlatformStatus == nil || infra.Status.PlatformStatus.Type != configv1.AWSPlatformType {
44-
g.Skip("Test requires AWS platform")
45-
}
46-
if infra.Status.PlatformStatus.AWS == nil {
47-
g.Skip("AWS platform status is not set")
48-
}
49-
ipFamily := infra.Status.PlatformStatus.AWS.IPFamily
50-
if ipFamily != configv1.DualStackIPv4Primary && ipFamily != configv1.DualStackIPv6Primary {
51-
g.Skip(fmt.Sprintf("Test requires DualStack IPFamily, got %q", ipFamily))
52-
}
40+
requireAWSDualStack(ctx, oc)
5341

5442
g.By("Getting the default ingress domain")
5543
defaultDomain, err := getDefaultIngressClusterDomainName(oc, time.Minute)
5644
o.Expect(err).NotTo(o.HaveOccurred(), "failed to find default domain name")
5745

5846
ns := oc.KubeFramework().Namespace.Name
5947
baseDomain := strings.TrimPrefix(defaultDomain, "apps.")
60-
shardFQDN := "hosts." + baseDomain
48+
shardFQDN := "nlb." + baseDomain
6149

6250
// Deploy the shard first so DNS and LB can provision while we set up the backend.
51+
// Use 2 replicas so the NLB allocates addresses in multiple AZs, which
52+
// mitigates instability of IPv6 hairpin traffic (cluster to cluster via LB).
6353
g.By("Deploying a new router shard with NLB")
6454
shardIngressCtrl, err := shard.DeployNewRouterShard(oc, 10*time.Minute, shard.Config{
65-
Domain: shardFQDN,
66-
Type: oc.Namespace(),
55+
Domain: shardFQDN,
56+
Type: oc.Namespace(),
57+
Replicas: 2,
6758
LoadBalancer: &operatorv1.LoadBalancerStrategy{
6859
Scope: operatorv1.ExternalLoadBalancer,
6960
ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{
@@ -87,148 +78,229 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:AWSDualStackInstall][Featu
8778
err = oc.AsAdmin().Run("label").Args("namespace", oc.Namespace(), "type="+oc.Namespace()).Execute()
8879
o.Expect(err).NotTo(o.HaveOccurred())
8980

90-
g.By("Creating backend service")
91-
service := &corev1.Service{
92-
ObjectMeta: metav1.ObjectMeta{
93-
Name: "dualstack-backend",
94-
Labels: map[string]string{
95-
"app": "dualstack-backend",
96-
},
97-
},
98-
Spec: corev1.ServiceSpec{
99-
Selector: map[string]string{
100-
"app": "dualstack-backend",
101-
},
102-
IPFamilyPolicy: func() *corev1.IPFamilyPolicy {
103-
p := corev1.IPFamilyPolicyPreferDualStack
104-
return &p
105-
}(),
106-
Ports: []corev1.ServicePort{
107-
{
108-
Name: "http",
109-
Port: 8080,
110-
Protocol: corev1.ProtocolTCP,
111-
TargetPort: intstr.FromInt(8080),
81+
g.By("Creating backend service and pod")
82+
createBackendServiceAndPod(ctx, oc, ns, "dualstack-backend")
83+
84+
g.By("Creating an edge-terminated route")
85+
routeHost := "dualstack-test." + shardFQDN
86+
createEdgeRoute(ctx, oc, ns, "dualstack-route", routeHost, "dualstack-backend")
87+
88+
g.By("Waiting for the route to be admitted")
89+
waitForRouteAdmitted(ctx, oc, ns, "dualstack-route", routeHost)
90+
91+
g.By("Creating exec pod for curl tests")
92+
execPod := exutil.CreateExecPodOrFail(oc.AdminKubeClient(), ns, "execpod")
93+
defer func() {
94+
oc.AdminKubeClient().CoreV1().Pods(ns).Delete(ctx, execPod.Name, *metav1.NewDeleteOptions(1))
95+
}()
96+
97+
g.By("Verifying route is reachable over IPv4")
98+
err = waitForRouteResponse(ns, execPod.Name, routeHost, "-4", 10*time.Minute)
99+
o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv4")
100+
101+
g.By("Verifying route is reachable over IPv6")
102+
err = waitForRouteResponse(ns, execPod.Name, routeHost, "-6", 10*time.Minute)
103+
o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv6")
104+
})
105+
106+
g.It("should be reachable via IPv4 through a Classic LB ingress controller on a dual-stack cluster", func() {
107+
ctx := context.Background()
108+
109+
g.By("Checking that the Infrastructure CR has a DualStack IPFamily")
110+
requireAWSDualStack(ctx, oc)
111+
112+
g.By("Getting the default ingress domain")
113+
defaultDomain, err := getDefaultIngressClusterDomainName(oc, time.Minute)
114+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to find default domain name")
115+
116+
ns := oc.KubeFramework().Namespace.Name
117+
baseDomain := strings.TrimPrefix(defaultDomain, "apps.")
118+
shardFQDN := "clb." + baseDomain
119+
120+
// Deploy the shard first so DNS and LB can provision while we set up the backend.
121+
g.By("Deploying a new router shard with Classic LB")
122+
shardIngressCtrl, err := shard.DeployNewRouterShard(oc, 10*time.Minute, shard.Config{
123+
Domain: shardFQDN,
124+
Type: oc.Namespace(),
125+
LoadBalancer: &operatorv1.LoadBalancerStrategy{
126+
Scope: operatorv1.ExternalLoadBalancer,
127+
ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{
128+
Type: operatorv1.AWSLoadBalancerProvider,
129+
AWS: &operatorv1.AWSLoadBalancerParameters{
130+
Type: operatorv1.AWSClassicLoadBalancer,
112131
},
113132
},
114133
},
115-
}
116-
_, err = oc.AdminKubeClient().CoreV1().Services(ns).Create(ctx, service, metav1.CreateOptions{})
134+
})
135+
defer func() {
136+
if shardIngressCtrl != nil {
137+
if err := oc.AdminOperatorClient().OperatorV1().IngressControllers(shardIngressCtrl.Namespace).Delete(ctx, shardIngressCtrl.Name, metav1.DeleteOptions{}); err != nil {
138+
e2e.Logf("deleting ingress controller failed: %v\n", err)
139+
}
140+
}
141+
}()
142+
o.Expect(err).NotTo(o.HaveOccurred(), "new router shard did not rollout")
143+
144+
g.By("Labelling the namespace for the shard")
145+
err = oc.AsAdmin().Run("label").Args("namespace", oc.Namespace(), "type="+oc.Namespace()).Execute()
117146
o.Expect(err).NotTo(o.HaveOccurred())
118147

119-
g.By("Creating backend pod")
120-
backendPod := &corev1.Pod{
121-
ObjectMeta: metav1.ObjectMeta{
122-
Name: "dualstack-backend",
123-
Labels: map[string]string{
124-
"app": "dualstack-backend",
148+
g.By("Creating backend service and pod")
149+
createBackendServiceAndPod(ctx, oc, ns, "classic-backend")
150+
151+
g.By("Creating an edge-terminated route")
152+
routeHost := "classic-test." + shardFQDN
153+
createEdgeRoute(ctx, oc, ns, "classic-route", routeHost, "classic-backend")
154+
155+
g.By("Waiting for the route to be admitted")
156+
waitForRouteAdmitted(ctx, oc, ns, "classic-route", routeHost)
157+
158+
g.By("Creating exec pod for curl tests")
159+
execPod := exutil.CreateExecPodOrFail(oc.AdminKubeClient(), ns, "execpod")
160+
defer func() {
161+
oc.AdminKubeClient().CoreV1().Pods(ns).Delete(ctx, execPod.Name, *metav1.NewDeleteOptions(1))
162+
}()
163+
164+
g.By("Verifying route is reachable over IPv4")
165+
err = waitForRouteResponse(ns, execPod.Name, routeHost, "-4", 10*time.Minute)
166+
o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv4")
167+
})
168+
})
169+
170+
func requireAWSDualStack(ctx context.Context, oc *exutil.CLI) {
171+
infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{})
172+
o.Expect(err).NotTo(o.HaveOccurred(), "failed to get infrastructure CR")
173+
174+
if infra.Status.PlatformStatus == nil || infra.Status.PlatformStatus.Type != configv1.AWSPlatformType {
175+
g.Skip("Test requires AWS platform")
176+
}
177+
if infra.Status.PlatformStatus.AWS == nil {
178+
g.Skip("AWS platform status is not set")
179+
}
180+
ipFamily := infra.Status.PlatformStatus.AWS.IPFamily
181+
if ipFamily != configv1.DualStackIPv4Primary && ipFamily != configv1.DualStackIPv6Primary {
182+
g.Skip(fmt.Sprintf("Test requires DualStack IPFamily, got %q", ipFamily))
183+
}
184+
}
185+
186+
func createBackendServiceAndPod(ctx context.Context, oc *exutil.CLI, ns, name string) {
187+
service := &corev1.Service{
188+
ObjectMeta: metav1.ObjectMeta{
189+
Name: name,
190+
Labels: map[string]string{"app": name},
191+
},
192+
Spec: corev1.ServiceSpec{
193+
Selector: map[string]string{"app": name},
194+
IPFamilyPolicy: func() *corev1.IPFamilyPolicy {
195+
p := corev1.IPFamilyPolicyPreferDualStack
196+
return &p
197+
}(),
198+
Ports: []corev1.ServicePort{
199+
{
200+
Name: "http",
201+
Port: 8080,
202+
Protocol: corev1.ProtocolTCP,
203+
TargetPort: intstr.FromInt(8080),
125204
},
126205
},
127-
Spec: corev1.PodSpec{
128-
TerminationGracePeriodSeconds: utilpointer.Int64(1),
129-
Containers: []corev1.Container{
130-
{
131-
Name: "server",
132-
Image: image.ShellImage(),
133-
ImagePullPolicy: corev1.PullIfNotPresent,
134-
Command: []string{"/bin/bash", "-c", `while true; do
206+
},
207+
}
208+
_, err := oc.AdminKubeClient().CoreV1().Services(ns).Create(ctx, service, metav1.CreateOptions{})
209+
o.Expect(err).NotTo(o.HaveOccurred())
210+
211+
pod := &corev1.Pod{
212+
ObjectMeta: metav1.ObjectMeta{
213+
Name: name,
214+
Labels: map[string]string{"app": name},
215+
},
216+
Spec: corev1.PodSpec{
217+
TerminationGracePeriodSeconds: utilpointer.Int64(1),
218+
Containers: []corev1.Container{
219+
{
220+
Name: "server",
221+
Image: image.ShellImage(),
222+
ImagePullPolicy: corev1.PullIfNotPresent,
223+
Command: []string{"/bin/bash", "-c", `while true; do
135224
printf "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: text/plain\r\n\r\nOK" | ncat -l 8080 --send-only || true
136225
done`},
137-
Ports: []corev1.ContainerPort{
138-
{
139-
ContainerPort: 8080,
140-
Name: "http",
141-
Protocol: corev1.ProtocolTCP,
142-
},
226+
Ports: []corev1.ContainerPort{
227+
{
228+
ContainerPort: 8080,
229+
Name: "http",
230+
Protocol: corev1.ProtocolTCP,
143231
},
144232
},
145233
},
146234
},
147-
}
148-
_, err = oc.AdminKubeClient().CoreV1().Pods(ns).Create(ctx, backendPod, metav1.CreateOptions{})
149-
o.Expect(err).NotTo(o.HaveOccurred())
235+
},
236+
}
237+
_, err = oc.AdminKubeClient().CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
238+
o.Expect(err).NotTo(o.HaveOccurred())
150239

151-
g.By("Waiting for backend pod to be running")
152-
e2e.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(ctx, oc.KubeClient(), "dualstack-backend", ns), "backend pod not running")
240+
e2e.ExpectNoError(e2epod.WaitForPodRunningInNamespaceSlow(ctx, oc.KubeClient(), name, ns), "backend pod not running")
241+
}
153242

154-
g.By("Creating an edge-terminated route")
155-
routeType := oc.Namespace()
156-
route := routev1.Route{
157-
ObjectMeta: metav1.ObjectMeta{
158-
Name: "dualstack-route",
159-
Labels: map[string]string{
160-
"type": routeType,
161-
},
243+
func createEdgeRoute(ctx context.Context, oc *exutil.CLI, ns, name, host, serviceName string) {
244+
route := routev1.Route{
245+
ObjectMeta: metav1.ObjectMeta{
246+
Name: name,
247+
Labels: map[string]string{
248+
"type": oc.Namespace(),
162249
},
163-
Spec: routev1.RouteSpec{
164-
Host: "dualstack-test." + shardFQDN,
165-
Port: &routev1.RoutePort{
166-
TargetPort: intstr.FromInt(8080),
167-
},
168-
TLS: &routev1.TLSConfig{
169-
Termination: routev1.TLSTerminationEdge,
170-
InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect,
171-
},
172-
To: routev1.RouteTargetReference{
173-
Kind: "Service",
174-
Name: "dualstack-backend",
175-
Weight: utilpointer.Int32(100),
176-
},
177-
WildcardPolicy: routev1.WildcardPolicyNone,
250+
},
251+
Spec: routev1.RouteSpec{
252+
Host: host,
253+
Port: &routev1.RoutePort{
254+
TargetPort: intstr.FromInt(8080),
178255
},
179-
}
180-
_, err = oc.RouteClient().RouteV1().Routes(ns).Create(ctx, &route, metav1.CreateOptions{})
181-
o.Expect(err).NotTo(o.HaveOccurred())
256+
TLS: &routev1.TLSConfig{
257+
Termination: routev1.TLSTerminationEdge,
258+
InsecureEdgeTerminationPolicy: routev1.InsecureEdgeTerminationPolicyRedirect,
259+
},
260+
To: routev1.RouteTargetReference{
261+
Kind: "Service",
262+
Name: serviceName,
263+
Weight: utilpointer.Int32(100),
264+
},
265+
WildcardPolicy: routev1.WildcardPolicyNone,
266+
},
267+
}
268+
_, err := oc.RouteClient().RouteV1().Routes(ns).Create(ctx, &route, metav1.CreateOptions{})
269+
o.Expect(err).NotTo(o.HaveOccurred())
270+
}
182271

183-
g.By("Waiting for the route to be admitted")
184-
routeHost := "dualstack-test." + shardFQDN
185-
err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
186-
r, err := oc.RouteClient().RouteV1().Routes(ns).Get(ctx, "dualstack-route", metav1.GetOptions{})
187-
if err != nil {
188-
e2e.Logf("failed to get route: %v, retrying...", err)
189-
return false, nil
190-
}
191-
for _, ingress := range r.Status.Ingress {
192-
if ingress.Host == routeHost {
193-
for _, condition := range ingress.Conditions {
194-
if condition.Type == routev1.RouteAdmitted && condition.Status == corev1.ConditionTrue {
195-
return true, nil
196-
}
272+
func waitForRouteAdmitted(ctx context.Context, oc *exutil.CLI, ns, name, host string) {
273+
err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
274+
r, err := oc.RouteClient().RouteV1().Routes(ns).Get(ctx, name, metav1.GetOptions{})
275+
if err != nil {
276+
e2e.Logf("failed to get route: %v, retrying...", err)
277+
return false, nil
278+
}
279+
for _, ingress := range r.Status.Ingress {
280+
if ingress.Host == host {
281+
for _, condition := range ingress.Conditions {
282+
if condition.Type == routev1.RouteAdmitted && condition.Status == corev1.ConditionTrue {
283+
return true, nil
197284
}
198285
}
199286
}
200-
return false, nil
201-
})
202-
o.Expect(err).NotTo(o.HaveOccurred(), "route was not admitted")
203-
204-
g.By("Creating exec pod for curl tests")
205-
execPod := exutil.CreateExecPodOrFail(oc.AdminKubeClient(), ns, "execpod")
206-
defer func() {
207-
oc.AdminKubeClient().CoreV1().Pods(ns).Delete(ctx, execPod.Name, *metav1.NewDeleteOptions(1))
208-
}()
209-
210-
g.By("Verifying route is reachable over IPv4")
211-
err = waitForDualStackRouteResponse(ns, execPod.Name, routeHost, "-4", 10*time.Minute)
212-
o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv4")
213-
214-
g.By("Verifying route is reachable over IPv6")
215-
err = waitForDualStackRouteResponse(ns, execPod.Name, routeHost, "-6", 10*time.Minute)
216-
o.Expect(err).NotTo(o.HaveOccurred(), "route not reachable over IPv6")
287+
}
288+
return false, nil
217289
})
218-
})
290+
o.Expect(err).NotTo(o.HaveOccurred(), "route was not admitted")
291+
}
219292

220-
func waitForDualStackRouteResponse(ns, execPodName, host, ipFlag string, timeout time.Duration) error {
293+
func waitForRouteResponse(ns, execPodName, host, ipFlag string, timeout time.Duration) error {
221294
curlCmd := fmt.Sprintf("curl %s -k -v -m 10 --connect-timeout 5 -o /dev/null https://%s 2>&1", ipFlag, host)
222295
var lastOutput string
223296
err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) {
224297
output, err := e2eoutput.RunHostCmd(ns, execPodName, curlCmd)
225298
lastOutput = output
226-
e2e.Logf("curl %s %s:\n%s", ipFlag, host, output)
227299
if err != nil {
228-
e2e.Logf("curl %s error: %v", ipFlag, err)
229300
return false, nil
230301
}
231302
if strings.Contains(output, "< HTTP/1.1 200") || strings.Contains(output, "< HTTP/2 200") {
303+
e2e.Logf("curl %s %s:\n%s", ipFlag, host, output)
232304
return true, nil
233305
}
234306
return false, nil

0 commit comments

Comments
 (0)