Adapt egressIP tests to ipv6primary and ipv4primary dualstack cluster

Author: MiguelCarpio

test/extended/openstack/egressip.go

@@ -5,14 +5,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"net/netip"
 	"os"
+	"strings"
 
 	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips"
 	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
-	"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
 	g "github.com/onsi/ginkgo/v2"
 	o "github.com/onsi/gomega"
+	configv1 "github.com/openshift/api/config/v1"
 	cloudnetwork "github.com/openshift/client-go/cloudnetwork/clientset/versioned"
 	exutil "github.com/openshift/origin/test/extended/util"
 
@@ -36,7 +38,6 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre
 	var networkClient *gophercloud.ServiceClient
 	var clientSet *kubernetes.Clientset
 	var err error
-	var infraID string
 	var workerNodeList *corev1.NodeList
 	var cloudNetworkClientset cloudnetwork.Interface
 	oc := exutil.NewCLI("openstack")
@@ -60,10 +61,6 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre
 			e2eskipper.Skipf("Test not applicable for proxy setup")
 		}
 
-		infrastructure, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{})
-		o.Expect(err).NotTo(o.HaveOccurred())
-		infraID = infrastructure.Status.InfrastructureName
-
 		g.By("Getting the worker node list")
 		workerNodeList, err = clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{
 			LabelSelector: "node-role.kubernetes.io/worker",
@@ -102,13 +99,15 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre
 		defer node.RemoveLabelOffNode(clientSet, primaryWorker.Name, egressAssignableLabelKey)
 
 		g.By(fmt.Sprintf("Getting the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey))
-		egressIPNetCidrStr, err := getEgressIPNetwork(primaryWorker)
+		egressIPNetCidrStr, egressPortId, err := getEgressNetworkInfo(primaryWorker, "ipv4")
 		o.Expect(err).NotTo(o.HaveOccurred())
 		o.Expect(egressIPNetCidrStr).NotTo(o.BeEmpty(), "Could not get the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey)
 		e2e.Logf("Found the EgressIP network: %s", egressIPNetCidrStr)
+		o.Expect(egressPortId).NotTo(o.BeEmpty(), "Could not get the Egress openstack portId from the '%s' annotation", egressIPConfigAnnotationKey)
+		e2e.Logf("Found the Egress PortID: %s", egressPortId)
 
-		g.By("Obtaining a not in use IP address from the EgressIP network (machineNetwork cidr)")
-		machineNetworkID, err := getNetworkIdFromSubnetCidr(networkClient, egressIPNetCidrStr, infraID)
+		g.By("Finding the openstack network ID from the egressPortId defined in openshift node annotation")
+		machineNetworkID, err := getNetworkIdFromPortId(networkClient, egressPortId)
 		o.Expect(err).NotTo(o.HaveOccurred())
 		o.Expect(machineNetworkID).NotTo(o.BeEmpty(), "Could not get the EgressIP network ID in openstack for '%s' subnet CIDR", egressIPNetCidrStr)
 		e2e.Logf("Found the EgressIP network ID '%s' in Openstack for the EgressIP CIDR '%s'", machineNetworkID, egressIPNetCidrStr)
@@ -124,29 +123,9 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre
 		e2e.Logf("Created '%s' temporary directory", egressIPTempDir)
 		defer os.RemoveAll(egressIPTempDir)
 
-		g.By("Creating an EgressIP yaml file")
-		var egressIPname = "egress-ip"
-		var egressIPYamlFileName = "egressip.yaml"
-		var egressIPYamlFilePath = egressIPTempDir + "/" + egressIPYamlFileName
-		var egressIPYamlTemplate = `apiVersion: k8s.ovn.org/v1
-kind: EgressIP
-metadata:
-  name: %s
-spec:
-  egressIPs:
-  - %s
-  namespaceSelector:
-    matchLabels:
-      %s`
-
-		egressIPYaml := fmt.Sprintf(egressIPYamlTemplate, egressIPname, egressIPAddrStr, "app: egress")
-		e2e.Logf("egressIPYaml: %s", egressIPYaml)
-
-		err = os.WriteFile(egressIPYamlFilePath, []byte(egressIPYaml), 0644)
-		o.Expect(err).NotTo(o.HaveOccurred())
-
-		g.By(fmt.Sprintf("Creating an EgressIP object from '%s'", egressIPYamlFilePath))
-		err = oc.AsAdmin().Run("create").Args("-f", egressIPYamlFilePath).Execute()
+		g.By("Create egressIP resource in openshift")
+		egressIPname := "egress-ip"
+		err = createEgressIpResource(oc, egressIPname, egressIPAddrStr, "app: egress")
 		o.Expect(err).NotTo(o.HaveOccurred())
 		defer oc.AsAdmin().Run("delete").Args("egressip", egressIPname).Execute()
 
@@ -228,6 +207,81 @@ spec:
 		e2e.Logf("Found the expected Fixed IP (%s) for the FIP '%s'", egressIPAddrStr, fip.FloatingIP)
 
 	})
+
+	// https://issues.redhat.com/browse/OCPBUGS-27222
+	g.It("with IPv6 format should be created on dualstack or ssipv6 cluster with OVN-Kubernetes NetworkType", func() {
+
+		networks, err := oc.AdminConfigClient().ConfigV1().Networks().Get(ctx, "cluster", metav1.GetOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred())
+		dualstack, err := isDualStackCluster(networks.Status.ClusterNetwork)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		ssipv6, err := isSingleStackIpv6Cluster(ctx, oc)
+		o.Expect(err).NotTo(o.HaveOccurred())
+
+		if !(dualstack || ssipv6) {
+			e2eskipper.Skipf("Test only applicable for dualstack or SingleStack IPv6 clusters")
+		}
+
+		g.By("Getting the network type")
+		networkType, err := getNetworkType(ctx, oc)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		if networkType != NetworkTypeOVNKubernetes {
+			e2eskipper.Skipf("Test not applicable for '%s' NetworkType (only valid for '%s')", networkType, NetworkTypeOVNKubernetes)
+		}
+
+		e2e.Logf("Getting the worker for the EgressIP")
+		worker := workerNodeList.Items[0]
+
+		g.By(fmt.Sprintf("Getting the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey))
+		egressIPNetCidrStr, egressPortId, err := getEgressNetworkInfo(worker, "ipv6")
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(egressIPNetCidrStr).NotTo(o.BeEmpty(), "Could not get the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey)
+		e2e.Logf("Found the EgressIP network: %s", egressIPNetCidrStr)
+		o.Expect(egressPortId).NotTo(o.BeEmpty(), "Could not get the Egress openstack portId from the '%s' annotation", egressPortId)
+		e2e.Logf("Found the Egress PortID: %s", egressPortId)
+
+		g.By("Finding the openstack network ID from the egressPortId defined in openshift node annotation")
+		machineNetworkID, err := getNetworkIdFromPortId(networkClient, egressPortId)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(machineNetworkID).NotTo(o.BeEmpty(), "Could not get the EgressIP network ID in openstack for '%s' subnet CIDR", egressIPNetCidrStr)
+		e2e.Logf("Found the EgressIP network ID '%s' in Openstack for the EgressIP CIDR '%s'", machineNetworkID, egressIPNetCidrStr)
+
+		// Label the worker node as EgressIP assignable node
+		g.By(fmt.Sprintf("Labeling the primary node '%s' with '%s'", worker.Name, egressAssignableLabelKey))
+		node.AddOrUpdateLabelOnNode(clientSet, worker.Name, egressAssignableLabelKey, "dummy")
+		defer node.RemoveLabelOffNode(clientSet, worker.Name, egressAssignableLabelKey)
+
+		g.By("Looking for a free IP in the subnet to use for the egressIP object in openshift")
+		egressIPAddrStr, err := getNotInUseEgressIP(networkClient, egressIPNetCidrStr, machineNetworkID)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(egressIPAddrStr).NotTo(o.BeEmpty(), "Couldn't find a free IP address in '%s' network in Openstack", egressIPNetCidrStr)
+		o.Expect(isIPv6(egressIPAddrStr)).To(o.BeTrue(), "egressIP should be IPv6 but it's %q", egressIPAddrStr)
+		e2e.Logf("Found '%s' free IP address in the EgressIP network in Openstack", egressIPAddrStr)
+
+		g.By("Create egressIP resource in openshift")
+		egressIPname := "egress-ipv6"
+		err = createEgressIpResource(oc, egressIPname, egressIPAddrStr, "app: egress")
+		o.Expect(err).NotTo(o.HaveOccurred())
+		defer oc.AsAdmin().Run("delete").Args("egressip", egressIPname).Execute()
+
+		g.By("Waiting until CloudPrivateIPConfig is created and assigned to the primary worker node")
+		cloudNetworkClientset, err = cloudnetwork.NewForConfig(oc.AdminConfig())
+		o.Expect(err).NotTo(o.HaveOccurred())
+		egressIPAddr, err := netip.ParseAddr(egressIPAddrStr)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		waitOk, err := waitCloudPrivateIPConfigAssignedNode(ctx, cloudNetworkClientset, strings.ReplaceAll(egressIPAddr.StringExpanded(), ":", "."), worker.Name)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(waitOk).To(o.BeTrue(), "Not found the expected assigned node '%s' in '%s' CloudPrivateIPConfig", worker.Name, egressIPAddrStr)
+		e2e.Logf("Found the expected assigned node '%s' in '%s' CloudPrivateIPConfig", worker.Name, egressIPAddrStr)
+
+		g.By("Checking that the port exists from openstack perspective")
+		egressNetInUseIPs, err := getInUseIPs(networkClient, machineNetworkID)
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(egressNetInUseIPs).To(o.ContainElement(egressIPAddrStr))
+
+		g.By("Checking that the allowed_addresses_pairs are properly updated for the worker in the Openstack")
+		checkAllowedAddressesPairs(networkClient, worker, corev1.Node{}, egressIPAddrStr, machineNetworkID)
+	})
 })
 
 // getNotInUseEgressIP returns a not in use IP address from the EgressIP network CIDR
@@ -248,8 +302,8 @@ func getNotInUseEgressIP(client *gophercloud.ServiceClient, egressCidr string, e
 	return freeIP, nil
 }
 
-// getEgressIPNetwork returns the IP address from the node egress-ipconfig annotation
-func getEgressIPNetwork(node corev1.Node) (string, error) {
+// getEgressNetworkInfo returns the IP address CIDR and openstack portId from the node egress-ipconfig annotation
+func getEgressNetworkInfo(node corev1.Node, ipVersion string) (string, string, error) {
 	type ifAddr struct {
 		IPv4 string `json:"ipv4,omitempty"`
 		IPv6 string `json:"ipv6,omitempty"`
@@ -268,40 +322,39 @@ func getEgressIPNetwork(node corev1.Node) (string, error) {
 	annotation, ok := node.Annotations[egressIPConfigAnnotationKey]
 	if !ok {
 		e2e.Logf("Annotation '%s' not found in '%s' node", egressIPConfigAnnotationKey, node.Name)
-		return "", nil
+		return "", "", nil
 	}
 	e2e.Logf("Found '%s' annotation in '%s': %s", egressIPConfigAnnotationKey, node.Name, annotation)
 	var nodeEgressIPConfigs []*NodeEgressIPConfiguration
 	err := json.Unmarshal([]byte(annotation), &nodeEgressIPConfigs)
 	if err != nil {
-		return "", err
+		return "", "", err
 	}
-	egressIPNetStr := nodeEgressIPConfigs[0].IFAddr.IPv4
-	if egressIPNetStr == "" {
-		e2e.Logf("Empty ifaddr.ipv4 in the '%s' annotation", egressIPConfigAnnotationKey)
-		return "", nil
+	if ipVersion == "ipv4" {
+		egressIPNetStr := nodeEgressIPConfigs[0].IFAddr.IPv4
+		if egressIPNetStr == "" {
+			e2e.Logf("Empty ifaddr.ipv4 in the '%s' annotation", egressIPConfigAnnotationKey)
+			return "", "", nil
+		}
+		return egressIPNetStr, nodeEgressIPConfigs[0].Interface, nil
+	} else if ipVersion == "ipv6" {
+		egressIPNetStr := nodeEgressIPConfigs[0].IFAddr.IPv6
+		if egressIPNetStr == "" {
+			e2e.Logf("Empty ifaddr.ipv6 in the '%s' annotation", egressIPConfigAnnotationKey)
+			return "", "", nil
+		}
+		return egressIPNetStr, nodeEgressIPConfigs[0].Interface, nil
 	}
-	return egressIPNetStr, nil
+	return "", "", fmt.Errorf("ipVersion %s is not supported, only ipv4 and ipv6", ipVersion)
 }
 
-// getNetworkIdFromSubnetCidr returns the Openstack network ID for a given Openstack subnet CIDR
-func getNetworkIdFromSubnetCidr(client *gophercloud.ServiceClient, subnetCidr string, infraID string) (string, error) {
-	listOpts := subnets.ListOpts{
-		CIDR: subnetCidr,
-		Tags: "openshiftClusterID=" + infraID,
-	}
-	allPages, err := subnets.List(client, listOpts).AllPages()
-	if err != nil {
-		return "", fmt.Errorf("failed to get subnets")
-	}
-	allSubnets, err := subnets.ExtractSubnets(allPages)
+// getNetworkIdFromPortId returns the Openstack network ID for a given Openstack port
+func getNetworkIdFromPortId(client *gophercloud.ServiceClient, portId string) (string, error) {
+	port, err := ports.Get(client, portId).Extract()
 	if err != nil {
-		return "", fmt.Errorf("failed to extract subnets")
+		return "", fmt.Errorf("failed to get port %s: %w", portId, err)
 	}
-	if len(allSubnets) != 1 {
-		return "", fmt.Errorf("unexpected number of subnets found  with '%s' CIDR: %d subnets", subnetCidr, len(allSubnets))
-	}
-	return allSubnets[0].NetworkID, nil
+	return port.NetworkID, nil
 }
 
 // getInUseIPs returns the in use IPs in a given network ID in Openstack
@@ -443,17 +496,37 @@ func waitCloudPrivateIPConfigAssignedNode(ctx context.Context, cloudNetClientset
 
 // Returns the list of IPs present on the openstack allowed_address_pairs attribute in the node main port
 func getAllowedIPsFromNode(client *gophercloud.ServiceClient, node corev1.Node, machineNetwork string) ([]string, error) {
+	var nodeOpenStackPort *ports.Port
+	var err error
 
-	result := []string{}
-	ip := node.GetAnnotations()["alpha.kubernetes.io/provided-node-ip"]
-	nodePorts, err := getPortsByIP(client, ip, machineNetwork)
-	if err != nil {
-		return nil, err
+	// In a dualstack environment, the node can have both ipv4 and ipv6 addresses. We are looking for the port associated
+	// to the machine network. We iterate over all the node's internal IPs until we find the one that has a port in the
+	// machine network.
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			var nodePorts []ports.Port
+			nodePorts, err = getPortsByIP(client, addr.Address, machineNetwork)
+			if err != nil {
+				// We can safely ignore the error because we are iterating over all the node's internal IPs, and we only
+				// need to find one that has a port in the machine network.
+				e2e.Logf("Can't find a port for IP %s in network %s, skipping.", addr.Address, machineNetwork)
+				continue
+			}
+			// We expect to find only one port for the node's internal IP in the machine network.
+			if len(nodePorts) == 1 {
+				nodeOpenStackPort = &nodePorts[0]
+				e2e.Logf("Found port %s for IP %s in network %s", nodeOpenStackPort.ID, addr.Address, machineNetwork)
+				break
+			}
+		}
 	}
-	if len(nodePorts) != 1 {
-		return nil, fmt.Errorf("unexpected number of openstack ports for IP %s", ip)
+
+	if nodeOpenStackPort == nil {
+		return nil, fmt.Errorf("failed to find the node's port in the machine network %s", machineNetwork)
 	}
-	for _, addressPair := range nodePorts[0].AllowedAddressPairs {
+
+	result := []string{}
+	for _, addressPair := range nodeOpenStackPort.AllowedAddressPairs {
 		result = append(result, addressPair.IPAddress)
 	}
 	return result, nil
@@ -488,3 +561,64 @@ func checkAllowedAddressesPairs(client *gophercloud.ServiceClient, nodeHoldingEg
 	}, "10s", "1s").Should(o.BeTrue(), "Timed out checking allowed address pairs for node %s", nodeNotHoldingEgressIp.Name)
 	e2e.Logf("egressIp %s correctly not included on the node allowed-address-pairs for %s", egressIp, nodeNotHoldingEgressIp.Name)
 }
+
+func createEgressIpResource(oc *exutil.CLI, egressIPname string, egressIPAddrStr string, labels string) error {
+
+	g.By("Creating a temp directory")
+	egressIPTempDir, err := os.MkdirTemp("", "egressip-e2e")
+	if err != nil {
+		return err
+	}
+	e2e.Logf("Created '%s' temporary directory", egressIPTempDir)
+	defer os.RemoveAll(egressIPTempDir)
+
+	g.By("Creating an EgressIP yaml file")
+	var egressIPYamlFileName = "egressip.yaml"
+	var egressIPYamlFilePath = egressIPTempDir + "/" + egressIPYamlFileName
+	var egressIPYamlTemplate = `apiVersion: k8s.ovn.org/v1
+kind: EgressIP
+metadata:
+  name: %s
+spec:
+  egressIPs:
+  - %s
+  namespaceSelector:
+    matchLabels:
+      %s`
+
+	egressIPYaml := fmt.Sprintf(egressIPYamlTemplate, egressIPname, egressIPAddrStr, labels)
+	e2e.Logf("egressIPYaml: %s", egressIPYaml)
+
+	err = os.WriteFile(egressIPYamlFilePath, []byte(egressIPYaml), 0644)
+	if err != nil {
+		return err
+	}
+
+	g.By(fmt.Sprintf("Creating an EgressIP object from '%s'", egressIPYamlFilePath))
+	err = oc.AsAdmin().Run("create").Args("-f", egressIPYamlFilePath).Execute()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// isDualStackCluster returns true if the cluster is dual stack
+func isDualStackCluster(clusterNetwork []configv1.ClusterNetworkEntry) (bool, error) {
+	return len(clusterNetwork) > 1, nil
+}
+
+// isSingleStackIpv6Cluster returns true if the cluster is single stack IPv6
+func isSingleStackIpv6Cluster(ctx context.Context, oc *exutil.CLI) (bool, error) {
+	networks, err := oc.AdminConfigClient().ConfigV1().Networks().Get(ctx, "cluster", metav1.GetOptions{})
+	if err != nil {
+		return false, err
+	}
+	if len(networks.Status.ClusterNetwork) == 1 && net.ParseIP(networks.Status.ClusterNetwork[0].CIDR).To4() == nil {
+		return true, nil
+	}
+	return false, nil
+}
+
+func isIPv6(ip string) bool {
+	return net.ParseIP(ip) != nil && net.ParseIP(ip).To4() == nil
+}