diff --git a/.gitmodules b/.gitmodules index 9dde877ce..80d741477 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "cluster-deployment-automation"] path = cluster-deployment-automation - url = https://github.com/bn222/cluster-deployment-automation + url = https://github.com/samd2021/cluster-deployment-automation [submodule "kubernetes-traffic-flow-tests"] path = kubernetes-traffic-flow-tests url = https://github.com/ovn-kubernetes/kubernetes-traffic-flow-tests diff --git a/README.md b/README.md index 0929cfe2f..69e61bc64 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,34 @@ The build target will use previously built images if they are available to speed task clean-image-all ``` -6. **Alternative: Remote Registry** +6. **Registry Configuration** -Alternatively, if you set up a registry remotely, define the `REGISTERY` variable. Note that you need to do this for the build step and the push/run step: +The DPU operator now uses in-cluster registries by default with predictable URLs based on your cluster configuration: + +- **Host Registry**: `default-route-openshift-image-registry.apps.ocpcluster.redhat.com` +- **DPU Registry**: `default-route-openshift-image-registry.apps.172.16.3.16.nip.io` + +You can check the current configuration: +```sh +task show-registry-config +``` + +To verify the configuration matches your actual clusters (when they're running): +```sh +task verify-registry-routes +``` + +**Custom Registry Configuration**: If your cluster domains or IPs differ, you can override them: ```sh -REGISTERY=... task ... +# Override cluster domain (affects host registry) +HOST_CLUSTER_DOMAIN=mycluster.example.com task build-image-all + +# Override DPU IP (affects DPU registry) +DPU_CLUSTER_IP=10.1.2.3 task build-image-all + +# Override registry URLs directly +REGISTRY_HOST=my-host-registry.com REGISTRY_DPU=my-dpu-registry.com task build-image-all ``` 7. **Alternative: Deploy only one of the cluster** diff --git a/cluster-deployment-automation b/cluster-deployment-automation index a8760d251..ce7ef8753 160000 --- a/cluster-deployment-automation +++ b/cluster-deployment-automation @@ -1 +1 @@ -Subproject commit a8760d2513538051427491d1fc1ae561c3e4258f +Subproject commit ce7ef875307352adfed3955539b89b247cde428f diff --git a/hack/cluster-configs/config-dpu-host.yaml b/hack/cluster-configs/config-dpu-host.yaml index 42555cbb9..97b531bad 100644 --- a/hack/cluster-configs/config-dpu-host.yaml +++ b/hack/cluster-configs/config-dpu-host.yaml @@ -6,6 +6,7 @@ clusters: version: "4.19.0-nightly" network_api_port: "{{api_network()}}" postconfig: + - name: "image_registry" - name: "dpu_operator_host" dpu_operator_path: "../../" - name: "monitoring_config" diff --git a/hack/cluster-configs/config-dpu.yaml b/hack/cluster-configs/config-dpu.yaml index e4f1f6e49..0f2b0cdd0 100644 --- a/hack/cluster-configs/config-dpu.yaml +++ b/hack/cluster-configs/config-dpu.yaml @@ -5,6 +5,7 @@ clusters: network_api_port: "{{secondary_network_port()}}" kind: "iso" install_iso: "{{iso_server()}}/RHEL-9.6.0-20250416.8-aarch64-dvd1-w-kickstart.iso" + kubeconfig: "/root/kubeconfig.microshift" masters: - name: "{{worker_number(0)}}-acc" node: "localhost" @@ -26,5 +27,7 @@ clusters: organization_id: "{{organization_id()}}" activation_key: "{{activation_key()}}" - name: "microshift" + - name: "image_registry" + registry_type: "microshift" - name: "dpu_operator_dpu" dpu_operator_path: "../../" diff --git a/internal/testutils/testcluster.go b/internal/testutils/testcluster.go index 3f993b704..ee8ace8bf 100644 --- a/internal/testutils/testcluster.go +++ b/internal/testutils/testcluster.go @@ -110,7 +110,7 @@ func GetDPUNodes(c client.Client) ([]corev1.Node, error) { func TrafficFlowTestsImage() string { localContainer := ContainerImage{ Registry: os.Getenv("REGISTRY"), - Name: "ovn-kubernetes/kubernetes-traffic-flow-tests", + Name: "kubernetes-traffic-flow-tests", Tag: "latest", } @@ -125,6 +125,7 @@ func TrafficFlowTestsImage() string { Expect(err).To(BeNil()) return localContainer.FullRef() } + return remoteContainer.FullRef() } diff --git a/taskfile.yaml b/taskfile.yaml index 335d9b7bf..4152baa61 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -14,8 +14,11 @@ vars: BINDIR: bin BINDIR_ABS: sh: if [[ "{{.BINDIR}}" = /* ]]; then echo "{{.BINDIR}}"; else echo "$(pwd)/{{.BINDIR}}"; fi - REGISTRY: - sh: hostname | sed 's/$/:5000/' + HOST_CLUSTER_DOMAIN: '{{ default "ocpcluster.redhat.com" .HOST_CLUSTER_DOMAIN }}' + DPU_CLUSTER_IP: '{{ default "172.16.3.16" .DPU_CLUSTER_IP }}' + REGISTRY_HOST: '{{ default (printf "default-route-openshift-image-registry.apps.%s/in-cluster-registry" .HOST_CLUSTER_DOMAIN) .REGISTRY_HOST }}' + REGISTRY_DPU: '{{ default (printf "default-route-openshift-image-registry.apps.%s.nip.io/in-cluster-registry" .DPU_CLUSTER_IP) .REGISTRY_DPU }}' + REGISTRY: '{{.REGISTRY_HOST}}' ENVTEST_K8S_VERSION: 1.27.1 KUSTOMIZE_VERSION: v5.6.0 GINKGO_VERSION: @@ -48,13 +51,52 @@ tasks: cmds: - echo "{{.BINDIR_ABS}}" + show-registry-config: + desc: "Show current registry configuration" + cmds: + - 'echo "Host Registry: {{.REGISTRY_HOST}}"' + - 'echo "DPU Registry: {{.REGISTRY_DPU}}"' + - 'echo "Default Registry: {{.REGISTRY}}"' + + verify-registry-routes: + desc: "Verify registry routes match actual cluster routes (requires running clusters)" + cmds: + - | + echo "Verifying registry routes..." + echo "Expected host registry: {{.REGISTRY_HOST}}" + if oc get route default-route -n openshift-image-registry --kubeconfig /root/kubeconfig.ocpcluster &>/dev/null; then + ACTUAL_HOST=$(oc get route default-route -n openshift-image-registry --template='{{`{{ .spec.host }}`}}' --kubeconfig /root/kubeconfig.ocpcluster) + echo "Actual host registry: $ACTUAL_HOST" + if [ "{{.REGISTRY_HOST}}" = "$ACTUAL_HOST" ]; then + echo "Host registry matches" + else + echo "Host registry mismatch - consider setting HOST_CLUSTER_DOMAIN environment variable" + fi + else + echo "Host cluster not available or registry route not found" + fi + + echo "" + echo "Expected DPU registry: {{.REGISTRY_DPU}}" + if oc get route default-route -n openshift-image-registry --kubeconfig /root/kubeconfig.microshift &>/dev/null; then + ACTUAL_DPU=$(oc get route default-route -n openshift-image-registry --template='{{`{{ .spec.host }}`}}' --kubeconfig /root/kubeconfig.microshift) + echo "Actual DPU registry: $ACTUAL_DPU" + if [ "{{.REGISTRY_DPU}}" = "$ACTUAL_DPU" ]; then + echo "DPU registry matches" + else + echo "DPU registry mismatch - consider setting DPU_CLUSTER_IP environment variable" + fi + else + echo "DPU cluster not available or registry route not found" + fi + push-image-helper: internal: true vars: SOURCE: '{{.SOURCE}}' IMAGE: '{{.IMAGE}}' cmds: - - buildah manifest push --all '{{.SOURCE}}-manifest' 'docker://{{.IMAGE}}' + - buildah manifest push --all --retry 3 --retry-delay 10s '{{.SOURCE}}-manifest' 'docker://{{.IMAGE}}' undeploy-helper: internal: true @@ -95,12 +137,23 @@ tasks: deps: - task: kustomize cmds: + - mkdir -p bin/host bin/dpu - > go run ./tools/config/config.go - -registry-url {{.REGISTRY}} + -registry-url {{.REGISTRY_HOST}} -template-file config/dev/local-images-template.yaml - -output-file bin/local-images.yaml - - cp config/dev/kustomization.yaml bin + -output-file bin/host/local-images.yaml + - > + go run ./tools/config/config.go + -registry-url {{.REGISTRY_DPU}} + -template-file config/dev/local-images-template.yaml + -output-file bin/dpu/local-images.yaml + - cp config/dev/kustomization.yaml bin/host/ + - cp config/dev/kustomization.yaml bin/dpu/ + - sed -i 's/local-images-template.yaml/local-images.yaml/' bin/host/kustomization.yaml + - sed -i 's/local-images-template.yaml/local-images.yaml/' bin/dpu/kustomization.yaml + - sed -i 's|../config/default|../../config/default|' bin/host/kustomization.yaml + - sed -i 's|../config/default|../../config/default|' bin/dpu/kustomization.yaml ## Download envtest-setup locally if necessary envtest: @@ -118,8 +171,8 @@ tasks: vars: KUBECONFIG_DPU: "/root/kubeconfig.microshift" KUBECONFIG_HOST: "/root/kubeconfig.ocpcluster" - - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.microshift" oc apply -f - - - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f - + - bin/kustomize build bin/dpu | KUBECONFIG="/root/kubeconfig.microshift" oc apply -f - + - bin/kustomize build bin/host | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f - - KUBECONFIG="/root/kubeconfig.microshift" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s - KUBECONFIG="/root/kubeconfig.ocpcluster" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s @@ -130,16 +183,21 @@ tasks: - task: undeploy-1c vars: KUBECONFIG_HOST: "/root/kubeconfig.ocpcluster" - - bin/kustomize build bin | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f - + - bin/kustomize build bin/host | KUBECONFIG="/root/kubeconfig.ocpcluster" oc apply -f - - KUBECONFIG="/root/kubeconfig.ocpcluster" oc -n openshift-dpu-operator wait --for=condition=ready pod --all --timeout=300s - prepare-e2e-test: + ensure-test-images-in-both-registries: + desc: "Ensure test images are available in both registries" cmds: - - > - if [ "{{.SUBMODULES}}" = "true" ]; then - hack/prepare-submodules.sh - fi - hack/prepare-venv.sh + - | + echo "Ensuring test images are available in both registries..." + skopeo copy \ + docker://ghcr.io/ovn-kubernetes/kubernetes-traffic-flow-tests:latest \ + docker://{{.REGISTRY_HOST}}/kubernetes-traffic-flow-tests:latest + skopeo copy \ + docker://ghcr.io/ovn-kubernetes/kubernetes-traffic-flow-tests:latest \ + docker://{{.REGISTRY_DPU}}/kubernetes-traffic-flow-tests:latest + echo "Test images copied to both registries" e2e-test: deps: @@ -159,6 +217,7 @@ tasks: - task: ginkgo - task: envtest cmds: + - task: ensure-test-images-in-both-registries - > FAST_TEST=true REGISTRY={{.REGISTRY}} diff --git a/taskfiles/images.yaml b/taskfiles/images.yaml index 2cd309c5a..7b32af1a4 100644 --- a/taskfiles/images.yaml +++ b/taskfiles/images.yaml @@ -180,7 +180,7 @@ tasks: build-cpagent-service: internal: true vars: - IMAGE: '{{.REGISTRY}}/mrvl-cpagent:dev' + IMAGE: '{{.REGISTRY_HOST}}/mrvl-cpagent:dev' CPIMAGE: "Environment=IMAGE='{{.IMAGE}}'" CPFILE: 'internal/daemon/vendor-specific-plugins/marvell/cp-agent/cp-agent.service' cmds: @@ -376,37 +376,71 @@ tasks: - task: push-image-all push-image-all: - deps: + cmds: # pushing images in parallel overwhelm the image registry and cause 503 errors + # Push to host registry + - task: push-image-helper + vars: + SOURCE: 'localhost/dpu-operator:dev' + IMAGE: '{{.REGISTRY_HOST}}/dpu-operator:dev' + - task: push-image-helper + vars: + SOURCE: 'localhost/dpu-daemon:dev' + IMAGE: '{{.REGISTRY_HOST}}/dpu-daemon:dev' + - task: push-image-helper + vars: + SOURCE: 'localhost/mrvl-vsp:dev' + IMAGE: '{{.REGISTRY_HOST}}/mrvl-vsp:dev' + - task: push-image-helper + vars: + SOURCE: 'localhost/mrvl-cpagent:dev' + IMAGE: '{{.REGISTRY_HOST}}/mrvl-cpagent:dev' + - task: push-image-helper + vars: + SOURCE: 'localhost/intel-vsp:dev' + IMAGE: '{{.REGISTRY_HOST}}/intel-vsp:dev' + - task: push-image-helper + vars: + SOURCE: 'localhost/intel-vsp-p4:dev' + IMAGE: '{{.REGISTRY_HOST}}/intel-vsp-p4:dev' + - task: push-image-helper + vars: + SOURCE: 'localhost/intel-netsec-vsp:dev' + IMAGE: '{{.REGISTRY_HOST}}/intel-netsec-vsp:dev' + - task: push-image-helper + vars: + SOURCE: 'localhost/network-resources-injector:dev' + IMAGE: '{{.REGISTRY_HOST}}/network-resources-injector:dev' + # Push to DPU registry - task: push-image-helper vars: SOURCE: 'localhost/dpu-operator:dev' - IMAGE: '{{.REGISTRY}}/dpu-operator:dev' + IMAGE: '{{.REGISTRY_DPU}}/dpu-operator:dev' - task: push-image-helper vars: SOURCE: 'localhost/dpu-daemon:dev' - IMAGE: '{{.REGISTRY}}/dpu-daemon:dev' + IMAGE: '{{.REGISTRY_DPU}}/dpu-daemon:dev' - task: push-image-helper vars: SOURCE: 'localhost/mrvl-vsp:dev' - IMAGE: '{{.REGISTRY}}/mrvl-vsp:dev' + IMAGE: '{{.REGISTRY_DPU}}/mrvl-vsp:dev' - task: push-image-helper vars: SOURCE: 'localhost/mrvl-cpagent:dev' - IMAGE: '{{.REGISTRY}}/mrvl-cpagent:dev' + IMAGE: '{{.REGISTRY_DPU}}/mrvl-cpagent:dev' - task: push-image-helper vars: SOURCE: 'localhost/intel-vsp:dev' - IMAGE: '{{.REGISTRY}}/intel-vsp:dev' + IMAGE: '{{.REGISTRY_DPU}}/intel-vsp:dev' - task: push-image-helper vars: SOURCE: 'localhost/intel-vsp-p4:dev' - IMAGE: '{{.REGISTRY}}/intel-vsp-p4:dev' + IMAGE: '{{.REGISTRY_DPU}}/intel-vsp-p4:dev' - task: push-image-helper vars: SOURCE: 'localhost/intel-netsec-vsp:dev' - IMAGE: '{{.REGISTRY}}/intel-netsec-vsp:dev' + IMAGE: '{{.REGISTRY_DPU}}/intel-netsec-vsp:dev' - task: push-image-helper vars: SOURCE: 'localhost/network-resources-injector:dev' - IMAGE: '{{.REGISTRY}}/network-resources-injector:dev' + IMAGE: '{{.REGISTRY_DPU}}/network-resources-injector:dev'