From 7db417e16530c93d685f81db6d4768fd0b793fd0 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 8 Apr 2026 17:03:46 +0200 Subject: [PATCH] wait for Kube resource instead of APIBinding to make e2e tests more reliable On-behalf-of: @SAP christoph.mewes@sap.com --- test/e2e/sync/apiexportendpointslice_test.go | 16 ++--- test/e2e/sync/primary_test.go | 66 ++++++++++---------- test/e2e/sync/related_test.go | 40 ++++++------ test/utils/wait.go | 39 ++++-------- 4 files changed, 72 insertions(+), 89 deletions(-) diff --git a/test/e2e/sync/apiexportendpointslice_test.go b/test/e2e/sync/apiexportendpointslice_test.go index f8073f2..3892433 100644 --- a/test/e2e/sync/apiexportendpointslice_test.go +++ b/test/e2e/sync/apiexportendpointslice_test.go @@ -116,10 +116,10 @@ func TestAPIExportEndpointSliceSameCluster(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // In kcp 0.27, the binding' status is not perfectly in-sync with the actual APIs available in @@ -260,10 +260,10 @@ func TestAPIExportEndpointSliceDifferentCluster(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // TODO: Remove this once we do not support kcp 0.27 anymore. diff --git a/test/e2e/sync/primary_test.go b/test/e2e/sync/primary_test.go index 6ed78cc..627159a 100644 --- a/test/e2e/sync/primary_test.go +++ b/test/e2e/sync/primary_test.go @@ -97,10 +97,10 @@ func TestSyncSimpleObject(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // create a Crontab object in a team workspace @@ -190,10 +190,10 @@ func TestSyncSimpleObjectOldNaming(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // create a Crontab object in a team workspace @@ -274,10 +274,10 @@ func TestSyncWithDefaultNamingRules(t *testing.T) { // wait until the API is available kcpClusterClient := utils.GetKcpAdminClusterClient(t) - crontabsGVR := schema.GroupVersionResource{ - Group: "kcp.example.com", - Version: "v1", - Resource: "crontabs", + crontabsGVK := schema.GroupVersionKind{ + Group: "kcp.example.com", + Version: "v1", + Kind: "CronTab", } // create a Crontab object in each team workspace, importantly using the same name and @@ -298,7 +298,7 @@ spec: teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join(team) teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, crontabsGVR) + utils.WaitForBoundAPI(t, ctx, teamClient, crontabsGVK) if err := teamClient.Create(ctx, utils.YAMLToUnstructured(t, crontabYAML)); err != nil { t.Fatalf("Failed to create %s's CronTab in kcp: %v", team, err) @@ -380,10 +380,10 @@ func TestLocalChangesAreKept(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // create a Crontab object in a team workspace @@ -577,10 +577,10 @@ func TestResourceFilter(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // create two Crontab objects in a team workspace @@ -694,10 +694,10 @@ func TestSyncingOverlyLongNames(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // create a namespace and CronTab with extremely long names @@ -796,10 +796,10 @@ func TestSyncWithWorkspacePath(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // create a Crontab object in a team workspace @@ -921,10 +921,10 @@ func TestSyncMultiResources(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(orgWorkspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: kcpGroupName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: kcpGroupName, + Version: "v1", + Kind: "CronTab", }) // create a Crontab object in a team workspace diff --git a/test/e2e/sync/related_test.go b/test/e2e/sync/related_test.go index 4c0ef5a..7a0f0d9 100644 --- a/test/e2e/sync/related_test.go +++ b/test/e2e/sync/related_test.go @@ -593,10 +593,10 @@ func TestSyncRelatedObjects(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(testcase.workspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: apiExportName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: apiExportName, + Version: "v1", + Kind: "CronTab", }) // create a Crontab object in a team workspace @@ -951,10 +951,10 @@ func TestSyncRelatedMultiObjects(t *testing.T) { utils.RunAgent(ctx, t, "bob", orgKubconfig, envtestKubeconfig, apiExportName, "") // wait until the API is available - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: apiExportName, - Version: "v1", - Resource: "backups", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: apiExportName, + Version: "v1", + Kind: "Backup", }) // create a Backup object in a team workspace @@ -1226,10 +1226,10 @@ func TestSyncNonStandardRelatedResources(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join(testcase.workspace).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: apiExportName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: apiExportName, + Version: "v1", + Kind: "CronTab", }) // create a Crontab object in a team workspace @@ -1470,14 +1470,14 @@ func TestSyncNonStandardRelatedResourcesMultipleAPIExports(t *testing.T) { utils.RunAgent(ctx, t, "initroid", initroidOrgKubconfig, envtestKubeconfig, initroidAPIExportName, "agent=initroid") // wait until the APIs are available - for orgWs, gvr := range map[string]schema.GroupVersionResource{ - initechOrgWorkspace: {Group: initechAPIExportName, Version: "v1", Resource: "crontabs"}, - initroidOrgWorkspace: {Group: initroidAPIExportName, Version: "v1", Resource: "backups"}, + for orgWs, gvk := range map[string]schema.GroupVersionKind{ + initechOrgWorkspace: {Group: initechAPIExportName, Version: "v1", Kind: "CronTab"}, + initroidOrgWorkspace: {Group: initroidAPIExportName, Version: "v1", Kind: "Backup"}, } { teamClusterPath := logicalcluster.NewPath("root").Join(orgWs).Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, gvr) + utils.WaitForBoundAPI(t, ctx, teamClient, gvk) } // Since we are claiming resources from other APIExports, the default accepted claims @@ -1620,10 +1620,10 @@ func TestDeletePrimaryWithRelatedKcpResource(t *testing.T) { teamClusterPath := logicalcluster.NewPath("root").Join("delete-primary-related-kcp").Join("team-1") teamClient := kcpClusterClient.Cluster(teamClusterPath) - utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionResource{ - Group: apiExportName, - Version: "v1", - Resource: "crontabs", + utils.WaitForBoundAPI(t, ctx, teamClient, schema.GroupVersionKind{ + Group: apiExportName, + Version: "v1", + Kind: "CronTab", }) // Step 1: Create a CronTab in kcp diff --git a/test/utils/wait.go b/test/utils/wait.go index c2ea631..33bc336 100644 --- a/test/utils/wait.go +++ b/test/utils/wait.go @@ -18,12 +18,10 @@ package utils import ( "context" - "slices" "testing" "time" - kcpapisv1alpha1 "github.com/kcp-dev/sdk/apis/apis/v1alpha1" - + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -45,36 +43,21 @@ func WaitForObject(t *testing.T, ctx context.Context, client ctrlruntimeclient.C t.Logf("%T is ready.", obj) } -func WaitForBoundAPI(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, gvr schema.GroupVersionResource) { +func WaitForBoundAPI(t *testing.T, ctx context.Context, client ctrlruntimeclient.Client, gvk schema.GroupVersionKind) { t.Helper() - t.Logf("Waiting for API %s/%s to be bound in kcp…", gvr.Group, gvr.Resource) - err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (bool, error) { - apiBindings := &kcpapisv1alpha1.APIBindingList{} - err := client.List(ctx, apiBindings) - if err != nil { - return false, err - } + t.Logf("Waiting for API %s/%s to be bound in kcp…", gvk.Group, gvk.Kind) - for _, binding := range apiBindings.Items { - if bindingHasGVR(binding, gvr) { - return true, nil - } - } + // Wait for actual resource availability instead of checking the APIBinding, because this is more + // reliable, especially on slower CI environments. + err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 1*time.Minute, false, func(ctx context.Context) (bool, error) { + // Try to list resources of this type - if the resource isn't ready, this will fail + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk) - return false, nil + return client.List(ctx, list) == nil, nil }) if err != nil { - t.Fatalf("Failed to wait for API %v to become available: %v", gvr, err) + t.Fatalf("Failed to wait for API %v to become available: %v", gvk, err) } } - -func bindingHasGVR(binding kcpapisv1alpha1.APIBinding, gvr schema.GroupVersionResource) bool { - for _, bound := range binding.Status.BoundResources { - if bound.Group == gvr.Group && bound.Resource == gvr.Resource && slices.Contains(bound.StorageVersions, gvr.Version) { - return true - } - } - - return false -}