From 9d45d976a085a5b655387a2b4aa30616995ae3e1 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 16:20:28 +0200 Subject: [PATCH 1/3] direct: ignore UC-managed schema properties as backend defaults UC auto-populates system-managed property keys (e.g. `unity.catalog.managed.delta.defaults.delta.enableRowTracking`) on schema creation. Without a backend_defaults rule, the planner sees the remote map as drift, emits Update, and DoUpdate sends an empty payload which UC rejects with "UpdateSchema Nothing to update". The rule only applies when both saved and new are nil, so user-set properties still drive real drift detection. Also mirror the UC behavior in the fake testserver so the no-drift invariant is exercised locally; added acceptance/.../schemas/drift/ managed_properties covering the reproducer. Co-authored-by: Isaac --- .../drift/managed_properties/databricks.yml | 8 +++++++ .../drift/managed_properties/out.test.toml | 3 +++ .../drift/managed_properties/output.txt | 24 +++++++++++++++++++ .../schemas/drift/managed_properties/script | 11 +++++++++ .../drift/managed_properties/test.toml | 5 ++++ .../resources/schemas/recreate/output.txt | 3 +++ bundle/direct/dresources/resources.yml | 6 +++++ libs/testserver/schemas.go | 7 ++++++ 8 files changed, 67 insertions(+) create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/output.txt create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/script create mode 100644 acceptance/bundle/resources/schemas/drift/managed_properties/test.toml diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml new file mode 100644 index 00000000000..cd05d222c48 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/databricks.yml @@ -0,0 +1,8 @@ +bundle: + name: test-bundle + +resources: + schemas: + schema1: + name: myschema + catalog_name: main diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt new file mode 100644 index 00000000000..869cd97c29d --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/output.txt @@ -0,0 +1,24 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan is a no-op despite UC auto-populating managed properties +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Redeploy is a no-op (no UpdateSchema call) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity +json.method = "POST"; +json.path = "/api/2.1/unity-catalog/schemas"; +json.body.catalog_name = "main"; +json.body.name = "myschema"; diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script new file mode 100644 index 00000000000..c3ef4ac39e4 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -0,0 +1,11 @@ +echo "*" > .gitignore + +title "Initial deployment" +trace $CLI bundle deploy + +title "Plan is a no-op despite UC auto-populating managed properties" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" + +title "Redeploy is a no-op (no UpdateSchema call)" +trace $CLI bundle deploy +trace print_requests.py //unity | gron.py | contains.py '!json.method = "PATCH"' diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml new file mode 100644 index 00000000000..5016e85b395 --- /dev/null +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/test.toml @@ -0,0 +1,5 @@ +RecordRequests = true + +# Terraform issues a spurious PATCH for enable_predictive_optimization on every +# deploy, which is outside the scope of backend-default handling in resources.yml. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/output.txt b/acceptance/bundle/resources/schemas/recreate/output.txt index 7c173eb11f0..bb33023f9d2 100644 --- a/acceptance/bundle/resources/schemas/recreate/output.txt +++ b/acceptance/bundle/resources/schemas/recreate/output.txt @@ -83,6 +83,9 @@ Error: Resource catalog.SchemaInfo not found: main.myschema "metastore_id": "[UUID]", "name": "myschema", "owner": "[USERNAME]", + "properties": { + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true" + }, "schema_id": "[UUID]", "updated_at": [UNIX_TIME_MILLIS][0], "updated_by": "[USERNAME]" diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 569fca9ee82..3375908f3ba 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -323,6 +323,12 @@ resources: reason: immutable - field: storage_root reason: immutable + backend_defaults: + # UC auto-populates system-managed keys like + # `unity.catalog.managed.delta.defaults.delta.enableRowTracking` after create. + # Without this, every subsequent plan produces an Update whose payload is empty, + # and UC rejects it with "UpdateSchema Nothing to update". + - field: properties external_locations: recreate_on_changes: diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 1d4dc79e7ac..92c01bf7fc2 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -39,6 +39,13 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { schema.MetastoreId = TestMetastore.MetastoreId schema.Owner = s.CurrentUser().UserName schema.SchemaId = nextUUID() + if schema.Properties == nil { + // Mirror UC behavior: managed system defaults are populated when the user + // doesn't specify any properties. Required to cover backend-default drift. + schema.Properties = map[string]string{ + "unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", + } + } s.Schemas[schema.FullName] = schema return Response{ From 1f336d8d738bcae71bff5687d0ccb68666e5623e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 18:18:15 +0200 Subject: [PATCH 2/3] Narrow properties --- bundle/direct/bundle_plan_test.go | 61 ++++++++++++++++++++++++++ bundle/direct/dresources/resources.yml | 6 +-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index ccfb7cb517f..1579217ae52 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -3,8 +3,12 @@ package direct import ( "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestDynPathToStructPath(t *testing.T) { @@ -35,3 +39,60 @@ func TestDynPathToStructPath(t *testing.T) { assert.Equal(t, tc.expected, node.String()) } } + +func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { + cfg := dresources.GetResourceConfig("schemas") + require.NotNil(t, cfg) + + tests := []struct { + name string + path string + remote any + expected bool + }{ + { + name: "managed delta row tracking property", + path: "properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking']", + remote: "true", + expected: true, + }, + { + name: "managed iceberg catalog property", + path: "properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged']", + remote: "true", + expected: true, + }, + { + name: "unmanaged remote-only property is not skipped", + path: "properties['custom.remote_only']", + remote: "true", + expected: false, + }, + { + name: "parent properties map is not skipped", + path: "properties", + remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true"}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path, err := structpath.ParsePath(tt.path) + require.NoError(t, err) + + reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{ + Old: nil, + New: nil, + Remote: tt.remote, + }) + + assert.Equal(t, tt.expected, ok) + if tt.expected { + assert.Equal(t, deployplan.ReasonBackendDefault, reason) + } else { + assert.Empty(t, reason) + } + }) + } +} diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 3375908f3ba..1d07bbf4ae7 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -324,11 +324,11 @@ resources: - field: storage_root reason: immutable backend_defaults: - # UC auto-populates system-managed keys like - # `unity.catalog.managed.delta.defaults.delta.enableRowTracking` after create. + # UC auto-populates these system-managed keys after create. # Without this, every subsequent plan produces an Update whose payload is empty, # and UC rejects it with "UpdateSchema Nothing to update". - - field: properties + - field: properties['unity.catalog.managed.delta.defaults.delta.enableRowTracking'] + - field: properties['unity.catalog.managed.iceberg.defaults.delta.feature.catalogManaged'] external_locations: recreate_on_changes: From acc3518cbe638e75d9b701238957a10d12585bcd Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 22 Apr 2026 18:26:37 +0200 Subject: [PATCH 3/3] direct: handle schema backend-default map drift --- bundle/direct/bundle_plan.go | 51 ++++++++++++++++++++++++++----- bundle/direct/bundle_plan_test.go | 28 ++++++++++++++++- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f6bcea316cd..d53c185ed83 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -452,19 +452,56 @@ func shouldSkipBackendDefault(cfg *dresources.ResourceLifecycleConfig, path *str return "", false } for _, rule := range cfg.BackendDefaults { - if !path.HasPatternPrefix(rule.Field) { - continue - } - if len(rule.Values) == 0 { - return deployplan.ReasonBackendDefault, true - } - if matchesAllowedValue(ch.Remote, rule.Values) { + if matchesBackendDefaultRule(path, ch.Remote, rule) { return deployplan.ReasonBackendDefault, true } } + if matchesBackendDefaultMap(cfg, path, ch.Remote) { + return deployplan.ReasonBackendDefault, true + } return "", false } +func matchesBackendDefaultRule(path *structpath.PathNode, remote any, rule dresources.BackendDefaultRule) bool { + if !path.HasPatternPrefix(rule.Field) { + return false + } + if len(rule.Values) == 0 { + return true + } + return matchesAllowedValue(remote, rule.Values) +} + +// matchesBackendDefaultMap handles the nil-vs-map case from structdiff, where a +// remote-only map change is emitted at the parent path rather than per key. +// We only skip the parent map if every remote entry matches a configured +// backend-default child rule; any unmanaged key must still surface as drift. +func matchesBackendDefaultMap(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode, remote any) bool { + rv := reflect.ValueOf(remote) + if !rv.IsValid() || rv.Kind() != reflect.Map || rv.IsNil() || rv.Type().Key().Kind() != reflect.String || rv.Len() == 0 { + return false + } + + iter := rv.MapRange() + for iter.Next() { + childPath := structpath.NewBracketString(path, iter.Key().String()) + childRemote := iter.Value().Interface() + + matched := false + for _, rule := range cfg.BackendDefaults { + if matchesBackendDefaultRule(childPath, childRemote, rule) { + matched = true + break + } + } + if !matched { + return false + } + } + + return true +} + // matchesAllowedValue checks if the remote value matches one of the allowed JSON values. // Each json.RawMessage is unmarshaled into the same type as remote for comparison. func matchesAllowedValue(remote any, values []json.RawMessage) bool { diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index 1579217ae52..0b3191aca30 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -69,9 +69,15 @@ func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { expected: false, }, { - name: "parent properties map is not skipped", + name: "managed-only parent properties map is skipped", path: "properties", remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true"}, + expected: true, + }, + { + name: "mixed parent properties map is not skipped", + path: "properties", + remote: map[string]string{"unity.catalog.managed.delta.defaults.delta.enableRowTracking": "true", "custom.remote_only": "true"}, expected: false, }, } @@ -96,3 +102,23 @@ func TestShouldSkipBackendDefault_SchemaManagedPropertiesOnly(t *testing.T) { }) } } + +// Map drift handling synthesizes child paths to match against rules. structdiff +// always emits map keys in bracket notation, so synthetic child paths must too; +// otherwise rules wouldn't match for identifier-like keys. +func TestShouldSkipBackendDefault_MapDriftUsesBracketKeys(t *testing.T) { + field, err := structpath.ParsePattern("properties['simple']") + require.NoError(t, err) + cfg := &dresources.ResourceLifecycleConfig{ + BackendDefaults: []dresources.BackendDefaultRule{{Field: field}}, + } + + path, err := structpath.ParsePath("properties") + require.NoError(t, err) + + reason, ok := shouldSkipBackendDefault(cfg, path, &deployplan.ChangeDesc{ + Remote: map[string]string{"simple": "v"}, + }) + assert.True(t, ok) + assert.Equal(t, deployplan.ReasonBackendDefault, reason) +}