From 63358ba329e6ad27bbbb53b421eacfc6984c9ad5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Mar 2026 16:43:13 +0100 Subject: [PATCH 1/5] Always write out.test.toml; check drift on CI with git diff Instead of failing tests when out.test.toml disagrees, always write it to the source directory. CI now runs git diff --exit-code after tests to catch stale files, consistent with how schema drift is detected. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/push.yml | 7 +++++++ acceptance/acceptance_test.go | 22 ++++++++++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bb096e0f92..ff5fdb9da7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -145,6 +145,13 @@ jobs: - name: Analyze slow tests run: make slowest + - name: Check out.test.toml files are up to date + run: | + if ! git diff --exit-code; then + echo "ERROR: detected changed files in the repository; Most likely you have out.test.toml files that are out of date. Run 'make generate-out-test-toml' to update." + exit 1 + fi + test-exp-aitools: needs: - cleanups diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index b17ab98806..2c39a5fa18 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -296,13 +296,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { config, configPath := internal.LoadConfig(t, dir) skipReason := getSkipReason(&config, configPath) - if testdiff.OverwriteMode || OnlyOutTestToml { - // Generate materialized config for this test - // We do this before skipping the test, so the configs are generated for all tests. - materializedConfig, err := internal.GenerateMaterializedConfig(config) - require.NoError(t, err) - testutil.WriteFile(t, filepath.Join(dir, internal.MaterializedConfigFile), materializedConfig) - } + // Generate materialized config for this test. + // We do this before skipping the test, so the configs are generated for all tests. + materializedConfig, err := internal.GenerateMaterializedConfig(config) + require.NoError(t, err) + testutil.WriteFile(t, filepath.Join(dir, internal.MaterializedConfigFile), materializedConfig) // If only regenerating out.test.toml, skip the actual test execution if OnlyOutTestToml { @@ -666,11 +664,15 @@ func runTest(t *testing.T, continue } - skipRepls := false if relPath == internal.MaterializedConfigFile { - skipRepls = true + // Always write out.test.toml to source dir without comparing. + // CI checks for drift with git diff after tests run. + content := testutil.ReadFile(t, filepath.Join(tmpDir, relPath)) + testutil.WriteFile(t, filepath.Join(dir, relPath), content) + continue } - doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, skipRepls) + + doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, false) } // Make sure there are not unaccounted for new files From 077639ac6c3f293f6ad3be45248e287166db55be Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Mar 2026 16:45:58 +0100 Subject: [PATCH 2/5] Remove redundant second write of out.test.toml The first write in testAccept covers all tests including skipped ones, making the write in runTest and the outputs tracking unnecessary. Co-Authored-By: Claude Sonnet 4.6 --- acceptance/acceptance_test.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 2c39a5fa18..b661f22828 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -537,17 +537,13 @@ func runTest(t *testing.T, testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents) // Generate materialized config for this test - materializedConfig, err := internal.GenerateMaterializedConfig(config) - require.NoError(t, err) - testutil.WriteFile(t, filepath.Join(tmpDir, internal.MaterializedConfigFile), materializedConfig) - inputs := make(map[string]bool, 2) outputs := make(map[string]bool, 2) err = CopyDir(dir, tmpDir, inputs, outputs) require.NoError(t, err) - // Add materialized config to outputs for comparison - outputs[internal.MaterializedConfigFile] = true + // out.test.toml is written to the source dir during test discovery, not compared. + delete(outputs, internal.MaterializedConfigFile) timeout := config.Timeout @@ -664,14 +660,6 @@ func runTest(t *testing.T, continue } - if relPath == internal.MaterializedConfigFile { - // Always write out.test.toml to source dir without comparing. - // CI checks for drift with git diff after tests run. - content := testutil.ReadFile(t, filepath.Join(tmpDir, relPath)) - testutil.WriteFile(t, filepath.Join(dir, relPath), content) - continue - } - doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, false) } From 9c45175026f9788ad2c66678b4ec572ada1aa808 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Mar 2026 17:13:05 +0100 Subject: [PATCH 3/5] Use bash shell for git diff check step on Windows --- .github/workflows/push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ff5fdb9da7..1c06b88db9 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -146,6 +146,7 @@ jobs: run: make slowest - name: Check out.test.toml files are up to date + shell: bash run: | if ! git diff --exit-code; then echo "ERROR: detected changed files in the repository; Most likely you have out.test.toml files that are out of date. Run 'make generate-out-test-toml' to update." From 3397e4e3d9228372ceb7a26c812fd2f307606750 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Mar 2026 17:22:14 +0100 Subject: [PATCH 4/5] Remove unused skipRepls parameter from doComparison The only caller that passed skipRepls=true was for out.test.toml, which is no longer compared via doComparison. Co-Authored-By: Claude Opus 4.6 --- acceptance/acceptance_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index b661f22828..766196d1cd 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -660,7 +660,7 @@ func runTest(t *testing.T, continue } - doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, false) + doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) } // Make sure there are not unaccounted for new files @@ -694,7 +694,7 @@ func runTest(t *testing.T, if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" // Show the contents & support overwrite mode for it: - doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, false) + doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) } } @@ -767,7 +767,7 @@ func addEnvVar(t *testing.T, env []string, repls *testdiff.ReplacementsContext, return append(env, key+"="+newValue) } -func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool, skipRepls bool) { +func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) { pathRef := filepath.Join(dirRef, relPath) pathNew := filepath.Join(dirNew, relPath) bufRef, okRef := tryReading(t, pathRef) @@ -782,7 +782,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN // Apply replacements to the new value only. // The reference value is stored after applying replacements. - if !NoRepl && !skipRepls { + if !NoRepl { valueNew = repls.Replace(valueNew) } From 562966f7b3b7d6738eadbcbe775df6cbb1f3da4d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 13 Mar 2026 18:03:10 +0100 Subject: [PATCH 5/5] Regenerate test outputs after out.test.toml handling change out.test.toml is no longer copied to the test tmpdir, so it no longer appears in sync/upload output. Co-Authored-By: Claude Opus 4.6 --- acceptance/bundle/artifacts/whl_change_version/output.txt | 1 - acceptance/bundle/sync/dryrun/output.txt | 3 +-- acceptance/bundle/sync/output.txt | 3 +-- acceptance/cmd/sync-from-file/output.txt | 3 +-- acceptance/cmd/sync/dryrun/output.txt | 6 ------ acceptance/cmd/sync/output.txt | 3 +-- 6 files changed, 4 insertions(+), 15 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 54c655fbf5..4fc7ff6674 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -37,7 +37,6 @@ dist/my_test_code-0.1.0-py3-none-any.whl "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.test.toml" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" diff --git a/acceptance/bundle/sync/dryrun/output.txt b/acceptance/bundle/sync/dryrun/output.txt index 05e4af4e9e..0fbcbe3d6a 100644 --- a/acceptance/bundle/sync/dryrun/output.txt +++ b/acceptance/bundle/sync/dryrun/output.txt @@ -1,11 +1,10 @@ >>> [CLI] bundle sync --dry-run --output text Warn: Running in dry-run mode. No actual changes will be made. -Action: PUT: .gitignore, databricks.yml, out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql +Action: PUT: .gitignore, databricks.yml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql Initial Sync Complete Uploaded .gitignore Uploaded databricks.yml -Uploaded out.test.toml Uploaded project-folder Uploaded project-folder/app.py Uploaded project-folder/app.yaml diff --git a/acceptance/bundle/sync/output.txt b/acceptance/bundle/sync/output.txt index 3f3ac7d4ce..6ff42f4191 100644 --- a/acceptance/bundle/sync/output.txt +++ b/acceptance/bundle/sync/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle sync --output text -Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py, out.test.toml +Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py Initial Sync Complete Uploaded .gitignore Uploaded databricks.yml @@ -10,4 +10,3 @@ Uploaded dryrun/out.test.toml Uploaded ignored-folder/folder1 Uploaded ignored-folder/folder1/script.py Uploaded ignored-folder/script.py -Uploaded out.test.toml diff --git a/acceptance/cmd/sync-from-file/output.txt b/acceptance/cmd/sync-from-file/output.txt index 8188a400ac..a47751a0d1 100644 --- a/acceptance/cmd/sync-from-file/output.txt +++ b/acceptance/cmd/sync-from-file/output.txt @@ -1,10 +1,9 @@ >>> [CLI] sync . /Users/[USERNAME] --exclude-from ignore.test-fixture -Action: PUT: .gitignore, ignore.test-fixture, out.test.toml, project-folder/app.py, project-folder/app.yaml +Action: PUT: .gitignore, ignore.test-fixture, project-folder/app.py, project-folder/app.yaml Initial Sync Complete Uploaded .gitignore Uploaded ignore.test-fixture -Uploaded out.test.toml Uploaded project-folder Uploaded project-folder/app.py Uploaded project-folder/app.yaml diff --git a/acceptance/cmd/sync/dryrun/output.txt b/acceptance/cmd/sync/dryrun/output.txt index 6a68ab4144..091b812701 100644 --- a/acceptance/cmd/sync/dryrun/output.txt +++ b/acceptance/cmd/sync/dryrun/output.txt @@ -3,7 +3,6 @@ Warn: Running in dry-run mode. No actual changes will be made. Initial Sync Complete Uploaded .gitignore -Uploaded out.test.toml Uploaded project-folder Uploaded project-folder/app.py Uploaded project-folder/app.yaml @@ -13,7 +12,6 @@ Uploaded project-folder/query.sql Warn: Running in dry-run mode. No actual changes will be made. Initial Sync Complete Uploaded .gitignore -Uploaded out.test.toml Uploaded project-folder Uploaded project-folder/query.sql @@ -21,7 +19,6 @@ Uploaded project-folder/query.sql Warn: Running in dry-run mode. No actual changes will be made. Initial Sync Complete Uploaded .gitignore -Uploaded out.test.toml >>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/*.py Warn: Running in dry-run mode. No actual changes will be made. @@ -29,7 +26,6 @@ Initial Sync Complete Uploaded .gitignore Uploaded ignored-folder Uploaded ignored-folder/script.py -Uploaded out.test.toml >>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/**/*.py Warn: Running in dry-run mode. No actual changes will be made. @@ -38,7 +34,6 @@ Uploaded .gitignore Uploaded ignored-folder/folder1 Uploaded ignored-folder/folder1/script.py Uploaded ignored-folder/script.py -Uploaded out.test.toml >>> [CLI] sync . /Users/[USERNAME] --dry-run --include ignored-folder/** --include !ignored-folder/folder1/big-blob Warn: Running in dry-run mode. No actual changes will be made. @@ -48,7 +43,6 @@ Uploaded ignored-folder/folder1 Uploaded ignored-folder/folder1/script.py Uploaded ignored-folder/folder1/script.yaml Uploaded ignored-folder/script.py -Uploaded out.test.toml Uploaded project-folder Uploaded project-folder/app.py Uploaded project-folder/app.yaml diff --git a/acceptance/cmd/sync/output.txt b/acceptance/cmd/sync/output.txt index ba778f7f87..04fb43abac 100644 --- a/acceptance/cmd/sync/output.txt +++ b/acceptance/cmd/sync/output.txt @@ -1,11 +1,10 @@ >>> [CLI] sync . /Users/[USERNAME] -Action: PUT: .gitignore, dryrun/out.test.toml, out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql +Action: PUT: .gitignore, dryrun/out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql Initial Sync Complete Uploaded .gitignore Uploaded dryrun Uploaded dryrun/out.test.toml -Uploaded out.test.toml Uploaded project-folder Uploaded project-folder/app.py Uploaded project-folder/app.yaml