Always write out.test.toml; check drift on CI with git diff

.github/workflows/push.yml

@@ -145,6 +145,14 @@ jobs:
       - name: Analyze slow tests
         run: make slowest
 
+      - name: Check out.test.toml files are up to date
+        shell: bash
+        run: |
+          if ! git diff --exit-code; then
+            echo "ERROR: detected changed files in the repository; Most likely you have out.test.toml files that are out of date. Run 'make generate-out-test-toml' to update."
+            exit 1
+          fi
+
   test-exp-aitools:
     needs:
       - cleanups

acceptance/acceptance_test.go

@@ -296,13 +296,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 			config, configPath := internal.LoadConfig(t, dir)
 			skipReason := getSkipReason(&config, configPath)
 
-			if testdiff.OverwriteMode || OnlyOutTestToml {
-				// Generate materialized config for this test
-				// We do this before skipping the test, so the configs are generated for all tests.
-				materializedConfig, err := internal.GenerateMaterializedConfig(config)
-				require.NoError(t, err)
-				testutil.WriteFile(t, filepath.Join(dir, internal.MaterializedConfigFile), materializedConfig)
-			}
+			// Generate materialized config for this test.
+			// We do this before skipping the test, so the configs are generated for all tests.
+			materializedConfig, err := internal.GenerateMaterializedConfig(config)
+			require.NoError(t, err)
+			testutil.WriteFile(t, filepath.Join(dir, internal.MaterializedConfigFile), materializedConfig)
 
 			// If only regenerating out.test.toml, skip the actual test execution
 			if OnlyOutTestToml {
@@ -539,17 +537,13 @@ func runTest(t *testing.T,
 	testutil.WriteFile(t, filepath.Join(tmpDir, EntryPointScript), scriptContents)
 
 	// Generate materialized config for this test
-	materializedConfig, err := internal.GenerateMaterializedConfig(config)
-	require.NoError(t, err)
-	testutil.WriteFile(t, filepath.Join(tmpDir, internal.MaterializedConfigFile), materializedConfig)
-
 	inputs := make(map[string]bool, 2)
 	outputs := make(map[string]bool, 2)
 	err = CopyDir(dir, tmpDir, inputs, outputs)
 	require.NoError(t, err)
 
-	// Add materialized config to outputs for comparison
-	outputs[internal.MaterializedConfigFile] = true
+	// out.test.toml is written to the source dir during test discovery, not compared.
+	delete(outputs, internal.MaterializedConfigFile)
 
 	timeout := config.Timeout
 
@@ -666,11 +660,7 @@ func runTest(t *testing.T,
 			continue
 		}
 
-		skipRepls := false
-		if relPath == internal.MaterializedConfigFile {
-			skipRepls = true
-		}
-		doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, skipRepls)
+		doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
 	}
 
 	// Make sure there are not unaccounted for new files
@@ -704,7 +694,7 @@ func runTest(t *testing.T,
 		if strings.HasPrefix(relPath, "out") {
 			// We have a new file starting with "out"
 			// Show the contents & support overwrite mode for it:
-			doComparison(t, repls, dir, tmpDir, relPath, &printedRepls, false)
+			doComparison(t, repls, dir, tmpDir, relPath, &printedRepls)
 		}
 	}
 
@@ -777,7 +767,7 @@ func addEnvVar(t *testing.T, env []string, repls *testdiff.ReplacementsContext,
 	return append(env, key+"="+newValue)
 }
 
-func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool, skipRepls bool) {
+func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) {
 	pathRef := filepath.Join(dirRef, relPath)
 	pathNew := filepath.Join(dirNew, relPath)
 	bufRef, okRef := tryReading(t, pathRef)
@@ -792,7 +782,7 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN
 
 	// Apply replacements to the new value only.
 	// The reference value is stored after applying replacements.
-	if !NoRepl && !skipRepls {
+	if !NoRepl {
 		valueNew = repls.Replace(valueNew)
 	}
 

acceptance/bundle/artifacts/whl_change_version/output.txt

@@ -37,7 +37,6 @@ dist/my_test_code-0.1.0-py3-none-any.whl
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__init__.py"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.test.toml"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json"
 "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script"

acceptance/bundle/sync/dryrun/output.txt

@@ -1,11 +1,10 @@
 
 >>> [CLI] bundle sync --dry-run --output text
 Warn: Running in dry-run mode. No actual changes will be made.
-Action: PUT: .gitignore, databricks.yml, out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
+Action: PUT: .gitignore, databricks.yml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
 Initial Sync Complete
 Uploaded .gitignore
 Uploaded databricks.yml
-Uploaded out.test.toml
 Uploaded project-folder
 Uploaded project-folder/app.py
 Uploaded project-folder/app.yaml

acceptance/bundle/sync/output.txt

@@ -1,6 +1,6 @@
 
 >>> [CLI] bundle sync --output text
-Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py, out.test.toml
+Action: PUT: .gitignore, databricks.yml, dryrun/databricks.yml, dryrun/out.test.toml, ignored-folder/folder1/script.py, ignored-folder/script.py
 Initial Sync Complete
 Uploaded .gitignore
 Uploaded databricks.yml
@@ -10,4 +10,3 @@ Uploaded dryrun/out.test.toml
 Uploaded ignored-folder/folder1
 Uploaded ignored-folder/folder1/script.py
 Uploaded ignored-folder/script.py
-Uploaded out.test.toml

acceptance/cmd/sync-from-file/output.txt

@@ -1,10 +1,9 @@
 
 >>> [CLI] sync . /Users/[USERNAME] --exclude-from ignore.test-fixture
-Action: PUT: .gitignore, ignore.test-fixture, out.test.toml, project-folder/app.py, project-folder/app.yaml
+Action: PUT: .gitignore, ignore.test-fixture, project-folder/app.py, project-folder/app.yaml
 Initial Sync Complete
 Uploaded .gitignore
 Uploaded ignore.test-fixture
-Uploaded out.test.toml
 Uploaded project-folder
 Uploaded project-folder/app.py
 Uploaded project-folder/app.yaml

acceptance/cmd/sync/dryrun/output.txt

@@ -3,7 +3,6 @@
 Warn: Running in dry-run mode. No actual changes will be made.
 Initial Sync Complete
 Uploaded .gitignore
-Uploaded out.test.toml
 Uploaded project-folder
 Uploaded project-folder/app.py
 Uploaded project-folder/app.yaml
@@ -13,23 +12,20 @@ Uploaded project-folder/query.sql
 Warn: Running in dry-run mode. No actual changes will be made.
 Initial Sync Complete
 Uploaded .gitignore
-Uploaded out.test.toml
 Uploaded project-folder
 Uploaded project-folder/query.sql
 
 >>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql
 Warn: Running in dry-run mode. No actual changes will be made.
 Initial Sync Complete
 Uploaded .gitignore
-Uploaded out.test.toml
 
 >>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/*.py
 Warn: Running in dry-run mode. No actual changes will be made.
 Initial Sync Complete
 Uploaded .gitignore
 Uploaded ignored-folder
 Uploaded ignored-folder/script.py
-Uploaded out.test.toml
 
 >>> [CLI] sync . /Users/[USERNAME] --dry-run --exclude project-folder/app.* --exclude project-folder/query.sql --include ignored-folder/**/*.py
 Warn: Running in dry-run mode. No actual changes will be made.
@@ -38,7 +34,6 @@ Uploaded .gitignore
 Uploaded ignored-folder/folder1
 Uploaded ignored-folder/folder1/script.py
 Uploaded ignored-folder/script.py
-Uploaded out.test.toml
 
 >>> [CLI] sync . /Users/[USERNAME] --dry-run --include ignored-folder/** --include !ignored-folder/folder1/big-blob
 Warn: Running in dry-run mode. No actual changes will be made.
@@ -48,7 +43,6 @@ Uploaded ignored-folder/folder1
 Uploaded ignored-folder/folder1/script.py
 Uploaded ignored-folder/folder1/script.yaml
 Uploaded ignored-folder/script.py
-Uploaded out.test.toml
 Uploaded project-folder
 Uploaded project-folder/app.py
 Uploaded project-folder/app.yaml

acceptance/cmd/sync/output.txt

@@ -1,11 +1,10 @@
 
 >>> [CLI] sync . /Users/[USERNAME]
-Action: PUT: .gitignore, dryrun/out.test.toml, out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
+Action: PUT: .gitignore, dryrun/out.test.toml, project-folder/app.py, project-folder/app.yaml, project-folder/query.sql
 Initial Sync Complete
 Uploaded .gitignore
 Uploaded dryrun
 Uploaded dryrun/out.test.toml
-Uploaded out.test.toml
 Uploaded project-folder
 Uploaded project-folder/app.py
 Uploaded project-folder/app.yaml