Add --force-refresh support for Databricks CLI token fetching

Try `--force-refresh` before the regular CLI command so the SDK can
bypass the CLI's own token cache when the SDK considers its token stale.
If the CLI is too old to recognise `--force-refresh` (or `--profile`),
gracefully fall back to the next command in the chain.

Chain order:
- with profile: forceCmd (--profile --force-refresh) -> profileCmd (--profile) -> fallbackCmd (--host)
- without profile: forceCmd (--host --force-refresh) -> profileCmd (--host)

Azure CLI callers are unchanged; they use constructors that leave
forceCmd null, preserving existing behavior.

Signed-off-by: Mihai Mitrea <mihai.mitrea@databricks.com>

Author: mihaimitrea-db

NEXT_CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### New Features and Improvements
 * Added automatic detection of AI coding agents (Antigravity, Claude Code, Cline, Codex, Copilot CLI, Cursor, Gemini CLI, OpenCode) in the user-agent string. The SDK now appends `agent/<name>` to HTTP request headers when running inside a known AI agent environment.
+* Pass `--force-refresh` to Databricks CLI `auth token` command so the SDK always receives a fresh token instead of a potentially stale one from the CLI's internal cache. Falls back gracefully on older CLIs that do not support this flag.
 
 ### Bug Fixes
 * Fixed Databricks CLI authentication to detect when the cached token's scopes don't match the SDK's configured scopes. Previously, a scope mismatch was silently ignored, causing requests to use wrong permissions. The SDK now raises an error with instructions to re-authenticate.

databricks-sdk-java/src/main/java/com/databricks/sdk/core/CliTokenSource.java

@@ -25,15 +25,17 @@
 public class CliTokenSource implements TokenSource {
   private static final Logger LOG = LoggerFactory.getLogger(CliTokenSource.class);
 
-  private List<String> cmd;
+  // forceCmd is tried before profileCmd when non-null. If the CLI rejects
+  // --force-refresh or --profile, execution falls through to profileCmd.
+  private List<String> forceCmd;
+
+  private List<String> profileCmd;
   private String tokenTypeField;
   private String accessTokenField;
   private String expiryField;
   private Environment env;
-  // fallbackCmd is tried when the primary command fails with "unknown flag: --profile",
-  // indicating the CLI is too old to support --profile. Can be removed once support
-  // for CLI versions predating --profile is dropped.
-  // See: https://github.com/databricks/databricks-sdk-go/pull/1497
+  // fallbackCmd is tried when profileCmd fails with "unknown flag: --profile",
+  // indicating the CLI is too old to support --profile.
   private List<String> fallbackCmd;
 
   /**
@@ -58,7 +60,7 @@ public CliTokenSource(
       String accessTokenField,
       String expiryField,
       Environment env) {
-    this(cmd, tokenTypeField, accessTokenField, expiryField, env, null);
+    this(cmd, tokenTypeField, accessTokenField, expiryField, env, null, null);
   }
 
   public CliTokenSource(
@@ -68,14 +70,26 @@ public CliTokenSource(
       String expiryField,
       Environment env,
       List<String> fallbackCmd) {
+    this(cmd, tokenTypeField, accessTokenField, expiryField, env, fallbackCmd, null);
+  }
+
+  public CliTokenSource(
+      List<String> cmd,
+      String tokenTypeField,
+      String accessTokenField,
+      String expiryField,
+      Environment env,
+      List<String> fallbackCmd,
+      List<String> forceCmd) {
     super();
-    this.cmd = OSUtils.get(env).getCliExecutableCommand(cmd);
+    this.profileCmd = OSUtils.get(env).getCliExecutableCommand(cmd);
     this.tokenTypeField = tokenTypeField;
     this.accessTokenField = accessTokenField;
     this.expiryField = expiryField;
     this.env = env;
     this.fallbackCmd =
         fallbackCmd != null ? OSUtils.get(env).getCliExecutableCommand(fallbackCmd) : null;
+    this.forceCmd = forceCmd != null ? OSUtils.get(env).getCliExecutableCommand(forceCmd) : null;
   }
 
   /**
@@ -153,18 +167,22 @@ private Token execCliCommand(List<String> cmdToRun) throws IOException {
     }
   }
 
-  @Override
-  public Token getToken() {
+  private String getErrorText(IOException e) {
+    return e instanceof CliCommandException
+        ? ((CliCommandException) e).getFullOutput()
+        : e.getMessage();
+  }
+
+  private boolean isUnknownFlagError(String errorText, String flag) {
+    return errorText != null && errorText.contains("unknown flag: " + flag);
+  }
+
+  private Token execProfileCmdWithFallback() {
     try {
-      return execCliCommand(this.cmd);
+      return execCliCommand(this.profileCmd);
     } catch (IOException e) {
-      String textToCheck =
-          e instanceof CliCommandException
-              ? ((CliCommandException) e).getFullOutput()
-              : e.getMessage();
-      if (fallbackCmd != null
-          && textToCheck != null
-          && textToCheck.contains("unknown flag: --profile")) {
+      String textToCheck = getErrorText(e);
+      if (fallbackCmd != null && isUnknownFlagError(textToCheck, "--profile")) {
         LOG.warn(
             "Databricks CLI does not support --profile flag. Falling back to --host. "
                 + "Please upgrade your CLI to the latest version.");
@@ -177,4 +195,26 @@ public Token getToken() {
       throw new DatabricksException(e.getMessage(), e);
     }
   }
+
+  @Override
+  public Token getToken() {
+    if (forceCmd == null) {
+      return execProfileCmdWithFallback();
+    }
+
+    try {
+      return execCliCommand(this.forceCmd);
+    } catch (IOException e) {
+      String textToCheck = getErrorText(e);
+      if (isUnknownFlagError(textToCheck, "--force-refresh")
+          || isUnknownFlagError(textToCheck, "--profile")) {
+        LOG.warn(
+            "Databricks CLI does not support --force-refresh flag. "
+                + "Falling back to regular token fetch. "
+                + "Please upgrade your CLI to the latest version.");
+        return execProfileCmdWithFallback();
+      }
+      throw new DatabricksException(e.getMessage(), e);
+    }
+  }
 }

databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksCliCredentialsProvider.java

@@ -69,6 +69,17 @@ List<String> buildHostArgs(String cliPath, DatabricksConfig config) {
     return cmd;
   }
 
+  List<String> buildProfileArgs(String cliPath, DatabricksConfig config) {
+    return new ArrayList<>(
+        Arrays.asList(cliPath, "auth", "token", "--profile", config.getProfile()));
+  }
+
+  private static List<String> withForceRefresh(List<String> cmd) {
+    List<String> forceCmd = new ArrayList<>(cmd);
+    forceCmd.add("--force-refresh");
+    return forceCmd;
+  }
+
   private CliTokenSource getDatabricksCliTokenSource(DatabricksConfig config) {
     String cliPath = config.getDatabricksCliPath();
     if (cliPath == null) {
@@ -79,25 +90,23 @@ private CliTokenSource getDatabricksCliTokenSource(DatabricksConfig config) {
       return null;
     }
 
-    List<String> cmd;
+    List<String> profileCmd;
     List<String> fallbackCmd = null;
+    List<String> forceCmd;
 
     if (config.getProfile() != null) {
-      // When profile is set, use --profile as the primary command.
-      // The profile contains the full config (host, account_id, etc.).
-      cmd =
-          new ArrayList<>(
-              Arrays.asList(cliPath, "auth", "token", "--profile", config.getProfile()));
-      // Build a --host fallback for older CLIs that don't support --profile.
+      profileCmd = buildProfileArgs(cliPath, config);
+      forceCmd = withForceRefresh(profileCmd);
       if (config.getHost() != null) {
         fallbackCmd = buildHostArgs(cliPath, config);
       }
     } else {
-      cmd = buildHostArgs(cliPath, config);
+      profileCmd = buildHostArgs(cliPath, config);
+      forceCmd = withForceRefresh(profileCmd);
     }
 
     return new CliTokenSource(
-        cmd, "token_type", "access_token", "expiry", config.getEnv(), fallbackCmd);
+        profileCmd, "token_type", "access_token", "expiry", config.getEnv(), fallbackCmd, forceCmd);
   }
 
   @Override