[pull] master from apache:master

.github/workflows/superset-docs-deploy.yml

@@ -1,6 +1,13 @@
 name: Docs Deployment
 
 on:
+  # Deploy after integration tests complete on master
+  workflow_run:
+    workflows: ["Python-Integration"]
+    types: [completed]
+    branches: [master]
+
+  # Also allow manual trigger and direct pushes to docs
   push:
     paths:
       - "docs/**"
@@ -30,9 +37,10 @@ jobs:
     name: Build & Deploy
     runs-on: ubuntu-24.04
     steps:
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      - name: "Checkout ${{ github.event.workflow_run.head_sha || github.sha }}"
         uses: actions/checkout@v6
         with:
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
           persist-credentials: false
           submodules: recursive
       - name: Set up Node.js
@@ -58,6 +66,35 @@ jobs:
         working-directory: docs
         run: |
           yarn install --check-cache
+      - name: Download database diagnostics (if triggered by integration tests)
+        if: github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success'
+        uses: dawidd6/action-download-artifact@v6
+        continue-on-error: true
+        with:
+          workflow: superset-python-integrationtest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+          name: database-diagnostics
+          path: docs/src/data/
+      - name: Try to download latest diagnostics (for push/dispatch triggers)
+        if: github.event_name != 'workflow_run'
+        uses: dawidd6/action-download-artifact@v6
+        continue-on-error: true
+        with:
+          workflow: superset-python-integrationtest.yml
+          name: database-diagnostics
+          path: docs/src/data/
+          branch: master
+          search_artifacts: true
+          if_no_artifact_found: warn
+      - name: Use diagnostics artifact if available
+        working-directory: docs
+        run: |
+          if [ -f "src/data/databases-diagnostics.json" ]; then
+            echo "Using fresh diagnostics from integration tests"
+            mv src/data/databases-diagnostics.json src/data/databases.json
+          else
+            echo "Using committed databases.json (no artifact found)"
+          fi
       - name: yarn build
         working-directory: docs
         run: |
@@ -71,5 +108,5 @@ jobs:
           destination-github-username: "apache"
           destination-repository-name: "superset-site"
           target-branch: "asf-site"
-          commit-message: "deploying docs: ${{ github.event.head_commit.message }} (apache/superset@${{ github.sha }})"
+          commit-message: "deploying docs: ${{ github.event.head_commit.message || 'triggered by integration tests' }} (apache/superset@${{ github.event.workflow_run.head_sha || github.sha }})"
           user-email: dev@superset.apache.org

.github/workflows/superset-docs-verify.yml

@@ -4,17 +4,23 @@ on:
   pull_request:
     paths:
       - "docs/**"
+      - "superset/db_engine_specs/**"
       - ".github/workflows/superset-docs-verify.yml"
     types: [synchronize, opened, reopened, ready_for_review]
+  workflow_run:
+    workflows: ["Python-Integration"]
+    types: [completed]
 
 # cancel previous workflow jobs for PRs
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.workflow_run.head_sha || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   linkinator:
     # See docs here: https://github.com/marketplace/actions/linkinator
+    # Only run on pull_request, not workflow_run
+    if: github.event_name == 'pull_request'
     name: Link Checking
     runs-on: ubuntu-latest
     steps:
@@ -50,8 +56,11 @@ jobs:
             https://timbr.ai/,
             https://opensource.org/license/apache-2-0,
             https://www.plaidcloud.com/
-  build-deploy:
-    name: Build & Deploy
+
+  build-on-pr:
+    # Build docs when PR changes docs/** (uses committed databases.json)
+    if: github.event_name == 'pull_request'
+    name: Build (PR trigger)
     runs-on: ubuntu-24.04
     defaults:
       run:
@@ -75,3 +84,50 @@ jobs:
       - name: yarn build
         run: |
           yarn build
+
+  build-after-tests:
+    # Build docs after integration tests complete (uses fresh diagnostics)
+    # Only runs if integration tests succeeded
+    if: >
+      github.event_name == 'workflow_run' &&
+      github.event.workflow_run.conclusion == 'success'
+    name: Build (after integration tests)
+    runs-on: ubuntu-24.04
+    defaults:
+      run:
+        working-directory: docs
+    steps:
+      - name: "Checkout PR head: ${{ github.event.workflow_run.head_sha }}"
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+          persist-credentials: false
+          submodules: recursive
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: './docs/.nvmrc'
+      - name: yarn install
+        run: |
+          yarn install --check-cache
+      - name: Download database diagnostics from integration tests
+        uses: dawidd6/action-download-artifact@v6
+        with:
+          workflow: superset-python-integrationtest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+          name: database-diagnostics
+          path: docs/src/data/
+      - name: Use fresh diagnostics
+        run: |
+          if [ -f "src/data/databases-diagnostics.json" ]; then
+            echo "Using fresh diagnostics from integration tests"
+            mv src/data/databases-diagnostics.json src/data/databases.json
+          else
+            echo "Warning: No diagnostics artifact found, using committed data"
+          fi
+      - name: yarn typecheck
+        run: |
+          yarn typecheck
+      - name: yarn build
+        run: |
+          yarn build

.github/workflows/superset-python-integrationtest.yml

@@ -73,6 +73,36 @@ jobs:
           flags: python,mysql
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
+      - name: Generate database diagnostics for docs
+        if: steps.check.outputs.python
+        env:
+          SUPERSET_CONFIG: tests.integration_tests.superset_test_config
+          SUPERSET__SQLALCHEMY_DATABASE_URI: |
+            mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
+        run: |
+          python -c "
+          import json
+          from superset.app import create_app
+          from superset.db_engine_specs.lib import generate_yaml_docs
+          app = create_app()
+          with app.app_context():
+              docs = generate_yaml_docs()
+              # Wrap in the expected format
+              output = {
+                  'generated': '$(date -Iseconds)',
+                  'databases': docs
+              }
+              with open('databases-diagnostics.json', 'w') as f:
+                  json.dump(output, f, indent=2, default=str)
+              print(f'Generated diagnostics for {len(docs)} databases')
+          "
+      - name: Upload database diagnostics artifact
+        if: steps.check.outputs.python
+        uses: actions/upload-artifact@v4
+        with:
+          name: database-diagnostics
+          path: databases-diagnostics.json
+          retention-days: 7
   test-postgres:
     runs-on: ubuntu-24.04
     strategy:

.pre-commit-config.yaml

@@ -142,3 +142,10 @@ repos:
             else
               echo "No Python files to lint."
             fi
+      - id: db-engine-spec-metadata
+        name: database engine spec metadata validation
+        entry: python superset/db_engine_specs/lint_metadata.py --strict
+        language: system
+        files: ^superset/db_engine_specs/.*\.py$
+        exclude: ^superset/db_engine_specs/(base|lib|lint_metadata|__init__)\.py$
+        pass_filenames: false

.rat-excludes

@@ -75,6 +75,9 @@ postgresql.svg
 snowflake.svg
 ydb.svg
 loading.svg
+apache-solr.svg
+azure.svg
+superset.svg
 
 # docs third-party logos, i.e. docs/static/img/logos/*
 logos/*

AGENTS.md

@@ -2,6 +2,27 @@
 
 Apache Superset is a data visualization platform with Flask/Python backend and React/TypeScript frontend.
 
+## ⚠️ CRITICAL: Always Run Pre-commit Before Pushing
+
+**ALWAYS run `pre-commit run --all-files` before pushing commits.** CI will fail if pre-commit checks don't pass. This is non-negotiable.
+
+```bash
+# Stage your changes first
+git add .
+
+# Run pre-commit on all files
+pre-commit run --all-files
+
+# If there are auto-fixes, stage them and commit
+git add .
+git commit --amend  # or new commit
+```
+
+Common pre-commit failures:
+- **Formatting** - black, prettier, eslint will auto-fix
+- **Type errors** - mypy failures need manual fixes
+- **Linting** - ruff, pylint issues need manual fixes
+
 ## ⚠️ CRITICAL: Ongoing Refactors (What NOT to Do)
 
 **These migrations are actively happening - avoid deprecated patterns:**

README.md

@@ -101,51 +101,54 @@ Superset provides:
 
 ## Supported Databases
 
-Superset can query data from any SQL-speaking datastore or data engine (Presto, Trino, Athena, [and more](https://superset.apache.org/docs/configuration/databases)) that has a Python DB-API driver and a SQLAlchemy dialect.
+Superset can query data from any SQL-speaking datastore or data engine (Presto, Trino, Athena, [and more](https://superset.apache.org/docs/databases)) that has a Python DB-API driver and a SQLAlchemy dialect.
 
 Here are some of the major database solutions that are supported:
 
+<!-- SUPPORTED_DATABASES_START -->
 <p align="center">
-  <img src="https://superset.apache.org/img/databases/redshift.png" alt="redshift" border="0" width="200"/>
-  <img src="https://superset.apache.org/img/databases/google-biquery.png" alt="google-bigquery" border="0" width="200"/>
-  <img src="https://superset.apache.org/img/databases/snowflake.png" alt="snowflake" border="0" width="200"/>
-  <img src="https://superset.apache.org/img/databases/trino.png" alt="trino" border="0" width="150" />
-  <img src="https://superset.apache.org/img/databases/presto.png" alt="presto" border="0" width="200"/>
-  <img src="https://superset.apache.org/img/databases/databricks.png" alt="databricks" border="0" width="160" />
-  <img src="https://superset.apache.org/img/databases/druid.png" alt="druid" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/firebolt.png" alt="firebolt" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/timescale.png" alt="timescale" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/postgresql.png" alt="postgresql" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/mysql.png" alt="mysql" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/mssql-server.png" alt="mssql-server" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/ibm-db2.svg" alt="db2" border="0" width="220" />
-  <img src="https://superset.apache.org/img/databases/sqlite.png" alt="sqlite" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/sybase.png" alt="sybase" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/mariadb.png" alt="mariadb" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/vertica.png" alt="vertica" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/oracle.png" alt="oracle" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/firebird.png" alt="firebird" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/greenplum.png" alt="greenplum" border="0" width="200"  />
-  <img src="https://superset.apache.org/img/databases/clickhouse.png" alt="clickhouse" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/exasol.png" alt="exasol" border="0" width="160" />
-  <img src="https://superset.apache.org/img/databases/monet-db.png" alt="monet-db" border="0" width="200"  />
-  <img src="https://superset.apache.org/img/databases/apache-kylin.png" alt="apache-kylin" border="0" width="80"/>
-  <img src="https://superset.apache.org/img/databases/hologres.png" alt="hologres" border="0" width="80"/>
-  <img src="https://superset.apache.org/img/databases/netezza.png" alt="netezza" border="0" width="80"/>
-  <img src="https://superset.apache.org/img/databases/pinot.png" alt="pinot" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/teradata.png" alt="teradata" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/yugabyte.png" alt="yugabyte" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/databend.png" alt="databend" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/starrocks.png" alt="starrocks" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/doris.png" alt="doris" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/oceanbase.svg" alt="oceanbase" border="0" width="220" />
-  <img src="https://superset.apache.org/img/databases/sap-hana.png" alt="sap-hana" border="0" width="220" />
-  <img src="https://superset.apache.org/img/databases/denodo.png" alt="denodo" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/ydb.svg" alt="ydb" border="0" width="200" />
-  <img src="https://superset.apache.org/img/databases/tdengine.png" alt="TDengine" border="0" width="200" />
+  <img src="https://superset.apache.org/img/databases/doris.png" alt="apache-doris" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/apache-drill.png" alt="apache-drill" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/druid.png" alt="apache-druid" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/apache-hive.svg" alt="apache-hive" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/apache-impala.png" alt="apache-impala" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/apache-kylin.png" alt="apache-kylin" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/apache-pinot.svg" alt="apache-pinot" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/amazon-athena.jpg" alt="aws-athena" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/redshift.png" alt="aws-redshift" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/clickhouse.png" alt="clickhouse" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/couchbase.svg" alt="couchbase" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/databend.png" alt="databend" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/databricks.png" alt="databricks" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/denodo.png" alt="denodo" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/dremio.png" alt="dremio" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/exasol.png" alt="exasol" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/firebird.png" alt="firebird" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/firebolt.png" alt="firebolt" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/google-big-query.svg" alt="google-bigquery" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/google-sheets.svg" alt="google-sheets" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/ibm-db2.svg" alt="ibm-db2" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/netezza.png" alt="ibm-netezza" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/mariadb.png" alt="mariadb" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/msql.png" alt="microsoft-sql-server" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/mysql.png" alt="mysql" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/oceanbase.svg" alt="oceanbase" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/oraclelogo.png" alt="oracle" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/postgresql.svg" alt="postgresql" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/presto-og.png" alt="presto" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/sap-hana.png" alt="sap-hana" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/snowflake.svg" alt="snowflake" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/sqlite.png" alt="sqlite" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/starrocks.png" alt="starrocks" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/tdengine.png" alt="tdengine" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/teradata.png" alt="teradata" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/trino.png" alt="trino" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/vertica.png" alt="vertica" border="0" width="120" height="60" class="database-logo" />
+  <img src="https://superset.apache.org/img/databases/ydb.svg" alt="ydb" border="0" width="120" height="60" class="database-logo" />
 </p>
+<!-- SUPPORTED_DATABASES_END -->
 
-**A more comprehensive list of supported databases** along with the configuration instructions can be found [here](https://superset.apache.org/docs/configuration/databases).
+**A more comprehensive list of supported databases** along with the configuration instructions can be found [here](https://superset.apache.org/docs/databases).
 
 Want to add support for your datastore or data engine? Read more [here](https://superset.apache.org/docs/frequently-asked-questions#does-superset-work-with-insert-database-engine-here) about the technical requirements.
 

docs/.gitignore

@@ -26,3 +26,10 @@ docs/intro.md
 
 # Generated badge images (downloaded at build time by remark-localize-badges plugin)
 static/badges/
+
+# Generated database documentation MDX files (regenerated at build time)
+# Source of truth is in superset/db_engine_specs/*.py metadata attributes
+docs/databases/
+
+# Note: src/data/databases.json is COMMITTED (not ignored) to preserve feature diagnostics
+# that require Flask context to generate. Update it locally with: npm run gen-db-docs