Merge branch 'main' into fix_project_name_change_error

Author: lafirm

.circleci/config.yml

@@ -7,14 +7,14 @@ on_main_or_tag_filter: &on_main_or_tag_filter
     branches:
       only: main
     tags:
-      only: /^v.+/
+      only: /^v\d+\.\d+\.\d+/
 
 on_tag_filter: &on_tag_filter
   filters:
     branches:
       ignore: /.*/
     tags:
-      only: /^v.+/
+      only: /^v\d+\.\d+\.\d+/
 
 orbs:
   path-filtering: circleci/path-filtering@1.2.0
@@ -39,7 +39,7 @@ jobs:
           command: unset TWINE_USERNAME TWINE_PASSWORD && make publish-tests
   gh-release:
     docker:
-      - image: cimg/node:16.14
+      - image: cimg/node:20.19.0
     resource_class: small
     steps:
       - run:
@@ -54,7 +54,7 @@ jobs:
 
   ui-build:
     docker:
-      - image: cimg/node:19.8
+      - image: cimg/node:20.19.0
     resource_class: medium
     steps:
       - checkout

.circleci/continue_config.yml

@@ -93,9 +93,13 @@ jobs:
       - run:
           name: Run linters and code style checks
           command: make py-style
-      - run:
-          name: Exercise the benchmarks
-          command: make benchmark-ci
+      - unless:
+          condition:
+            equal: ["3.9", << parameters.python_version >>]
+          steps:
+            - run:
+                name: Exercise the benchmarks
+                command: make benchmark-ci
       - run:
           name: Run cicd tests
           command: make cicd-test
@@ -144,8 +148,11 @@ jobs:
       - halt_unless_core
       - checkout
       - run:
-          name: Run the migration test
-          command: ./.circleci/test_migration.sh
+          name: Run the migration test - sushi
+          command: ./.circleci/test_migration.sh sushi "--gateway duckdb_persistent"
+      - run:
+          name: Run the migration test - sushi_dbt
+          command: ./.circleci/test_migration.sh sushi_dbt "--config migration_test_config"
 
   ui_style:
     docker:
@@ -172,7 +179,7 @@ jobs:
 
   ui_test:
     docker:
-      - image: mcr.microsoft.com/playwright:v1.52.0-jammy
+      - image: mcr.microsoft.com/playwright:v1.54.1-jammy
     resource_class: medium
     steps:
       - halt_unless_client
@@ -201,39 +208,6 @@ jobs:
           name: Run tests
           command: npm --prefix web/client run test
 
-  trigger_private_tests:
-    docker:
-      - image: cimg/python:3.12.0
-    resource_class: small
-    steps:
-      - checkout
-      - run:
-          name: Install setuptools scm
-          command: pip install setuptools_scm
-      - run:
-          name: Trigger private tests
-          command: |
-            export COMMIT_MESSAGE="$(git log --format=%s -n 1 $CIRCLE_SHA1)"
-            export FORMATTED_COMMIT_MESSAGE="${COMMIT_MESSAGE//\"/\\\"}"
-            # returns a version string like 0.1.0.dev11
-            export PACKAGE_VERSION="$(python ./.circleci/get_scm_version.py)"
-            curl --request POST \
-              --url $TOBIKO_PRIVATE_CIRCLECI_URL \
-              --header "Circle-Token: $TOBIKO_PRIVATE_CIRCLECI_KEY" \
-              --header "content-type: application/json" \
-              --data '{
-                "branch":"main",
-                "parameters":{
-                  "run_main_pr":false,
-                  "run_sqlmesh_commit":true,
-                  "sqlmesh_branch":"'$CIRCLE_BRANCH'",
-                  "sqlmesh_commit_author":"'$CIRCLE_USERNAME'",
-                  "sqlmesh_commit_hash":"'$CIRCLE_SHA1'",
-                  "sqlmesh_commit_message":"'"$FORMATTED_COMMIT_MESSAGE"'",
-                  "sqlmesh_package_version":"'$PACKAGE_VERSION'"
-                  }
-              }'
-
   engine_tests_docker:
     parameters:
       engine:
@@ -270,6 +244,9 @@ jobs:
     steps:
       - halt_unless_core
       - checkout
+      - run:
+          name: Install OS-level dependencies
+          command: ./.circleci/install-prerequisites.sh "<< parameters.engine >>"
       - run:
           name: Generate database name
           command: |
@@ -279,6 +256,12 @@ jobs:
             echo "export SNOWFLAKE_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
             echo "export DATABRICKS_CATALOG='$TEST_DB_NAME'" >> "$BASH_ENV"
             echo "export REDSHIFT_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export GCP_POSTGRES_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+            echo "export FABRIC_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
+
+            # Make snowflake private key available            
+            echo $SNOWFLAKE_PRIVATE_KEY_RAW | base64 -d > /tmp/snowflake-keyfile.p8
+            echo "export SNOWFLAKE_PRIVATE_KEY_FILE='/tmp/snowflake-keyfile.p8'" >> "$BASH_ENV"
       - run:
           name: Create test database
           command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" up
@@ -306,6 +289,7 @@ workflows:
                 - "3.10"
                 - "3.11"
                 - "3.12"
+                - "3.13"
       - cicd_tests_windows
       - engine_tests_docker:
           name: engine_<< matrix.engine >>
@@ -335,13 +319,8 @@ workflows:
                 - bigquery
                 - clickhouse-cloud
                 - athena
-          filters:
-            branches:
-              only:
-                - main
-      - trigger_private_tests:
-          requires:
-            - style_and_cicd_tests
+                - fabric
+                - gcp-postgres
           filters:
             branches:
               only:

.circleci/install-prerequisites.sh

@@ -12,17 +12,31 @@ fi
 
 ENGINE="$1"
 
-COMMON_DEPENDENCIES="libpq-dev netcat-traditional"
+COMMON_DEPENDENCIES="libpq-dev netcat-traditional unixodbc-dev"
 ENGINE_DEPENDENCIES=""
 
 if [ "$ENGINE" == "spark" ]; then
     ENGINE_DEPENDENCIES="default-jdk"
+elif [ "$ENGINE" == "fabric" ]; then
+    echo "Installing Microsoft package repository"
+
+    # ref: https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server
+    curl -sSL -O https://packages.microsoft.com/config/ubuntu/$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2)/packages-microsoft-prod.deb
+    sudo dpkg -i packages-microsoft-prod.deb
+    rm packages-microsoft-prod.deb
+
+    ENGINE_DEPENDENCIES="msodbcsql18"    
 fi
 
 ALL_DEPENDENCIES="$COMMON_DEPENDENCIES $ENGINE_DEPENDENCIES"
 
 echo "Installing OS-level dependencies: $ALL_DEPENDENCIES"
 
-sudo apt-get clean && sudo apt-get -y update && sudo apt-get -y install $ALL_DEPENDENCIES
+sudo apt-get clean && sudo apt-get -y update && sudo ACCEPT_EULA='Y' apt-get -y install $ALL_DEPENDENCIES
+
+if [ "$ENGINE" == "spark" ]; then
+    echo "Using Java version for spark:"
+    java -version
+fi
 
 echo "All done"

.circleci/manage-test-db.sh

@@ -25,7 +25,7 @@ function_exists() {
 # Snowflake
 snowflake_init() {
     echo "Installing Snowflake CLI"
-    pip install "snowflake-cli-labs<3.8.0"
+    pip install "snowflake-cli"
 }
 
 snowflake_up() {
@@ -40,18 +40,6 @@ snowflake_down() {
 databricks_init() {
     echo "Installing Databricks CLI"
     curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh || true
-
-    echo "Writing out Databricks CLI config file"
-    echo -e "[DEFAULT]\nhost = $DATABRICKS_SERVER_HOSTNAME\ntoken = $DATABRICKS_ACCESS_TOKEN" > ~/.databrickscfg
-
-    # this takes a path like 'sql/protocolv1/o/2934659247569/0723-005339-foobar' and extracts '0723-005339-foobar' from it
-    CLUSTER_ID=${DATABRICKS_HTTP_PATH##*/}
-
-    echo "Extracted cluster id: $CLUSTER_ID from '$DATABRICKS_HTTP_PATH'"
-
-    # Note: the cluster doesnt need to be running to create / drop catalogs, but it does need to be running to run the integration tests
-    echo "Ensuring cluster is running"
-    databricks clusters start $CLUSTER_ID || true
 }
 
 databricks_up() {
@@ -80,7 +68,11 @@ redshift_down() {
     EXIT_CODE=1
     ATTEMPTS=0
     while [ $EXIT_CODE -ne 0 ] && [ $ATTEMPTS -lt 5 ]; do
-        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'"
+        # note: sometimes this pg_terminate_backend() call can randomly fail with: ERROR:  Insufficient privileges 
+        # if it does, let's proceed with the drop anyway rather than aborting and never attempting the drop
+        redshift_exec "select pg_terminate_backend(procpid) from pg_stat_activity where datname = '$1'" || true
+        
+        # perform drop
         redshift_exec "drop database $1;" && EXIT_CODE=$? || EXIT_CODE=$?
         if [ $EXIT_CODE -ne 0 ]; then
             echo "Unable to drop database; retrying..."
@@ -109,6 +101,51 @@ clickhouse-cloud_init() {
     echo "Clickhouse Cloud instance $CLICKHOUSE_CLOUD_HOST is up and running"
 }
 
+# GCP Postgres
+gcp-postgres_init() {
+    # Download and start Cloud SQL Proxy
+    curl -fsSL -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.18.0/cloud-sql-proxy.linux.amd64
+    chmod +x cloud-sql-proxy
+    echo "$GCP_POSTGRES_KEYFILE_JSON" > /tmp/keyfile.json
+    ./cloud-sql-proxy --credentials-file /tmp/keyfile.json $GCP_POSTGRES_INSTANCE_CONNECTION_STRING &
+
+    # Wait for proxy to start
+    sleep 5
+}
+
+gcp-postgres_exec() {
+    PGPASSWORD=$GCP_POSTGRES_PASSWORD psql -h 127.0.0.1 -U $GCP_POSTGRES_USER -c "$1" postgres
+}
+
+gcp-postgres_up() {
+    gcp-postgres_exec "create database $1"
+}
+
+gcp-postgres_down() {
+    gcp-postgres_exec "drop database $1"
+}
+
+# Fabric
+fabric_init() {    
+    python --version #note: as at 2025-08-20, ms-fabric-cli is pinned to Python >= 3.10, <3.13
+    pip install ms-fabric-cli
+    
+    # to prevent the '[EncryptionFailed] An error occurred with the encrypted cache.' error
+    # ref: https://microsoft.github.io/fabric-cli/#switch-to-interactive-mode-optional
+    fab config set encryption_fallback_enabled true 
+
+    echo "Logging in to Fabric"
+    fab auth login -u $FABRIC_CLIENT_ID -p $FABRIC_CLIENT_SECRET --tenant $FABRIC_TENANT_ID
+}
+
+fabric_up() {
+    fab create "SQLMesh CircleCI.Workspace/$1.Warehouse"
+}
+
+fabric_down() {
+    fab rm -f "SQLMesh CircleCI.Workspace/$1.Warehouse" || true
+}
+
 INIT_FUNC="${ENGINE}_init"
 UP_FUNC="${ENGINE}_up"
 DOWN_FUNC="${ENGINE}_down"

.circleci/test_migration.sh

@@ -1,11 +1,6 @@
 #!/usr/bin/env bash
 set -ex
 
-GATEWAY_NAME="duckdb_persistent"
-TMP_DIR=$(mktemp -d)
-SUSHI_DIR="$TMP_DIR/sushi"
-
-
 if [[ -z $(git tag --points-at HEAD) ]]; then
     # If the current commit is not tagged, we need to find the last tag
     LAST_TAG=$(git describe --tags --abbrev=0)
@@ -14,28 +9,48 @@ else
     LAST_TAG=$(git tag --sort=-creatordate | head -n 2 | tail -n 1)
 fi
 
+if [ "$1" == "" ]; then
+    echo "Usage: $0 <example name> <sqlmesh opts>"
+    echo "eg $0 sushi '--gateway duckdb_persistent'"
+    exit 1
+fi
+
+
+TMP_DIR=$(mktemp -d)
+EXAMPLE_NAME="$1"
+SQLMESH_OPTS="$2"
+EXAMPLE_DIR="./examples/$EXAMPLE_NAME"
+TEST_DIR="$TMP_DIR/$EXAMPLE_NAME"
+
+echo "Running migration test for '$EXAMPLE_NAME' in '$TEST_DIR' for example project '$EXAMPLE_DIR' using options '$SQLMESH_OPTS'"
+
 git checkout $LAST_TAG
 
 # Install dependencies from the previous release.
 make install-dev
 
-cp -r ./examples/sushi $TMP_DIR
+cp -r $EXAMPLE_DIR $TEST_DIR
+
+# this is only needed temporarily until the released tag for $LAST_TAG includes this config
+if [ "$EXAMPLE_NAME" == "sushi_dbt" ]; then
+    echo 'migration_test_config = sqlmesh_config(Path(__file__).parent, dbt_target_name="duckdb")' >> $TEST_DIR/config.py
+fi    
 
 # Run initial plan
-pushd $SUSHI_DIR
+pushd $TEST_DIR
 rm -rf ./data/*
-sqlmesh --gateway $GATEWAY_NAME plan --no-prompts --auto-apply
+sqlmesh $SQLMESH_OPTS plan --no-prompts --auto-apply
 rm -rf .cache
 popd
 
-# Switch back to the starting state of the repository
+# Switch back to the starting state of the repository    
 git checkout -
 
 # Install updated dependencies.
 make install-dev
 
 # Migrate and make sure the diff is empty
-pushd $SUSHI_DIR
-sqlmesh --gateway $GATEWAY_NAME migrate
-sqlmesh --gateway $GATEWAY_NAME diff prod
-popd
+pushd $TEST_DIR
+sqlmesh $SQLMESH_OPTS migrate
+sqlmesh $SQLMESH_OPTS diff prod
+popd