Add Azure SQL jobs and remove stale CodeQL workarounds

Author: gargsaumya

eng/pipelines/pr-validation-pipeline.yml

@@ -6,12 +6,6 @@ trigger:
     include:
       - main
 
-parameters:
-  - name: enableAzureSQL
-    displayName: 'Enable Azure SQL Database testing'
-    type: boolean
-    default: false
-
 jobs:
 - job: CodeQLAnalysis
   displayName: 'CodeQL Security Analysis'
@@ -55,18 +49,9 @@ jobs:
   displayName: 'Windows x64'
   pool:
     vmImage: 'windows-latest'
-  variables:
-    # Enable CodeQL for this job to update the old stale snapshot (build_jobname=pytestonwindows)
-    # This can be removed once the old CodeQL issue SM02986 is cleared
-    Codeql.Enabled: true
 
   strategy:
     matrix:
-      pytestonwindows:
-        # Temporary entry to clear stale CodeQL snapshot SM02986
-        # Remove this once the issue is resolved
-        sqlVersion: 'SQL2022'
-        pythonVersion: '3.13'
       SQLServer2022:
         sqlVersion: 'SQL2022'
         pythonVersion: '3.13'
@@ -76,10 +61,6 @@ jobs:
       LocalDB_Python314:
         sqlVersion: 'LocalDB'
         pythonVersion: '3.14'
-      ${{ if eq(parameters.enableAzureSQL, true) }}:
-        AzureSQL:
-          sqlVersion: 'AzureSQL'
-          pythonVersion: '3.13'
 
   steps:
   - task: UsePythonVersion@0
@@ -244,22 +225,11 @@ jobs:
     env:
       DB_PASSWORD: $(DB_PASSWORD)
 
-  # ============== CodeQL Init (temporary - remove after SM02986 is cleared) ==============
-  - task: CodeQL3000Init@0
-    inputs:
-      Enabled: true
-    displayName: 'Initialize CodeQL (temporary)'
-
   - script: |
       cd mssql_python\pybind
       build.bat x64
     displayName: 'Build .pyd file'
 
-  # ============== CodeQL Finalize (temporary - remove after SM02986 is cleared) ==============
-  - task: CodeQL3000Finalize@0
-    condition: always()
-    displayName: 'Finalize CodeQL (temporary)'
-
   # Run tests for LocalDB
   - script: |
       python -m pytest -v --junitxml=test-results-localdb.xml --cov=. --cov-report=xml:coverage-localdb.xml --capture=tee-sys --cache-clear
@@ -284,14 +254,6 @@ jobs:
     env:
       DB_CONNECTION_STRING: 'Server=localhost;Database=TestDB;Uid=testuser;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
 
-  # Run tests for Azure SQL Database
-  - script: |
-      python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear
-    displayName: 'Run tests with coverage on Azure SQL Database'
-    condition: eq(variables['sqlVersion'], 'AzureSQL')
-    env:
-      DB_CONNECTION_STRING: $(AZURE_CONNECTION_STRING)
-
   # Download and restore AdventureWorks2022 database for benchmarking
   - powershell: |
       Write-Host "Downloading AdventureWorks2022.bak..."
@@ -467,16 +429,9 @@ jobs:
       SQL2022:
         sqlServerImage: 'mcr.microsoft.com/mssql/server:2022-latest'
         sqlVersion: 'SQL2022'
-        useAzureSQL: 'false'
       SQL2025:
         sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest'
         sqlVersion: 'SQL2025'
-        useAzureSQL: 'false'
-      ${{ if eq(parameters.enableAzureSQL, true) }}:
-        AzureSQL:
-          sqlServerImage: ''
-          sqlVersion: 'AzureSQL'
-          useAzureSQL: 'true'
 
   steps:
   - task: UsePythonVersion@0
@@ -507,7 +462,6 @@ jobs:
       docker version
       docker ps
     displayName: 'Install and start Colima-based Docker'
-    condition: eq(variables['useAzureSQL'], 'false')
 
   - script: |
       # Pull and run SQL Server container
@@ -530,7 +484,6 @@ jobs:
         sleep 2
       done
     displayName: 'Pull & start SQL Server (Docker)'
-    condition: eq(variables['useAzureSQL'], 'false')
     env:
       DB_PASSWORD: $(DB_PASSWORD)
 
@@ -548,19 +501,10 @@ jobs:
       echo "Build successful, running tests now"
       python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear
     displayName: 'Run pytest with coverage'
-    condition: eq(variables['useAzureSQL'], 'false')
     env:
       DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
       DB_PASSWORD: $(DB_PASSWORD)
 
-  - script: |
-      echo "Build successful, running tests now against Azure SQL Database"
-      python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear
-    displayName: 'Run pytest with coverage on Azure SQL Database'
-    condition: eq(variables['useAzureSQL'], 'true')
-    env:
-      DB_CONNECTION_STRING: $(AZURE_CONNECTION_STRING)
-
   - task: PublishTestResults@2
     condition: succeededOrFailed()
     inputs:
@@ -584,17 +528,12 @@ jobs:
         distroName: 'Ubuntu-SQL2025'
         sqlServerImage: 'mcr.microsoft.com/mssql/server:2025-latest'
         useAzureSQL: 'false'
-      ${{ if eq(parameters.enableAzureSQL, true) }}:
+      ${{ if ne(variables['AZURE_CONNECTION_STRING'], '') }}:
         Ubuntu_AzureSQL:
           dockerImage: 'ubuntu:22.04'
           distroName: 'Ubuntu-AzureSQL'
           sqlServerImage: ''
           useAzureSQL: 'true'
-        Debian_AzureSQL:
-          dockerImage: 'debian:12'
-          distroName: 'Debian-AzureSQL'
-          sqlServerImage: ''
-          useAzureSQL: 'true'
       Debian:
         dockerImage: 'debian:12'
         distroName: 'Debian'
@@ -2016,6 +1955,192 @@ jobs:
       testResultsFiles: '**/test-results-alpine-arm64.xml'
       testRunTitle: 'Publish pytest results on Alpine ARM64'
 
+# ===========================================================================================
+# Azure SQL Database Testing Jobs
+# Gated on the AZURE_CONNECTION_STRING pipeline variable being set.
+# No local SQL Server install needed — tests run directly against the cloud database.
+# ===========================================================================================
+
+- job: PytestOnWindows_AzureSQL
+  displayName: 'Windows x64 AzureSQL'
+  condition: ne(variables['AZURE_CONNECTION_STRING'], '')
+  pool:
+    vmImage: 'windows-latest'
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.13'
+      addToPath: true
+      githubToken: $(GITHUB_TOKEN)
+    displayName: 'Use Python 3.13'
+
+  - script: |
+      python -m pip install --upgrade pip
+      pip install -r requirements.txt
+    displayName: 'Install dependencies'
+
+  - script: |
+      cd mssql_python\pybind
+      build.bat x64
+    displayName: 'Build .pyd file'
+
+  - script: |
+      python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear
+    displayName: 'Run tests on Azure SQL Database'
+    env:
+      DB_CONNECTION_STRING: $(AZURE_CONNECTION_STRING)
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '**/test-results-azuresql.xml'
+      testRunTitle: 'Publish test results for Windows AzureSQL'
+
+- job: PytestOnMacOS_AzureSQL
+  displayName: 'macOS x86_64 AzureSQL'
+  condition: ne(variables['AZURE_CONNECTION_STRING'], '')
+  pool:
+    vmImage: 'macos-latest'
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '3.13'
+      addToPath: true
+    displayName: 'Use Python 3.13'
+
+  - script: |
+      brew update
+      brew uninstall cmake --ignore-dependencies || echo "CMake not installed or already removed"
+      brew install cmake
+    displayName: 'Install CMake'
+
+  - script: |
+      python -m pip install --upgrade pip
+      pip install -r requirements.txt
+    displayName: 'Install Python dependencies'
+
+  - script: |
+      cd mssql_python/pybind
+      ./build.sh
+    displayName: 'Build pybind bindings (.so)'
+
+  - script: |
+      python -m pytest -v --junitxml=test-results-azuresql.xml --cov=. --cov-report=xml:coverage-azuresql.xml --capture=tee-sys --cache-clear
+    displayName: 'Run tests on Azure SQL Database'
+    env:
+      DB_CONNECTION_STRING: $(AZURE_CONNECTION_STRING)
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '**/test-results-azuresql.xml'
+      testRunTitle: 'Publish pytest results on macOS AzureSQL'
+
+- job: PytestOnLinux_AzureSQL
+  displayName: 'Linux x86_64 AzureSQL'
+  condition: ne(variables['AZURE_CONNECTION_STRING'], '')
+  pool:
+    vmImage: 'ubuntu-latest'
+
+  steps:
+  - script: |
+      docker run -d --name test-container-ubuntu-azuresql \
+        -v $(Build.SourcesDirectory):/workspace \
+        -w /workspace \
+        --network bridge \
+        ubuntu:22.04 \
+        tail -f /dev/null
+    displayName: 'Create Ubuntu container'
+
+  - script: |
+      docker exec test-container-ubuntu-azuresql bash -c "
+        export DEBIAN_FRONTEND=noninteractive
+        export TZ=UTC
+        ln -snf /usr/share/zoneinfo/\$TZ /etc/localtime && echo \$TZ > /etc/timezone
+        apt-get update && 
+        apt-get install -y python3 python3-pip python3-venv python3-full cmake curl wget gnupg software-properties-common build-essential python3-dev pybind11-dev
+      "
+    displayName: 'Install basic dependencies in Ubuntu container'
+
+  - script: |
+      docker exec test-container-ubuntu-azuresql bash -c "
+        export DEBIAN_FRONTEND=noninteractive
+        curl -sSL -O https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb
+        dpkg -i packages-microsoft-prod.deb || true
+        rm packages-microsoft-prod.deb
+        apt-get update
+        ACCEPT_EULA=Y apt-get install -y msodbcsql18
+        ACCEPT_EULA=Y apt-get install -y mssql-tools18
+        apt-get install -y unixodbc-dev
+      "
+    displayName: 'Install ODBC Driver in Ubuntu container'
+
+  - script: |
+      docker exec test-container-ubuntu-azuresql bash -c "
+        python3 -m venv /opt/venv
+        source /opt/venv/bin/activate
+        python -m pip install --upgrade pip
+        python -m pip install -r requirements.txt
+        echo 'source /opt/venv/bin/activate' >> ~/.bashrc
+      "
+    displayName: 'Install Python dependencies in Ubuntu container'
+
+  - script: |
+      docker exec test-container-ubuntu-azuresql bash -c "
+        source /opt/venv/bin/activate
+        cd mssql_python/pybind
+        chmod +x build.sh
+        ./build.sh
+      "
+    displayName: 'Build pybind bindings (.so) in Ubuntu container'
+
+  - script: |
+      docker exec test-container-ubuntu-azuresql bash -c "
+        export DEBIAN_FRONTEND=noninteractive
+        apt-get remove --purge -y msodbcsql18 mssql-tools18 unixodbc-dev
+        rm -f /usr/bin/sqlcmd
+        rm -f /usr/bin/bcp
+        rm -rf /opt/microsoft/msodbcsql
+        rm -f /lib/x86_64-linux-gnu/libodbcinst.so.2
+        odbcinst -u -d -n 'ODBC Driver 18 for SQL Server' || true
+        echo 'Uninstalled ODBC Driver and cleaned up libraries'
+        echo 'Verifying x86_64 debian_ubuntu driver library signatures:'
+        ldd mssql_python/libs/linux/debian_ubuntu/x86_64/lib/libmsodbcsql-18.5.so.1.1
+      "
+    displayName: 'Uninstall ODBC Driver before running tests in Ubuntu container'
+
+  - script: |
+      docker exec \
+        -e DB_CONNECTION_STRING="$(AZURE_CONNECTION_STRING)" \
+        test-container-ubuntu-azuresql bash -c "
+        source /opt/venv/bin/activate
+        echo 'Running tests on Ubuntu against Azure SQL Database'
+        python -m pytest -v --junitxml=test-results-ubuntu-azuresql.xml --cov=. --cov-report=xml:coverage-ubuntu-azuresql.xml --capture=tee-sys --cache-clear
+      "
+    displayName: 'Run pytest on Azure SQL in Ubuntu container'
+    env:
+      AZURE_CONNECTION_STRING: $(AZURE_CONNECTION_STRING)
+
+  - script: |
+      docker cp test-container-ubuntu-azuresql:/workspace/test-results-ubuntu-azuresql.xml $(Build.SourcesDirectory)/
+      docker cp test-container-ubuntu-azuresql:/workspace/coverage-ubuntu-azuresql.xml $(Build.SourcesDirectory)/
+    displayName: 'Copy test results from Ubuntu container'
+    condition: always()
+
+  - script: |
+      docker stop test-container-ubuntu-azuresql || true
+      docker rm test-container-ubuntu-azuresql || true
+    displayName: 'Clean up Ubuntu container'
+    condition: always()
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '**/test-results-ubuntu-azuresql.xml'
+      testRunTitle: 'Publish pytest results on Ubuntu AzureSQL'
+
 - job: CodeCoverageReport
   displayName: 'Full Code Coverage Report in Ubuntu x86_64'
   pool: