[WIP] Default thin-client to enabled and add proxy connectivity-probe gate

sdk/cosmos/azure-cosmos-tests/THINCLIENT_TEST_MATRIX.md

@@ -0,0 +1,212 @@
+# Thin Client E2E Test Matrix — Gateway V2 QueryPlan Support
+
+---
+
+## 1. Query Tests (`ThinClientQueryE2ETest`) — 80 tests
+
+### Filtering (WHERE clause)
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testSelectAll` | `SELECT * FROM c` | Full scan |
+| `testWhereEquality` | `SELECT * FROM c WHERE c.category = 'electronics'` | Equality filter |
+| `testWhereEqualityParameterized` | `SELECT * FROM c WHERE c.category = @cat` | Parameterized query |
+| `testWhereRangeGreaterThan` | `SELECT * FROM c WHERE c.age > 30` | Range (>) |
+| `testWhereRangeLessThanOrEqual` | `SELECT * FROM c WHERE c.price <= 25.00` | Range (<=) |
+| `testWhereRangeBetween` | `SELECT * FROM c WHERE c.age >= 18 AND c.age <= 40` | Range (between) |
+| `testWhereIn` | `SELECT * FROM c WHERE c.category IN ('electronics', 'toys')` | IN operator |
+| `testWhereCompoundAndOr` | `SELECT * FROM c WHERE c.status = 'active' AND (...)` | Compound AND/OR |
+| `testWhereNotEqual` | `SELECT * FROM c WHERE c.status != 'inactive'` | Not equal |
+| `testWhereBooleanField` | `SELECT * FROM c WHERE c.isActive = true` | Boolean filter |
+| `testWhereIsDefined` | `SELECT * FROM c WHERE IS_DEFINED(c.address)` | IS_DEFINED |
+| `testWhereStartsWith` | `SELECT * FROM c WHERE STARTSWITH(c.category, 'elec')` | STARTSWITH |
+| `testWhereContains` | `SELECT * FROM c WHERE CONTAINS(c.category, 'ook')` | CONTAINS |
+| `testWhereArrayContains` | `SELECT * FROM c WHERE ARRAY_CONTAINS(c.scores, 50)` | ARRAY_CONTAINS |
+| `testWhereNestedProperty` | `SELECT * FROM c WHERE c.address.city = 'Seattle'` | Nested property |
+| `testBetween` | `SELECT * FROM c WHERE c.age BETWEEN 18 AND 40` | BETWEEN keyword |
+
+### Projection
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testSelectSpecificFields` | `SELECT c.id, c.category, c.price FROM c` | Field projection |
+| `testSelectComputedAlias` | `SELECT c.id, c.price * 1.1 AS taxedPrice FROM c` | Computed alias |
+| `testSelectValueObject` | `SELECT VALUE { name: c.category, loc: c.address.city } FROM c` | VALUE with JSON object |
+| `testSelectValueScalar` | `SELECT VALUE c.category FROM c` | VALUE scalar |
+
+### ORDER BY
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testOrderByAsc` | `SELECT * FROM c ORDER BY c.age` | ORDER BY ASC |
+| `testOrderByDesc` | `SELECT * FROM c ORDER BY c.price DESC` | ORDER BY DESC |
+
+### DISTINCT
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testDistinctValue` | `SELECT DISTINCT VALUE c.category FROM c` | DISTINCT VALUE (string) |
+| `testDistinctValueBoolean` | `SELECT DISTINCT VALUE c.isActive FROM c` | DISTINCT VALUE (boolean) |
+
+### TOP
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testTop` | `SELECT TOP 3 * FROM c` | TOP |
+| `testTopWithOrderBy` | `SELECT TOP 5 * FROM c ORDER BY c.price DESC` | TOP + ORDER BY |
+
+### Aggregates
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testCount` | `SELECT VALUE COUNT(1) FROM c` | COUNT |
+| `testSum` | `SELECT VALUE SUM(c.price) FROM c` | SUM |
+| `testAvg` | `SELECT VALUE AVG(c.age) FROM c` | AVG |
+| `testMin` | `SELECT VALUE MIN(c.price) FROM c` | MIN |
+| `testMax` | `SELECT VALUE MAX(c.age) FROM c` | MAX |
+
+### GROUP BY
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testGroupByCount` | `SELECT c.category, COUNT(1) as cnt FROM c GROUP BY c.category` | GROUP BY + COUNT |
+| `testGroupBySumAvg` | `SELECT c.category, SUM(c.price) as total, AVG(c.price) as avg FROM c GROUP BY c.category` | GROUP BY + SUM + AVG |
+
+### OFFSET / LIMIT
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testOffsetLimit` | `SELECT * FROM c ORDER BY c.idx OFFSET 3 LIMIT 4` | OFFSET + LIMIT |
+
+### JOIN (self-join on arrays)
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testJoinScoresArray` | `SELECT c.id, s AS score FROM c JOIN s IN c.scores` | JOIN (int array) |
+| `testJoinWithFilter` | `SELECT c.id, s AS score FROM c JOIN s IN c.scores WHERE s >= 50` | JOIN + WHERE |
+| `testJoinTagsArray` | `SELECT c.id, t AS tag FROM c JOIN t IN c.tags` | JOIN (string array) |
+
+### EXISTS subquery
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testExistsSubquery` | `SELECT * FROM c WHERE EXISTS (SELECT VALUE s FROM s IN c.scores WHERE s > 60)` | EXISTS |
+| `testExistsSubqueryWithStringMatch` | `SELECT * FROM c WHERE EXISTS (SELECT VALUE t FROM t IN c.tags WHERE t = 'on-sale')` | EXISTS + string match |
+| `testExistsAliasInProjection` | `SELECT c.id, EXISTS (...) AS hasHighScore FROM c` | EXISTS in projection |
+
+### LIKE
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testLikePrefix` | `SELECT * FROM c WHERE c.category LIKE 'elec%'` | LIKE prefix |
+| `testLikeSuffix` | `SELECT * FROM c WHERE c.category LIKE '%ing'` | LIKE suffix |
+| `testLikeContains` | `SELECT * FROM c WHERE c.category LIKE '%ook%'` | LIKE contains |
+
+### String Functions
+| Test | SQL | Function |
+|------|-----|----------|
+| `testStringConcat` | `SELECT CONCAT(c.category, '-', c.status) AS label FROM c` | CONCAT |
+| `testStringEndsWith` | `SELECT * FROM c WHERE ENDSWITH(c.category, 'ics')` | ENDSWITH |
+| `testStringLower` | `SELECT LOWER(c.category) AS lowerCat FROM c` | LOWER |
+| `testStringUpper` | `SELECT UPPER(c.status) AS upperStatus FROM c` | UPPER |
+| `testStringLength` | `SELECT c.category, LENGTH(c.category) AS len FROM c` | LENGTH |
+| `testStringSubstring` | `SELECT SUBSTRING(c.category, 0, 4) AS prefix FROM c` | SUBSTRING |
+| `testStringReplace` | `SELECT REPLACE(c.category, 'o', '0') AS replaced FROM c` | REPLACE |
+| `testStringIndexOf` | `SELECT INDEX_OF(c.category, 'o') AS pos FROM c` | INDEX_OF |
+| `testStringLeft` | `SELECT LEFT(c.category, 3) AS l FROM c` | LEFT |
+| `testStringReverse` | `SELECT REVERSE(c.category) AS rev FROM c` | REVERSE |
+| `testStringTrim` | `SELECT TRIM(c.status) AS trimmed FROM c` | TRIM |
+| `testRegexMatch` | `SELECT * FROM c WHERE RegexMatch(c.category, '^elec.*')` | RegexMatch |
+
+### Type Checking Functions
+| Test | SQL | Function |
+|------|-----|----------|
+| `testIsArray` | `SELECT c.id, IS_ARRAY(c.scores) AS isArr FROM c` | IS_ARRAY |
+| `testIsBool` | `SELECT c.id, IS_BOOL(c.isActive) AS isBool FROM c` | IS_BOOL |
+| `testIsNull` | `SELECT * FROM c WHERE IS_NULL(c.nonExistentField)` | IS_NULL |
+| `testIsNumber` | `SELECT c.id, IS_NUMBER(c.age) AS isNum FROM c` | IS_NUMBER |
+| `testIsString` | `SELECT c.id, IS_STRING(c.category) AS isStr FROM c` | IS_STRING |
+| `testIsObject` | `SELECT c.id, IS_OBJECT(c.address) AS isObj FROM c` | IS_OBJECT |
+
+### Math Functions
+| Test | SQL | Function |
+|------|-----|----------|
+| `testMathAbs` | `SELECT ABS(c.age - 30) AS diff FROM c` | ABS |
+| `testMathCeilingFloor` | `SELECT CEILING(c.price) AS ceil, FLOOR(c.price) AS flr FROM c` | CEILING, FLOOR |
+| `testMathRound` | `SELECT ROUND(c.price) AS rounded FROM c` | ROUND |
+| `testMathPower` | `SELECT POWER(c.age, 2) AS ageSq FROM c` | POWER |
+| `testMathSqrt` | `SELECT SQRT(c.price) AS sqrtPrice FROM c` | SQRT |
+
+### Array Functions
+| Test | SQL | Function |
+|------|-----|----------|
+| `testArrayLength` | `SELECT c.id, ARRAY_LENGTH(c.scores) AS len FROM c` | ARRAY_LENGTH |
+| `testArraySlice` | `SELECT c.id, ARRAY_SLICE(c.tags, 0, 1) AS firstTag FROM c` | ARRAY_SLICE |
+
+### Conditional Functions
+| Test | SQL | Function |
+|------|-----|----------|
+| `testIif` | `SELECT c.id, IIF(c.age >= 18, 'adult', 'minor') AS ageGroup FROM c` | IIF |
+
+### Date/Time Functions
+| Test | SQL | Function |
+|------|-----|----------|
+| `testGetCurrentDateTime` | `SELECT VALUE GetCurrentDateTime()` | GetCurrentDateTime |
+
+### Cross-Partition
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testCrossPartitionSelectAll` | `SELECT * FROM c ORDER BY c.idx` | Cross-partition (no PK filter) |
+| `testCrossPartitionWhereFilter` | `SELECT * FROM c WHERE c.category = 'electronics' ORDER BY c.idx` | Cross-partition + filter |
+
+### Multi-Range (creates dedicated container with multiple PKs)
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testMultiRangePartitionKeyInClause` | `SELECT * FROM c WHERE c.mypk IN (pk1, pk3, pk5)` | Multi-range IN |
+| `testMultiRangePartitionKeyOrClause` | `SELECT * FROM c WHERE c.mypk = 'pk-or-1' OR c.mypk = 'pk-or-3'` | Multi-range OR |
+| `testMultiRangeManyPartitionKeys` | `SELECT * FROM c WHERE c.mypk IN (pk1..pk10)` | Multi-range (10 PKs) |
+
+### Continuation Token
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testContinuationTokenDraining` | `SELECT * FROM c` (page size 2) | Pagination / continuation tokens |
+
+### Error Handling
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testInvalidQueryReturnsBadRequest` | `SELEC * FORM c` (invalid) | 400 BadRequest validation |
+
+### Vector Search (requires `EnableNoSQLVectorSearch` capability)
+| Test | SQL | Query Feature |
+|------|-----|---------------|
+| `testVectorSearchGatewayVsThinClient` | `SELECT TOP 5 c.id, VectorDistance(c.embedding, [...]) AS score FROM c ORDER BY VectorDistance(...)` | VectorDistance + FLAT index |
+| `testFullTextSearchGatewayVsThinClient` | `SELECT TOP 10 * FROM c WHERE FullTextContains(c.text, 'mountain')` | FullTextContains |
+| `testHybridSearchGatewayVsThinClient` | `SELECT TOP 3 * FROM c ORDER BY RANK RRF(VectorDistance(...), FullTextScore(...))` | Hybrid RRF (vector + full-text) |
+
+---
+
+## 2. Point Operations (`ThinClientPointOperationE2ETest`) — 3 tests
+
+| Test | Operation | Coverage |
+|------|-----------|----------|
+| `testThinClientDocumentPointOperations` | Create, Read, Replace, Upsert, Patch, Delete | Full CRUD + Patch lifecycle |
+| `testThinClientBulk` | Bulk create + bulk read | Bulk operations |
+| `testThinClientBatch` | Transactional batch (create + read) | CosmosBatch |
+
+---
+
+## 3. Change Feed (`ThinClientChangeFeedE2ETest`) — 3 tests
+
+| Test | FeedRange | Coverage |
+|------|-----------|----------|
+| `testThinClientIncrementalChangeFeed` | `FeedRange.forLogicalPartition(pk)` | Incremental change feed (via batch insert) |
+| `testThinClientChangeFeedFullRange` | `FeedRange.forFullRange()` | Cross-partition change feed |
+| `testThinClientChangeFeedPartitionKey` | `FeedRange.forLogicalPartition(pk)` | Single-PK feed with exact count + PK validation |
+
+---
+
+## 4. Stored Procedures (`ThinClientStoredProcedureE2ETest`) — 3 tests
+
+| Test | Operation | Coverage |
+|------|-----------|----------|
+| `testThinClientStoredProcedure` | Create + execute sproc | Sproc creates a document, verifies execution |
+| `testStoredProcedureExecutionWithoutPartitionKeyThrows` | Execute without PK | Validates 400 error |
+| `testThinClientStoredProcedureWithPartitionKeyNone` | Execute with `PartitionKey.NONE` | Non-partitioned sproc execution |
+
+---
+
+## Test Infrastructure
+
+- **Test data**: 10 diverse documents seeded per partition (categories, prices, ages, nested objects, arrays, tags, booleans)
+- **Shared container**: `/mypk` partition key, reused across query tests
+- **Comparison method**: Direct TCP vs Thin Client (HTTP/2 → Proxy), assert identical results
+- **Endpoint validation**: Every test asserts thin client used `:10250` endpoint, gateway used `:443`

sdk/cosmos/azure-cosmos-tests/pom.xml

@@ -848,6 +848,9 @@ Licensed under the MIT License.
               </suiteXmlFiles>
               <systemPropertyVariables>
                 <COSMOS.CLIENT_LEAK_DETECTION_ENABLED>true</COSMOS.CLIENT_LEAK_DETECTION_ENABLED>
+                <COSMOS.USE_AAD_AUTH>${cosmos.use.aad.auth}</COSMOS.USE_AAD_AUTH>
+                <COSMOS.THINCLIENT_ENABLED>true</COSMOS.THINCLIENT_ENABLED>
+                <COSMOS.HTTP2_ENABLED>true</COSMOS.HTTP2_ENABLED>
                 <io.netty.leakDetection.samplingInterval>1</io.netty.leakDetection.samplingInterval>
                 <io.netty.leakDetection.targetRecords>256</io.netty.leakDetection.targetRecords>
                 <io.netty.leakDetection.level>paranoid</io.netty.leakDetection.level>

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java

@@ -1981,14 +1981,21 @@ private void validateChannelAcquisitionContext(CosmosDiagnostics diagnostics, bo
     private String generateHttp2OptedInUserAgentIfRequired(String userAgent) {
         // Mirrors RxDocumentClientImpl.addUserAgentSuffix + UserAgentContainer.setFeatureEnabledFlagsAsSuffix:
         // when HTTP/2 is enabled, the Http2 bit is set; when PING keepalive is also effectively enabled
-        // (kill-switch on AND positive interval), the Http2PingHealth bit is OR'd in.
+        // (kill-switch on AND positive interval), the Http2PingHealth bit is OR'd in. ThinClient is set when
+        // COSMOS.THINCLIENT_ENABLED is true (default true after Gateway V2 default enablement).
         // Tests here do not override Http2ConnectionConfig.setEnabled(...) so the per-client override branch
         // in addUserAgentSuffix is a no-op for this helper.
+        int featureValue = 0;
+        if (Configs.isThinClientEnabled()) {
+            featureValue |= UserAgentFeatureFlags.ThinClient.getValue();
+        }
         if (Configs.isHttp2Enabled()) {
-            int featureValue = UserAgentFeatureFlags.Http2.getValue();
+            featureValue |= UserAgentFeatureFlags.Http2.getValue();
             if (Configs.isHttp2PingHealthEnabled() && Configs.getHttp2PingIntervalInSeconds() > 0) {
                 featureValue |= UserAgentFeatureFlags.Http2PingHealth.getValue();
             }
+        }
+        if (featureValue != 0) {
             userAgent = userAgent + "|F" + Integer.toHexString(featureValue).toUpperCase(Locale.ROOT);
         }
 

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/CosmosNotFoundTests.java

@@ -52,6 +52,16 @@ public CosmosNotFoundTests(CosmosClientBuilder clientBuilder) {
 
     @BeforeClass(groups = {"fast", "thinclient"}, timeOut = SETUP_TIMEOUT)
     public void before_CosmosNotFoundTests() {
+        // Thin-client routing tests in this class (the "thinclient" group) assert that
+        // requests actually went through the proxy on port 10250 via assertThinClientEndpointUsed
+        // and rely on proxy-specific substatus codes (e.g. OWNER_RESOURCE_NOT_EXISTS = 1003).
+        // The connectivity probe is enabled by default in production, but the proxy-side
+        // /connectivity-probe endpoint is not deployed in every CI test account yet. With the
+        // default failure threshold of 1, a single failed probe cycle flips routing from the
+        // proxy to Gateway V1, breaking these assertions. Disable the probe here so the routing
+        // path under test is exercised deterministically; production callers still get the
+        // probe ON by default. Cleared in @AfterClass to avoid leaking into other test classes.
+        System.setProperty("COSMOS.THINCLIENT_PROBE_ENABLED", "false");
         executeWithRetry(() -> {
             safeClose(this.commonAsyncClient);
             this.commonAsyncClient = getClientBuilder().buildAsyncClient();
@@ -88,6 +98,7 @@ public static Object[][] operationTypeProvider() {
     @AfterClass(groups = {"fast", "thinclient"}, timeOut = SHUTDOWN_TIMEOUT, alwaysRun = true)
     public void afterClass() {
         safeClose(this.commonAsyncClient);
+        System.clearProperty("COSMOS.THINCLIENT_PROBE_ENABLED");
     }
 
     @Test(groups = {"fast"}, dataProvider = "operationTypeProvider", timeOut = TIMEOUT)

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/PerPartitionCircuitBreakerE2ETests.java

@@ -240,6 +240,13 @@ public PerPartitionCircuitBreakerE2ETests(CosmosClientBuilder cosmosClientBuilde
 
     @BeforeClass(groups = {"circuit-breaker-misc-gateway", "circuit-breaker-misc-direct", "circuit-breaker-read-all-read-many", "multi-region", "fi-thinclient-multi-master"})
     public void beforeClass() {
+        // The "fi-thinclient-multi-master" tests assert thinclient routing via
+        // assertThinClientEndpointUsed. The connectivity probe is ON by default in production, but the
+        // proxy-side /connectivity-probe endpoint is not deployed in every CI account yet, so with the
+        // default failure threshold of 1 it would flip routing to Gateway V1 and break those assertions.
+        // Disable here so the routing path under test is exercised deterministically; other groups in
+        // this class (circuit-breaker-* / multi-region) are unaffected. Cleared in @AfterClass.
+        System.setProperty("COSMOS.THINCLIENT_PROBE_ENABLED", "false");
         try (CosmosAsyncClient testClient = getClientBuilder().buildAsyncClient()) {
             RxDocumentClientImpl documentClient = (RxDocumentClientImpl) ReflectionUtils.getAsyncDocumentClient(testClient);
             GlobalEndpointManager globalEndpointManager = documentClient.getGlobalEndpointManager();
@@ -4597,6 +4604,7 @@ public void afterClass() {
                 safeClose(dummyClient);
             }
         }
+        System.clearProperty("COSMOS.THINCLIENT_PROBE_ENABLED");
     }
 
     private static class ResponseWrapper<T> {

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/UserAgentSuffixTest.java

@@ -120,12 +120,19 @@ private void validateUserAgentSuffix(String actualUserAgent, String expectedUser
 
         // Mirrors RxDocumentClientImpl.addUserAgentSuffix + UserAgentContainer.setFeatureEnabledFlagsAsSuffix:
         // when HTTP/2 is enabled, the Http2 bit is set; when PING keepalive is also effectively enabled
-        // (kill-switch on AND positive interval), the Http2PingHealth bit is OR'd in.
+        // (kill-switch on AND positive interval), the Http2PingHealth bit is OR'd in. ThinClient is set when
+        // COSMOS.THINCLIENT_ENABLED is true (default true after Gateway V2 default enablement).
+        int featureValue = 0;
+        if (Configs.isThinClientEnabled()) {
+            featureValue |= UserAgentFeatureFlags.ThinClient.getValue();
+        }
         if (Configs.isHttp2Enabled()) {
-            int featureValue = UserAgentFeatureFlags.Http2.getValue();
+            featureValue |= UserAgentFeatureFlags.Http2.getValue();
             if (Configs.isHttp2PingHealthEnabled() && Configs.getHttp2PingIntervalInSeconds() > 0) {
                 featureValue |= UserAgentFeatureFlags.Http2PingHealth.getValue();
             }
+        }
+        if (featureValue != 0) {
             expectedUserAgentSuffix = expectedUserAgentSuffix + "|F" + Integer.toHexString(featureValue).toUpperCase(Locale.ROOT);
         }
 

sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/faultinjection/Http2PingKeepaliveTest.java

@@ -86,6 +86,15 @@ public void beforeClass() {
         // set externally (Maven profile or -D) so a single test class covers both transports
         // across two pipeline runs.
         System.setProperty("COSMOS.HTTP2_ENABLED", "true");
+        // Disable the thin-client connectivity probe for the duration of this test.
+        // The probe fires HTTP/2 POSTs to thin-client port 10250 on every account refresh;
+        // when this test installs an iptables DROP on port 10250 (THIN_CLIENT_ENABLED=true
+        // branch) those probe requests time out, the probe trips
+        // isProxyProbeHealthy()->false, and routing falls back to Gateway V1 on port 443
+        // -- bypassing the very PING handler this test is exercising. Disable the probe
+        // so EndpointProbeClient.proxyHealthy stays optimistically true and the data plane
+        // request actually flows over port 10250 where the iptables DROP can take effect.
+        System.setProperty("COSMOS.THINCLIENT_PROBE_ENABLED", "false");
         logger.info("Transport selected: thinClient={}, h2Port={}", THIN_CLIENT_ENABLED, H2_PORT);
 
         this.client = getClientBuilder().buildAsyncClient();
@@ -101,6 +110,7 @@ public void beforeClass() {
     public void afterClass() {
         safeClose(this.client);
         System.clearProperty("COSMOS.HTTP2_ENABLED");
+        System.clearProperty("COSMOS.THINCLIENT_PROBE_ENABLED");
     }
 
     @BeforeMethod(groups = {"manual-http-network-fault"})