diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index b1c0677f..dccefb86 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1036,7 +1036,6 @@ final class LineSenderBuilder { private int maxBackoffMillis = PARAMETER_NOT_SET_EXPLICITLY; private int maxDatagramSize = PARAMETER_NOT_SET_EXPLICITLY; private int maxNameLength = PARAMETER_NOT_SET_EXPLICITLY; - private int maxSchemasPerConnection = PARAMETER_NOT_SET_EXPLICITLY; private int maximumBufferCapacity = PARAMETER_NOT_SET_EXPLICITLY; private final HttpClientConfiguration httpClientConfiguration = new DefaultHttpClientConfiguration() { @Override @@ -1369,8 +1368,6 @@ public Sender build() { long actualAutoFlushIntervalNanos = autoFlushIntervalMillis == PARAMETER_NOT_SET_EXPLICITLY ? DEFAULT_WS_AUTO_FLUSH_INTERVAL_NANOS : TimeUnit.MILLISECONDS.toNanos(autoFlushIntervalMillis); - int actualMaxSchemasPerConnection = maxSchemasPerConnection == PARAMETER_NOT_SET_EXPLICITLY - ? QwpWebSocketSender.DEFAULT_MAX_SCHEMAS_PER_CONNECTION : maxSchemasPerConnection; String wsAuthHeader = buildWebSocketAuthHeader(); @@ -1499,7 +1496,6 @@ public Sender build() { actualAutoFlushBytes, actualAutoFlushIntervalNanos, wsAuthHeader, - actualMaxSchemasPerConnection, requestDurableAck, cursorEngine, actualCloseFlushTimeoutMillis, @@ -2217,25 +2213,6 @@ public LineSenderBuilder maxNameLength(int maxNameLength) { return this; } - /** - * Sets the maximum number of distinct schemas the WebSocket sender may assign on one connection. - */ - public LineSenderBuilder maxSchemasPerConnection(int maxSchemasPerConnection) { - if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { - throw new LineSenderException("max schemas per connection is only supported for WebSocket transport"); - } - if (this.maxSchemasPerConnection != PARAMETER_NOT_SET_EXPLICITLY) { - throw new LineSenderException("max schemas per connection was already configured") - .put("[maxSchemasPerConnection=").put(this.maxSchemasPerConnection).put("]"); - } - if (maxSchemasPerConnection < 1) { - throw new LineSenderException("max schemas per connection must be positive") - .put("[maxSchemasPerConnection=").put(maxSchemasPerConnection).put("]"); - } - this.maxSchemasPerConnection = maxSchemasPerConnection; - return this; - } - /** * Minimum expected throughput in bytes per second for HTTP requests. *
@@ -3154,13 +3131,6 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else { throw new LineSenderException("invalid transaction [value=").put(sink).put(", allowed-values=[on, off]]"); } - } else if (Chars.equals("max_schemas_per_connection", sink)) { - if (protocol != PROTOCOL_WEBSOCKET) { - throw new LineSenderException("max_schemas_per_connection is only supported for WebSocket transport"); - } - pos = getValue(configurationString, pos, sink, "max_schemas_per_connection"); - int maxSchemas = parseIntValue(sink, "max_schemas_per_connection"); - maxSchemasPerConnection(maxSchemas); } else if (Chars.equals("sf_dir", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("sf_dir is only supported for WebSocket transport"); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ColumnView.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ColumnView.java index db7a51ba..1282d9f3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/ColumnView.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/ColumnView.java @@ -59,8 +59,8 @@ * {@code arrayRowAddr} accessors expose raw native pointers into the WebSocket * payload buffer for SIMD or JNI consumers. The layout of the bytes behind * these addresses follows the QWP wire format (see - * {@code docs/qwp/wire-egress.md}); these methods are an expert API and - * may shift as the wire format evolves. + * {@code https://questdb.com/docs/connect/wire-protocols/qwp-egress-websocket/}); + * these methods are an expert API and may shift as the wire format evolves. *

* Type contract / NULL handling. Each typed accessor delegates * to the same read path as the corresponding {@code QwpColumnBatch} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java index 20f0e82a..03f58141 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatch.java @@ -618,7 +618,8 @@ public long getUuidLo(int col, int row) { /** * True if the cell is NULL on the wire. *

- * Note on type-specific sentinels (see {@code docs/qwp/wire-egress.md} sec 11.5): + * Note on type-specific sentinels (see + * {@code https://questdb.com/docs/connect/wire-protocols/qwp-egress-websocket/}): * QuestDB stores NULL as a sentinel value for several types -- {@code Long.MIN_VALUE} * for LONG/INT/etc., {@code 0.0.0.0} for IPv4, {@code -1} for GEOHASH, and crucially * {@code NaN} for FLOAT and DOUBLE. Egress preserves these conventions: a row carrying diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java index a4ac77ee..6237e1de 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnBatchHandler.java @@ -106,8 +106,7 @@ default void onError(long requestId, byte status, String message) { * the query (with {@code failover=on}, the default). *

* {@code newNode} is the {@link QwpServerInfo} of the endpoint the client - * just bound to, or {@code null} if the new server negotiated the v1 - * protocol (no SERVER_INFO frame). + * just bound to. *

* After this callback fires, {@link #onBatch} will be invoked again with * {@code batch_seq} restarting at 0 on the new connection. Handlers that @@ -127,7 +126,7 @@ default void onFailoverReset(QwpServerInfo newNode) { * {@link #onFailoverReset(QwpServerInfo)} for backwards compatibility. * * @param requestId client-assigned request id of the query being replayed - * @param newNode the endpoint just bound to, or {@code null} if the new server negotiated v1 + * @param newNode the {@link QwpServerInfo} of the endpoint just bound to */ default void onFailoverReset(long requestId, QwpServerInfo newNode) { onFailoverReset(newNode); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index 37833b99..f8cc669f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -279,26 +279,16 @@ private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int } } - private void writeTableHeaderWithSchema(String tableName, int rowCount, int schemaId, QwpColumnDef[] columns) { + private void writeTableHeader(String tableName, int rowCount, QwpColumnDef[] columns) { buffer.putString(tableName); buffer.putVarint(rowCount); buffer.putVarint(columns.length); - buffer.putByte(SCHEMA_MODE_FULL); - buffer.putVarint(schemaId); for (QwpColumnDef col : columns) { buffer.putString(col.getName()); buffer.putByte(col.getWireTypeCode()); } } - private void writeTableHeaderWithSchemaRef(String tableName, int rowCount, int schemaId, int columnCount) { - buffer.putString(tableName); - buffer.putVarint(rowCount); - buffer.putVarint(columnCount); - buffer.putByte(SCHEMA_MODE_REFERENCE); - buffer.putVarint(schemaId); - } - private void writeTimestampColumn(long addr, int count, boolean useGorilla) { if (useGorilla && count > 2) { // Single pass: check feasibility and compute encoded size together @@ -325,12 +315,8 @@ private void writeTimestampColumn(long addr, int count, boolean useGorilla) { } } - void encodeTable(QwpTableBuffer tableBuffer, boolean useSchemaRef, boolean useGlobalSymbols, boolean useGorilla) { - int schemaId = tableBuffer.getSchemaId(); - if (schemaId < 0) { - schemaId = 0; - } - encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useSchemaRef, useGlobalSymbols, useGorilla, schemaId); + void encodeTable(QwpTableBuffer tableBuffer, boolean useGlobalSymbols, boolean useGorilla) { + encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useGlobalSymbols, useGorilla); } void encodeTable( @@ -339,23 +325,12 @@ void encodeTable( int[] limitedValueCounts, long[] limitedStringDataSizes, int[] limitedSymbolDictionarySizes, - boolean useSchemaRef, boolean useGlobalSymbols, - boolean useGorilla, - int schemaId + boolean useGorilla ) { QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); - if (useSchemaRef) { - writeTableHeaderWithSchemaRef( - tableBuffer.getTableName(), - rowCount, - schemaId, - columnDefs.length - ); - } else { - writeTableHeaderWithSchema(tableBuffer.getTableName(), rowCount, schemaId, columnDefs); - } + writeTableHeader(tableBuffer.getTableName(), rowCount, columnDefs); for (int i = 0; i < tableBuffer.getColumnCount(); i++) { QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java index 9752ace8..228360c7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressIoThread.java @@ -214,11 +214,10 @@ public void onBinaryMessage(long payloadPtr, int payloadLen) { decodeAndEmitError(payloadPtr, payloadLen); currentQueryDone = true; } else if (msgKind == QwpEgressMsgKind.CACHE_RESET) { - // Server reached a configured soft cap on the connection-scoped - // SYMBOL dict or schema-fingerprint cache. Drop the indicated - // caches on this side so the next RESULT_BATCH's deltaStart and - // schema-reference ids line up with the server's fresh counter. - // No user-visible event -- CACHE_RESET never arrives between the + // Server reached its configured soft cap on the connection-scoped + // SYMBOL dict. Drop the dict on this side so the next RESULT_BATCH's + // deltaStart lines up with the server's fresh counter. No + // user-visible event -- CACHE_RESET never arrives between the // RESULT_BATCH / RESULT_END / EXEC_DONE / QUERY_ERROR of a query // and the user callback, only after it. handleCacheReset(payloadPtr, payloadLen); @@ -318,6 +317,10 @@ public void run() { currentQueryDone = false; currentRequestId = req.requestId; creditEnabled = req.initialCredit > 0L; + // The schema rides the first RESULT_BATCH (batch_seq == 0) of each + // query; invalidate any schema left from the prior query so a + // continuation batch can't bind rows to a stale schema. + decoder.resetQuerySchema(); sendQueryRequest(req); while (!currentQueryDone && !shutdown) { @@ -573,8 +576,8 @@ private void emitTransportErrorBlocking(String message) { /** * Decodes a {@code CACHE_RESET} frame body and clears the indicated - * connection-scoped caches on the client side. Body is a single byte mask: - * bit 0 = SYMBOL dict, bit 1 = schema-fingerprint cache. + * connection-scoped caches on the client side. Body is a single byte mask; + * bit 0 = SYMBOL dict is the only bit defined. */ private void handleCacheReset(long payloadPtr, int payloadLen) { int bodyStart = QwpConstants.HEADER_SIZE + 1; // msg_kind byte consumed by caller diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java index 9dd6287f..63d1fd43 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpEgressMsgKind.java @@ -32,9 +32,9 @@ public final class QwpEgressMsgKind { /** * Server -> client. Connection-scoped cache reset. Body: {@code reset_mask:u8} - * with bit 0 = SYMBOL dict, bit 1 = schema-fingerprint cache. Sent between - * queries when a cache hits its server-side soft cap. Recipient clears the - * indicated caches; subsequent RESULT_BATCH delta sections start fresh. + * with bit 0 = SYMBOL dict (the only bit defined). Sent between queries when + * the dict hits its server-side soft cap. Recipient clears the dict; + * subsequent RESULT_BATCH delta sections start fresh. */ public static final byte CACHE_RESET = 0x17; public static final byte CANCEL = 0x14; @@ -59,10 +59,6 @@ public final class QwpEgressMsgKind { * Reset mask bit: clear the connection-scoped SYMBOL dict. */ public static final byte RESET_MASK_DICT = 0x01; - /** - * Reset mask bit: clear the connection-scoped schema-fingerprint cache. - */ - public static final byte RESET_MASK_SCHEMAS = 0x02; public static final byte RESULT_BATCH = 0x11; public static final byte RESULT_END = 0x12; /** @@ -88,7 +84,7 @@ public final class QwpEgressMsgKind { public static final byte ROLE_STANDALONE = 0; /** * Server -> client. Unsolicited frame delivered as the first QWP message - * on every v2 WebSocket connection. Body (little-endian): {@code + * on every WebSocket connection. Body (little-endian): {@code * msg_kind:u8, role:u8, epoch:u64, capabilities:u32, server_wall_ns:i64, * cluster_id:u16_len+utf8, node_id:u16_len+utf8} followed by an optional * {@code zone_id:u16_len+utf8} when the {@link #CAP_ZONE} bit is set in diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java index 4c4d76cd..166c0331 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpHostHealthTracker.java @@ -274,7 +274,7 @@ public void recordTransportError(int idx) { /** * Records a server-advertised zone for the given host index per * failover.md §2.1. Called once with {@code SERVER_INFO.zone_id} after a - * successful upgrade on a v2 connection (gated by {@code CAP_ZONE}), and + * successful upgrade (gated by {@code CAP_ZONE}), and * once with the {@code X-QuestDB-Zone} HTTP header on a {@code 421} * upgrade reject. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java index 21431170..584602e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpProtocolVersionException.java @@ -28,7 +28,7 @@ import io.questdb.client.std.str.StringSink; /** - * Server negotiated a QWP version outside {@code [VERSION_1, MAX_SUPPORTED_VERSION]}. + * Server negotiated a QWP version other than {@code QwpConstants.VERSION}. * Terminal: version negotiation is cluster-wide, so failover masks the disagreement. */ public class QwpProtocolVersionException extends QwpDecodeException { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java index d21aeffd..4051e1ac 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpQueryClient.java @@ -63,7 +63,7 @@ * Multi-endpoint routing: the connection string accepts a comma-separated list * of {@code addr=host:port[,host:port...]} endpoints plus a {@code target=} * filter ({@code any} | {@code primary} | {@code replica}). {@link #connect()} - * walks the list in order, reads the server's role from the v2 + * walks the list in order, reads the server's role from the * {@code SERVER_INFO} frame, and picks the first endpoint matching the target. * When every endpoint reports a role the filter rejects, {@link #connect()} * throws {@link QwpRoleMismatchException} with the last observed info attached @@ -76,7 +76,7 @@ * {@link QwpColumnBatchHandler#onFailoverReset} callback just before replayed * batches start arriving (batch_seq restarts at 0 on the new node), so * accumulating handlers can discard rows the old connection already delivered. - * {@code failover=off} restores the pre-v2 behaviour -- terminal failures + * {@code failover=off} disables this -- terminal failures * surface immediately through {@link QwpColumnBatchHandler#onError}. *

* Terminal-failure latching: transport- or protocol-level faults detected by @@ -109,7 +109,7 @@ public class QwpQueryClient implements QuietCloseable { * sanity bound. */ public static final int MAX_BATCH_ROWS_UPPER_BOUND = 1_048_576; - public static final int QWP_MAX_VERSION = QwpConstants.MAX_SUPPORTED_VERSION; + public static final int QWP_MAX_VERSION = QwpConstants.VERSION; public static final String TARGET_ANY = "any"; public static final String TARGET_PRIMARY = "primary"; public static final String TARGET_REPLICA = "replica"; @@ -141,7 +141,7 @@ public class QwpQueryClient implements QuietCloseable { private static final long DEFAULT_FAILOVER_MAX_DURATION_MS = 30_000L; private static final int DEFAULT_IO_BUFFER_POOL_SIZE = 4; /** - * How long {@link #connect()} waits to read the v2 {@code SERVER_INFO} frame + * How long {@link #connect()} waits to read the {@code SERVER_INFO} frame * from each endpoint before giving up and moving to the next. 5 seconds is * comfortable on a WAN; the server writes SERVER_INFO into the same send * buffer as the 101 upgrade response so under normal conditions the frame @@ -262,9 +262,9 @@ public class QwpQueryClient implements QuietCloseable { // worker thread's post-requestId read. private volatile boolean pendingCancel; // Decoded SERVER_INFO from the current connection's handshake. Null before - // connect() has succeeded, and on connections that negotiated v1 (which - // doesn't emit the frame). Volatile so the I/O thread's read on the - // {@code onFailoverReset} path sees the latest reconnect. + // connect() has succeeded; non-null on every established connection (the + // server always emits the frame). Volatile so getServerInfo(), callable + // from any thread, observes the latest reconnect's value. private volatile QwpServerInfo serverInfo; private int serverInfoTimeoutMs = DEFAULT_SERVER_INFO_TIMEOUT_MS; // Maximum time close() will wait for the I/O thread to exit before giving up @@ -312,7 +312,7 @@ private QwpQueryClient(String host, int port) { * Per {@code failover.md} section 1, the comma form and repeated {@code addr=} keys * both accumulate into a single ordered list; empty entries are rejected. *

  • {@code target=any|primary|replica} -- endpoint filter applied against the role - * byte from the v2 {@code SERVER_INFO} frame. Default {@code any}. {@code primary} + * byte from the {@code SERVER_INFO} frame. Default {@code any}. {@code primary} * accepts {@code PRIMARY}, {@code PRIMARY_CATCHUP} and {@code STANDALONE}.
    • *
  • {@code failover=on|off} -- default {@code on}. On transport failure during * {@link #execute}, reconnect to another endpoint and re-submit the query. @@ -602,7 +602,6 @@ public static QwpQueryClient fromConfig(CharSequence configurationString) { case "max_buf_size": case "max_datagram_size": case "max_name_len": - case "max_schemas_per_connection": case "multicast_ttl": case "pass": case "protocol_version": @@ -826,7 +825,7 @@ public void close() { * performs the WebSocket upgrade. Must be called before any query is submitted. *

    * Walks the endpoint list in order: for each entry it opens the TCP socket, - * runs the HTTP upgrade, reads the v2 {@code SERVER_INFO} frame, and accepts + * runs the HTTP upgrade, reads the {@code SERVER_INFO} frame, and accepts * the endpoint if the server's role matches the configured target. An * endpoint that matches becomes the bound connection and the I/O thread is * spawned. An endpoint whose role doesn't match is closed and the walk @@ -855,7 +854,6 @@ public synchronized void connect() { } QwpServerInfo lastObservedMismatch = null; QwpIngressRoleRejectedException lastUpgradeRoleReject = null; - boolean sawV1Mismatch = false; Throwable lastTransportError = null; while (true) { int i = hostTracker.pickNext(); @@ -884,19 +882,13 @@ public synchronized void connect() { cleanupFailedConnect(); continue; } + // info is non-null: connectToEndpoint() returns only after + // receiveServerInfoSync() set serverInfo (it returns non-null or throws). QwpServerInfo info = serverInfo; - if (info != null && (info.getCapabilities() & QwpEgressMsgKind.CAP_ZONE) != 0) { + if ((info.getCapabilities() & QwpEgressMsgKind.CAP_ZONE) != 0) { hostTracker.recordZone(i, info.getZoneId()); } - if (!TARGET_ANY.equals(target) && info == null) { - sawV1Mismatch = true; - hostTracker.recordRoleReject(i, false); - LOG.info("QwpQueryClient {}:{} negotiated v1 (no SERVER_INFO) and target={} requires v2; trying next", - ep.host, ep.port, target); - cleanupFailedConnect(); - continue; - } - if (info != null && !matchesTarget(info.getRole(), target)) { + if (!matchesTarget(info.getRole(), target)) { lastObservedMismatch = info; boolean isTransient = info.getRole() == QwpEgressMsgKind.ROLE_PRIMARY_CATCHUP; hostTracker.recordRoleReject(i, isTransient); @@ -920,14 +912,6 @@ public synchronized void connect() { + " cluster=" + lastObservedMismatch.getClusterId() ); } - if (sawV1Mismatch) { - throw new QwpRoleMismatchException( - target, - null, - "no endpoint matches target=" + target - + "; at least one endpoint negotiated v1 and cannot supply a role" - ); - } if (lastUpgradeRoleReject != null) { // 421 + X-QuestDB-Role on every host: the cluster answered with role // information at upgrade time but no host satisfied target=. Surface @@ -1061,8 +1045,7 @@ public int getNegotiatedZstdLevel() { /** * Returns the {@link QwpServerInfo} decoded from the currently-bound - * server's {@code SERVER_INFO} frame, or {@code null} if the server - * negotiated the v1 protocol (no frame sent) or the client is not + * server's {@code SERVER_INFO} frame, or {@code null} if the client is not * connected. The value is refreshed on every successful failover reconnect. */ public QwpServerInfo getServerInfo() { @@ -1309,7 +1292,7 @@ public QwpQueryClient withInsecureTls() { * Useful for latency-sensitive streaming consumers that want to start * processing the first row as soon as possible -- a smaller cap flushes * the first batch sooner, at the cost of more per-batch overhead (WS - * header, send syscall, schema-reference decode). The server clamps down + * header, send syscall). The server clamps down * to its own hard limit; a value of {@code 0} (default) omits the header * and the server uses its own cap. *

    @@ -1326,7 +1309,7 @@ public QwpQueryClient withMaxBatchRows(int rows) { } /** - * Overrides the {@link #DEFAULT_SERVER_INFO_TIMEOUT_MS} wait for the v2 + * Overrides the {@link #DEFAULT_SERVER_INFO_TIMEOUT_MS} wait for the * {@code SERVER_INFO} frame. Must be called before {@link #connect}. */ public void withServerInfoTimeout(int ms) { @@ -1610,16 +1593,12 @@ private void connectToEndpoint(Endpoint ep) { negotiatedQwpVersion = webSocketClient.getServerQwpVersion(); negotiatedZstdLevel = webSocketClient.getServerNegotiatedZstdLevel(); - // v2 servers send SERVER_INFO as the first WebSocket frame after the + // The server sends SERVER_INFO as the first WebSocket frame after the // upgrade response. Consume it synchronously on the user thread before // spawning the I/O thread, so the role filter can run without any // cross-thread synchronisation and so a mismatched role doesn't waste // the I/O thread setup + teardown. - if (negotiatedQwpVersion >= QwpConstants.VERSION_2) { - serverInfo = receiveServerInfoSync(); - } else { - serverInfo = null; - } + serverInfo = receiveServerInfoSync(); // Early probe: if we told the server we can accept zstd, make sure the // bundled native library actually provides the decompression symbols @@ -1926,7 +1905,6 @@ private void reconnectViaTracker() { lastCloseTimedOut = false; hostTracker.beginRound(false); QwpServerInfo lastMismatch = null; - boolean sawV1Mismatch = false; Throwable lastError = null; boolean retriedAfterReset = false; while (true) { @@ -1957,17 +1935,13 @@ private void reconnectViaTracker() { cleanupFailedConnect(); continue; } + // info is non-null: connectToEndpoint() returns only after + // receiveServerInfoSync() set serverInfo (it returns non-null or throws). QwpServerInfo info = serverInfo; - if (info != null && (info.getCapabilities() & QwpEgressMsgKind.CAP_ZONE) != 0) { + if ((info.getCapabilities() & QwpEgressMsgKind.CAP_ZONE) != 0) { hostTracker.recordZone(i, info.getZoneId()); } - if (!TARGET_ANY.equals(target) && info == null) { - sawV1Mismatch = true; - hostTracker.recordRoleReject(i, false); - cleanupFailedConnect(); - continue; - } - if (info != null && !matchesTarget(info.getRole(), target)) { + if (!matchesTarget(info.getRole(), target)) { lastMismatch = info; boolean isTransient = info.getRole() == QwpEgressMsgKind.ROLE_PRIMARY_CATCHUP; hostTracker.recordRoleReject(i, isTransient); @@ -1985,11 +1959,6 @@ private void reconnectViaTracker() { "no endpoint matches target=" + target + " on failover; last observed role=" + QwpServerInfo.roleName(lastMismatch.getRole())); } - if (sawV1Mismatch) { - throw new QwpRoleMismatchException(target, null, - "no endpoint matches target=" + target - + " on failover; at least one endpoint negotiated v1 and cannot supply a role"); - } throw new HttpClientException( "all QWP endpoints unreachable on failover [count=" + total + ", lastError=" + (lastError == null ? "" : lastError.getMessage()) + ']'); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java index 46dfe558..f1c8d706 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpResultBatchDecoder.java @@ -83,16 +83,8 @@ public class QwpResultBatchDecoder implements QuietCloseable { * to leave head-room for future server-side batch enlargement without breaking clients. */ private static final int MAX_ROWS_PER_BATCH = 1_048_576; - /** - * Hard cap on registered schema ids per connection. Matches - * {@code QwpConstants.DEFAULT_MAX_SCHEMAS_PER_CONNECTION} on the server side. - * Capping protects the client from a hostile or buggy server that could - * otherwise force unbounded {@code schemaRegistry} growth (or AIOOBE on a - * negative schema id) by encoding {@code schemaId = Integer.MAX_VALUE} (or - * a negative varint that long-to-int casts negative). - */ - private static final int MAX_SCHEMAS_PER_CONNECTION = 65_535; - // Reusable scratch for decoding column names during SCHEMA_MODE_FULL parsing. + // Reusable scratch for decoding column names from the first batch's inline + // schema (batch_seq == 0). // Sized to the wire-level cap so {@link #readColumnName} can copy bytes // without allocating a fresh {@code byte[]} per column. The returned // {@code String} is still a fresh allocation (immutable + retained @@ -115,9 +107,23 @@ public class QwpResultBatchDecoder implements QuietCloseable { // Storing offsets (rather than absolute addresses) means heap reallocs don't // invalidate entries; the hot-path accessor does one 64-bit load + base add, // no object dereferences. Grows but never shrinks; freed on {@link #close}. - // Registry indexed by schemaId. null = not registered. Schema ids are server-assigned - // and small (monotonic from 0). - private final ObjList> schemaRegistry = new ObjList<>(); + // Per-query column schema. The full column list rides only the first + // RESULT_BATCH of a query (batch_seq == 0); continuation batches carry rows + // against it. Reused across queries (slots pooled); the IoThread invalidates + // it via resetQuerySchema() when a new query starts, so a stray continuation + // frame can't bind rows to a stale schema. + // + // Dropping the old connection-scoped schema registry means every query's + // batch_seq == 0 re-parses the inline schema and allocates a fresh + // column-name String per column, even when the same query shape repeats -- + // the trade-off for not carrying schema ids on the wire. + // + // A single slot is correct only because the IoThread runs one query at a + // time (one pendingRequest, one currentRequestId). If client-side + // pipelining is ever enabled so batches from different request_ids + // interleave, this must become a map keyed by request_id; the single + // querySchemaValid flag would otherwise mis-bind rows across queries. + private final ObjList querySchema = new ObjList<>(); private long connDictEntriesAddr; private int connDictEntriesCapacity; // Monotonic generation counter for the connection-scoped dict. Incremented @@ -145,19 +151,21 @@ public class QwpResultBatchDecoder implements QuietCloseable { // TIMESTAMP / TIMESTAMP_NANOS / DATE columns are prefixed by a 1-byte // encoding discriminator (0x00 raw, 0x01 Gorilla). private boolean gorillaMode; + // True once the schema-bearing batch_seq == 0 of the current query has been + // parsed into {@link #querySchema}. Reset by {@link #resetQuerySchema()} when + // a new query starts, so a continuation batch can never bind to a stale schema. + private boolean querySchemaValid; // Reusable varint decode state: value in varintValue, new position in varintPos. // Instance-level so no {@code long[2]} scratch is allocated per call. private long varintPos; private long varintValue; /** - * Clears the caches indicated by {@code resetMask} (bitwise OR of - * {@link QwpEgressMsgKind#RESET_MASK_DICT} and - * {@link QwpEgressMsgKind#RESET_MASK_SCHEMAS}). Drives the client-side - * state machine when the server emits a {@code CACHE_RESET} frame after - * hitting a connection-level soft cap: discards the SYMBOL dict and / or - * schema-fingerprint cache so the next batch's {@code deltaStart} and - * schema-reference ids line up with the server's fresh counter. + * Clears the caches indicated by {@code resetMask}. The only bit defined is + * {@link QwpEgressMsgKind#RESET_MASK_DICT}: the server emits a + * {@code CACHE_RESET} frame after hitting the connection-level SYMBOL dict + * soft cap, and dropping the dict here keeps the next batch's + * {@code deltaStart} aligned with the server's fresh counter. *

    * The native dict buffers are reused (positions reset to 0, capacity * retained) so a workload that churns just above the cap does not reallocate @@ -172,9 +180,6 @@ public void applyCacheReset(byte resetMask) { // and wipes before reading. connDictGeneration++; } - if ((resetMask & QwpEgressMsgKind.RESET_MASK_SCHEMAS) != 0) { - schemaRegistry.clear(); - } } @Override @@ -200,6 +205,10 @@ public void close() { dctx = 0; } connDictSize = 0; + // Invalidate any per-query schema captured before close(). The decoder is + // single-use per connection today, but resetting here keeps a future + // reuse from binding a continuation batch to a prior connection's schema. + querySchemaValid = false; } /** @@ -222,6 +231,16 @@ public void decode(QwpBatchBuffer buffer, long payloadPtr, int payloadLen) throw decodePayload(buffer, payloadPtr, payloadLen); } + /** + * Invalidates the per-query schema captured from the last {@code batch_seq == 0}. + * The IoThread calls this when a new query starts so the next query's + * continuation batches can't bind rows to a stale schema; the underlying + * {@link QwpEgressColumnInfo} slots are retained for reuse. + */ + public void resetQuerySchema() { + querySchemaValid = false; + } + // Pool helpers private static long advanceFixed(QwpColumnLayout layout, long p, long limit, int sizeBytes) throws QwpDecodeException { @@ -297,7 +316,7 @@ private void decodePayload(QwpBatchBuffer buffer, long payload, int payloadLen) throw new QwpDecodeException("bad magic 0x" + Integer.toHexString(magic)); } byte version = Unsafe.getUnsafe().getByte(payload + 4); - if (version < QwpConstants.VERSION_1 || version > QwpConstants.MAX_SUPPORTED_VERSION) { + if (version != QwpConstants.VERSION) { throw QwpProtocolVersionException.unsupported(version & 0xFF); } byte flags = Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS); @@ -411,7 +430,11 @@ private void decodePayload(QwpBatchBuffer buffer, long payload, int payloadLen) p = parseDeltaSymbolDict(p, limit); } - // Table block: name_length, name, row_count, column_count, schema, columns + // Table block. The schema (column_count + inline column descriptors) + // rides only the first batch of a query (batch_seq == 0); continuation + // batches carry rows against the schema parsed there: + // batch_seq == 0: name_length, name, row_count, column_count, columns + // batch_seq > 0: name_length, name, row_count decodeVarint(p, limit); long nameLen = varintValue; p = varintPos; @@ -433,29 +456,19 @@ private void decodePayload(QwpBatchBuffer buffer, long payload, int payloadLen) } int rowCount = (int) varintValue; p = varintPos; - decodeVarint(p, limit); - if (varintValue < 0 || varintValue > QwpConstants.MAX_COLUMNS_PER_TABLE) { - throw new QwpDecodeException("column_count out of range: " + varintValue); - } - int columnCount = (int) varintValue; - p = varintPos; - - // Schema section - if (p >= limit) throw new QwpDecodeException("truncated schema mode"); - byte schemaMode = Unsafe.getUnsafe().getByte(p++); - decodeVarint(p, limit); - // Reject schema ids that wouldn't fit in our registry (or that cast negative - // from a hostile high varint). Without this guard, ensureSchemaSlot would - // either OOM appending billions of nulls or AIOOBE on a negative index. - if (varintValue < 0 || varintValue >= MAX_SCHEMAS_PER_CONNECTION) { - throw new QwpDecodeException("schema_id out of range: " + varintValue); - } - int schemaId = (int) varintValue; - p = varintPos; ObjList columns; - if (schemaMode == QwpConstants.SCHEMA_MODE_FULL) { - columns = ensureSchemaSlot(schemaId, columnCount); + int columnCount; + if (batchSeq == 0) { + // First batch carries the full schema: column_count then one + // (name_length, name, wire_type) descriptor per column. + decodeVarint(p, limit); + if (varintValue < 0 || varintValue > QwpConstants.MAX_COLUMNS_PER_TABLE) { + throw new QwpDecodeException("column_count out of range: " + varintValue); + } + columnCount = (int) varintValue; + p = varintPos; + columns = ensureQuerySchema(columnCount); for (int i = 0; i < columnCount; i++) { decodeVarint(p, limit); // Same negative / out-of-range guard as nameLen above; without @@ -472,16 +485,17 @@ private void decodePayload(QwpBatchBuffer buffer, long payload, int payloadLen) byte wireType = Unsafe.getUnsafe().getByte(p++); columns.getQuick(i).of(colName, wireType); } - } else if (schemaMode == QwpConstants.SCHEMA_MODE_REFERENCE) { - if (schemaId >= schemaRegistry.size() || schemaRegistry.getQuick(schemaId) == null) { - throw new QwpDecodeException("schema id " + schemaId + " not registered on this connection"); - } - columns = schemaRegistry.getQuick(schemaId); - if (columns.size() != columnCount) { - throw new QwpDecodeException("schema id " + schemaId + " column count mismatch"); - } + querySchemaValid = true; } else { - throw new QwpDecodeException("unknown schema mode 0x" + Integer.toHexString(schemaMode & 0xFF)); + // Continuation batch: reuse the schema delivered on batch_seq == 0. + // A continuation that arrives before any schema-bearing batch (a + // malformed or hostile server) must not bind rows to a stale schema. + if (!querySchemaValid) { + throw new QwpDecodeException("RESULT_BATCH batch_seq=" + batchSeq + + " arrived before the schema-bearing batch_seq=0"); + } + columns = querySchema; + columnCount = querySchema.size(); } // Reset batch view and parse columns into per-column layouts owned by the buffer. @@ -550,27 +564,19 @@ private void ensureConnDictHeapCapacity(int required) { // addresses, so the hot-path accessor resolves against the current heap base. } - private ObjList ensureSchemaSlot(int schemaId, int columnCount) { - while (schemaRegistry.size() <= schemaId) { - schemaRegistry.add(null); - } - ObjList slot = schemaRegistry.getQuick(schemaId); - if (slot == null) { - slot = new ObjList<>(); - schemaRegistry.setQuick(schemaId, slot); - } - int currentPos = slot.size(); + private ObjList ensureQuerySchema(int columnCount) { + int currentPos = querySchema.size(); if (columnCount > currentPos) { - slot.setPos(columnCount); + querySchema.setPos(columnCount); for (int i = currentPos; i < columnCount; i++) { - if (slot.getQuick(i) == null) { - slot.setQuick(i, new QwpEgressColumnInfo()); + if (querySchema.getQuick(i) == null) { + querySchema.setQuick(i, new QwpEgressColumnInfo()); } } } else { - slot.setPos(columnCount); + querySchema.setPos(columnCount); } - return slot; + return querySchema; } private long parseArrayColumn(QwpColumnLayout layout, int rowCount, long p, long limit) throws QwpDecodeException { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfo.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfo.java index bdf072b4..c6f0778e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfo.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfo.java @@ -25,7 +25,7 @@ package io.questdb.client.cutlass.qwp.client; /** - * Decoded {@code SERVER_INFO} frame delivered by a v2 QWP egress server as the + * Decoded {@code SERVER_INFO} frame delivered by a QWP egress server as the * first WebSocket frame after the upgrade handshake. Immutable; safe to publish * across threads without synchronisation once assigned. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfoDecoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfoDecoder.java index b8a58322..bb7b6845 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfoDecoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpServerInfoDecoder.java @@ -83,10 +83,10 @@ public static QwpServerInfo decode(long payload, int payloadLen) throws QwpDecod throw new QwpDecodeException("SERVER_INFO truncated before node_id length"); } String nodeId = readUtf8U16(p, payloadEnd, "node_id"); - // Trailing zone_id is gated by the CAP_ZONE bit so a v2.0 client - // reading a v2.0 server (capabilities=0) never reaches this branch and - // sees the unchanged byte layout. When the bit is set the trailer is - // mandatory; a missing or truncated zone_id is a wire error. + // Trailing zone_id is gated by the CAP_ZONE bit so a client reading a + // server with no zone configured (capabilities=0) never reaches this + // branch and sees the zone-less byte layout. When the bit is set the + // trailer is mandatory; a missing or truncated zone_id is a wire error. String zoneId = null; if ((capabilities & QwpEgressMsgKind.CAP_ZONE) != 0) { int nodeLen = Unsafe.getUnsafe().getShort(p) & 0xFFFF; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java index cbae2e03..f02e87c3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpUdpSender.java @@ -60,7 +60,7 @@ *

    * Each {@link #flush()} encodes all buffered table data into self-contained * datagrams (one per table) and sends them via UDP. Datagrams use local - * symbol dictionaries (no global/delta dict) and full schema (no schema refs). + * symbol dictionaries (no global/delta dict) and inline column schemas. *

    * When {@code maxDatagramSize > 0}, the sender automatically flushes before * a datagram exceeds the size limit. The in-progress row stays staged in sender @@ -1005,9 +1005,7 @@ private int encodeCommittedPrefixPayloadForUdp(QwpTableBuffer tableBuffer) { prefixStringDataSizeBefore, prefixSymbolDictionarySizeBefore, false, - false, - false, - 0 + false ); payloadWriter.finish(); return payloadWriter.getPosition(); @@ -1016,7 +1014,7 @@ private int encodeCommittedPrefixPayloadForUdp(QwpTableBuffer tableBuffer) { private int encodeTablePayloadForUdp(QwpTableBuffer tableBuffer) { payloadWriter.reset(); columnWriter.setBuffer(payloadWriter); - columnWriter.encodeTable(tableBuffer, false, false, false); + columnWriter.encodeTable(tableBuffer, false, false); payloadWriter.finish(); return payloadWriter.getPosition(); } @@ -1091,8 +1089,6 @@ private long estimateBaseForCurrentSchema() { estimate += NativeBufferWriter.varintSize(tableNameUtf8) + tableNameUtf8; estimate += VARINT_INT_UPPER_BOUND; estimate += VARINT_INT_UPPER_BOUND; - estimate += 1; // schema mode byte - estimate += VARINT_INT_UPPER_BOUND; // schemaId QwpColumnDef[] defs = currentTableBuffer.getColumnDefs(); for (int i = 0, n = defs.length; i < n; i++) { @@ -1231,7 +1227,7 @@ private void sendEncodedPayload(CharSequence tableName, int payloadLength) { headerBuffer.putByte((byte) 'W'); headerBuffer.putByte((byte) 'P'); headerBuffer.putByte((byte) '1'); - headerBuffer.putByte(VERSION_1); + headerBuffer.putByte(VERSION); headerBuffer.putByte((byte) 0); headerBuffer.putShort((short) 1); headerBuffer.putInt(payloadLength); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index f4b13733..85db3f0d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -41,7 +41,7 @@ public class QwpWebSocketEncoder implements QuietCloseable { private NativeBufferWriter buffer; private byte flags; private int payloadStart; - private byte version = VERSION_1; + private byte version = VERSION; public QwpWebSocketEncoder() { this.buffer = new NativeBufferWriter(); @@ -53,8 +53,8 @@ public QwpWebSocketEncoder(int bufferSize) { this.flags = 0; } - public void addTable(QwpTableBuffer tableBuffer, boolean useSchemaRef) { - columnWriter.encodeTable(tableBuffer, useSchemaRef, true, isGorillaEnabled()); + public void addTable(QwpTableBuffer tableBuffer) { + columnWriter.encodeTable(tableBuffer, true, isGorillaEnabled()); } public void beginMessage( @@ -89,12 +89,12 @@ public void close() { } } - public int encode(QwpTableBuffer tableBuffer, boolean useSchemaRef) { + public int encode(QwpTableBuffer tableBuffer) { buffer.reset(); writeHeader(1, 0); int payloadStart = buffer.getPosition(); columnWriter.setBuffer(buffer); - columnWriter.encodeTable(tableBuffer, useSchemaRef, false, isGorillaEnabled()); + columnWriter.encodeTable(tableBuffer, false, isGorillaEnabled()); int payloadLength = buffer.getPosition() - payloadStart; buffer.patchInt(8, payloadLength); return buffer.getPosition(); @@ -104,11 +104,10 @@ public int encodeWithDeltaDict( QwpTableBuffer tableBuffer, GlobalSymbolDictionary globalDict, int confirmedMaxId, - int batchMaxId, - boolean useSchemaRef + int batchMaxId ) { beginMessage(1, globalDict, confirmedMaxId, batchMaxId); - addTable(tableBuffer, useSchemaRef); + addTable(tableBuffer); return finishMessage(); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 7d43d72f..c8af5fc5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -126,7 +126,6 @@ public class QwpWebSocketSender implements Sender { public static final int DEFAULT_AUTO_FLUSH_BYTES = 8 * 1024 * 1024; public static final long DEFAULT_AUTO_FLUSH_INTERVAL_NANOS = 100_000_000L; // 100ms public static final int DEFAULT_AUTO_FLUSH_ROWS = 1_000; - public static final int DEFAULT_MAX_SCHEMAS_PER_CONNECTION = 65_535; private static final int DEFAULT_BUFFER_SIZE = 8192; private static final int DEFAULT_MICROBATCH_BUFFER_SIZE = 1024 * 1024; // 1MB private static final Logger LOG = LoggerFactory.getLogger(QwpWebSocketSender.class); @@ -149,7 +148,6 @@ public class QwpWebSocketSender implements Sender { // Global symbol dictionary for delta encoding private final GlobalSymbolDictionary globalSymbolDictionary; private final QwpHostHealthTracker hostTracker; - private final int maxSchemasPerConnection; private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -234,10 +232,6 @@ public class QwpWebSocketSender implements Sender { // to the SenderError dispatcher rather than thrown from the // constructor. private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF; - private int maxSentSchemaId = -1; - // Track the highest symbol ID sent to server (for delta encoding) - // Once sent over TCP, server is guaranteed to receive it (or connection dies) - private int nextSchemaId; private boolean ownsCursorEngine; private long pendingBytes; private int pendingRowCount; @@ -280,8 +274,7 @@ private QwpWebSocketSender( int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos, - String authorizationHeader, - int maxSchemasPerConnection + String authorizationHeader ) { if (endpoints == null || endpoints.isEmpty()) { throw new IllegalArgumentException("endpoints must be non-empty"); @@ -304,7 +297,6 @@ private QwpWebSocketSender( // X-QWP-Max-Batch-Size is known. this.effectiveAutoFlushBytes = autoFlushBytes; this.autoFlushIntervalNanos = autoFlushIntervalNanos; - this.maxSchemasPerConnection = maxSchemasPerConnection; this.globalSymbolDictionary = new GlobalSymbolDictionary(); int microbatchBufferSize = Math.max(DEFAULT_MICROBATCH_BUFFER_SIZE, autoFlushBytes * 2); @@ -355,7 +347,7 @@ public static QwpWebSocketSender connect(String host, int port, ClientTlsConfigu return connect( host, port, tlsConfig, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, - null, DEFAULT_MAX_SCHEMAS_PER_CONNECTION, + null, false, engine ); } catch (Throwable t) { @@ -381,12 +373,11 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine ) { return connect(host, port, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - authorizationHeader, maxSchemasPerConnection, + authorizationHeader, requestDurableAck, cursorEngine, 5_000L); } @@ -404,14 +395,13 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine, long closeFlushTimeoutMillis ) { return connect(host, port, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, authorizationHeader, - maxSchemasPerConnection, requestDurableAck, cursorEngine, + requestDurableAck, cursorEngine, closeFlushTimeoutMillis, CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS, CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS, @@ -432,7 +422,6 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine, long closeFlushTimeoutMillis, @@ -442,7 +431,7 @@ public static QwpWebSocketSender connect( ) { return connect(host, port, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, authorizationHeader, - maxSchemasPerConnection, requestDurableAck, cursorEngine, + requestDurableAck, cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, Sender.InitialConnectMode.OFF); @@ -463,7 +452,6 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine, long closeFlushTimeoutMillis, @@ -474,7 +462,7 @@ public static QwpWebSocketSender connect( ) { return connect(host, port, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, authorizationHeader, - maxSchemasPerConnection, requestDurableAck, cursorEngine, + requestDurableAck, cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, initialConnectMode, null, SenderErrorDispatcher.DEFAULT_CAPACITY); @@ -493,7 +481,6 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine, long closeFlushTimeoutMillis, @@ -506,7 +493,7 @@ public static QwpWebSocketSender connect( ) { return connect(host, port, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, authorizationHeader, - maxSchemasPerConnection, requestDurableAck, cursorEngine, + requestDurableAck, cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, initialConnectMode, errorHandler, errorInboxCapacity, @@ -527,7 +514,6 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine, long closeFlushTimeoutMillis, @@ -542,7 +528,7 @@ public static QwpWebSocketSender connect( return connect( singleEndpoint(host, port), tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - authorizationHeader, maxSchemasPerConnection, + authorizationHeader, requestDurableAck, cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, @@ -566,7 +552,6 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine, long closeFlushTimeoutMillis, @@ -582,7 +567,7 @@ public static QwpWebSocketSender connect( ) { return connect(endpoints, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, authorizationHeader, - maxSchemasPerConnection, requestDurableAck, cursorEngine, + requestDurableAck, cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, initialConnectMode, errorHandler, errorInboxCapacity, @@ -601,7 +586,6 @@ public static QwpWebSocketSender connect( int autoFlushBytes, long autoFlushIntervalNanos, String authorizationHeader, - int maxSchemasPerConnection, boolean requestDurableAck, CursorSendEngine cursorEngine, long closeFlushTimeoutMillis, @@ -620,7 +604,7 @@ public static QwpWebSocketSender connect( QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - authorizationHeader, maxSchemasPerConnection + authorizationHeader ); try { sender.requestDurableAck = requestDurableAck; @@ -672,7 +656,7 @@ public static QwpWebSocketSender createForTesting(String host, int port, String return new QwpWebSocketSender( singleEndpoint(host, port), null, DEFAULT_AUTO_FLUSH_ROWS, DEFAULT_AUTO_FLUSH_BYTES, DEFAULT_AUTO_FLUSH_INTERVAL_NANOS, - authorizationHeader, DEFAULT_MAX_SCHEMAS_PER_CONNECTION + authorizationHeader ); } @@ -692,29 +676,11 @@ public static QwpWebSocketSender createForTesting( int autoFlushRows, int autoFlushBytes, long autoFlushIntervalNanos - ) { - return createForTesting( - host, - port, - autoFlushRows, - autoFlushBytes, - autoFlushIntervalNanos, - DEFAULT_MAX_SCHEMAS_PER_CONNECTION - ); - } - - public static QwpWebSocketSender createForTesting( - String host, - int port, - int autoFlushRows, - int autoFlushBytes, - long autoFlushIntervalNanos, - int maxSchemasPerConnection ) { return new QwpWebSocketSender( singleEndpoint(host, port), null, autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, - null, maxSchemasPerConnection + null ); } @@ -2421,7 +2387,7 @@ private synchronized WebSocketClient buildAndConnect(ReconnectSupplier ctx) { ? WebSocketClientFactory.newTlsInstance(tlsConfig) : WebSocketClientFactory.newPlainTextInstance(); try { - newClient.setQwpMaxVersion(QwpConstants.MAX_SUPPORTED_INGEST_VERSION); + newClient.setQwpMaxVersion(QwpConstants.VERSION); newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); newClient.connect(ep.host, ep.port); @@ -2762,12 +2728,12 @@ private void ensureConnected() { case ASYNC: // Defer the actual connect to the I/O thread. The user thread // returns immediately; rows accumulate in the cursor SF engine. - // Encoder stays at its default (V1 — the only supported wire - // version today). When v2+ ships, frames written before the - // first successful connect will commit to V1 because cursor - // segments are immutable. Auth/upgrade rejects and budget - // exhaustion are surfaced via the error inbox by the I/O - // thread, not thrown here. + // Encoder stays at its default (V1 -- the only supported wire + // version today). Frames written before the first successful + // connect commit to V1 because cursor segments are immutable; + // a future version bump must account for that. Auth/upgrade + // rejects and budget exhaustion are surfaced via the error + // inbox by the I/O thread, not thrown here. client = null; break; case OFF: @@ -2843,19 +2809,19 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). Schema state still gets reset for - // consistency with the sync path; the post-connect replay path - // does not need a producer-side reset signal because every + // its default version (V1). The symbol-dict watermark still gets + // reset for consistency with the sync path; the post-connect replay + // path does not need a producer-side reset signal because every // cursor frame is self-sufficient. Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection — discard any schema IDs - // retained from prior state. Cursor frames are self-sufficient (every - // frame carries full schema + full symbol-dict delta from id 0), so - // post-reconnect replay needs no producer-side schema-reset signal. - resetSchemaStateForNewConnection(); + // Server starts fresh on each connection, so reset the symbol-dict + // watermark. Cursor frames are self-sufficient (every frame carries its + // full inline schema + a symbol-dict delta from id 0), so post-reconnect + // replay needs no producer-side reset signal. + resetSymbolDictStateForNewConnection(); connectionError.set(null); connected = true; @@ -2907,16 +2873,11 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient - // — its schema definition must travel with the row data, not - // as a back-reference to an ID the server may not have seen - // (orphan-slot drainers and post-reconnect replay both deliver - // recorded frames to fresh server connections). So always emit - // the full symbol-dict delta from id=0, and always send the - // full schema definition for each table — never a ref. With - // self-sufficient frames there's no encode-vs-reconnect race - // to defend against: the bytes are valid against any server. - int batchMaxSchemaId = maxSentSchemaId; + // Cursor SF requires every on-disk frame to be self-sufficient: + // recorded frames replay to fresh server connections (orphan-slot + // drainers and post-reconnect replay), so always emit the full + // symbol-dict delta from id=0 and the full column schema inline, + // never a back-reference the target server may not have seen. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); @@ -2930,27 +2891,18 @@ private void flushPendingRows(boolean deferCommit) { continue; } - if (tableBuffer.getSchemaId() < 0) { - if (nextSchemaId >= maxSchemasPerConnection) { - throw new LineSenderException("maximum schemas per connection exceeded") - .put("[maxSchemasPerConnection=").put(maxSchemasPerConnection).put(']'); - } - tableBuffer.setSchemaId(nextSchemaId++); - } - batchMaxSchemaId = Math.max(batchMaxSchemaId, tableBuffer.getSchemaId()); - if (LOG.isDebugEnabled()) { - LOG.debug("Encoding table [name={}, rows={}, batchMaxId={}, useSchemaRef=false (cursor SF)]", + LOG.debug("Encoding table [name={}, rows={}, batchMaxId={}]", tableName, tableBuffer.getRowCount(), currentBatchMaxSymbolId); } - encoder.addTable(tableBuffer, /*useSchemaRef=*/ false); + encoder.addTable(tableBuffer); } int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { - flushPendingRowsSplit(keys, batchMaxSchemaId, deferCommit); + flushPendingRowsSplit(keys, deferCommit); return; } @@ -2961,11 +2913,6 @@ private void flushPendingRows(boolean deferCommit) { hasDeferredMessages = deferCommit; - // Update sent state only after successful enqueue. - // If sealAndSwapBuffer() threw, these remain unchanged so the - // next batch's delta dictionary will correctly re-include the - // symbols and schema that the server never received. - maxSentSchemaId = batchMaxSchemaId; resetTableBuffersAfterFlush(keys); } @@ -2980,7 +2927,7 @@ private void flushPendingRows(boolean deferCommit) { * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, int batchMaxSchemaId, boolean deferCommit) { + private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) { if (LOG.isDebugEnabled()) { LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); } @@ -3016,7 +2963,7 @@ private void flushPendingRowsSplit(ObjList keys, int batchMaxSchem encoder.setDeferCommit(deferThis); encoder.beginMessage(1, globalSymbolDictionary, /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer, /*useSchemaRef=*/ false); + encoder.addTable(tableBuffer); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); @@ -3037,7 +2984,6 @@ private void flushPendingRowsSplit(ObjList keys, int batchMaxSchem encoder.setDeferCommit(false); hasDeferredMessages = deferCommit; - maxSentSchemaId = batchMaxSchemaId; resetTableBuffersAfterFlush(keys); } @@ -3081,31 +3027,13 @@ private void sendCommitMessage() { hasDeferredMessages = false; } - private void resetSchemaStateForNewConnection() { - maxSentSchemaId = -1; - nextSchemaId = 0; - // The new server has an empty symbol dictionary. The encoder's - // delta-dictionary range is computed as - // deltaStart = maxSentSymbolId + 1 - // deltaCount = max(0, currentBatchMaxSymbolId - maxSentSymbolId) - // so a non-reset watermark would skip every symbol id <= the old - // server's confirmed max, leaving column refs into a dictionary the - // new server has never seen. Reset both so the next batch ships a - // delta starting at id 0 covering every referenced symbol. + private void resetSymbolDictStateForNewConnection() { + // The new server has an empty symbol dictionary, so the next batch + // must ship a delta starting at id 0. beginMessage() always passes + // confirmedMaxId = -1; resetting the batch watermark here keeps a + // stale value from suppressing re-emission of symbol ids the new + // server has never seen. currentBatchMaxSymbolId = -1; - - ObjList keys = tableBuffers.keys(); - for (int i = 0, n = keys.size(); i < n; i++) { - CharSequence tableName = keys.getQuick(i); - if (tableName == null) { - continue; - } - - QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer != null) { - tableBuffer.setSchemaId(-1); - } - } } private void rollbackRow() { @@ -3167,9 +3095,9 @@ private void sealAndSwapBuffer() { // payload never reached the engine, so no I/O thread will // recycle toSend. Recycle it here so a later flush can swap // back to it; flushPendingRows aborts its post-enqueue state - // updates after this throw, so the source rows and the - // sent-schema watermark stay intact and the next batch re-emits - // the same rows along with the full schema + symbol-dict delta. + // updates after this throw, so the source rows stay intact and the + // next batch re-emits the same rows along with the full inline + // schema and symbol-dict delta from id 0. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 964b6b2d..72fd8509 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -75,7 +75,7 @@ public class WebSocketResponse { */ public static final byte STATUS_DURABLE_ACK = 0x02; public static final byte STATUS_INTERNAL_ERROR = 0x06; - // Status codes (must match docs/qwp/wire-ingress.md) + // Status codes (must match https://questdb.com/docs/connect/wire-protocols/qwp-ingress-websocket/) public static final byte STATUS_OK = 0x00; public static final byte STATUS_PARSE_ERROR = 0x05; public static final byte STATUS_SCHEMA_MISMATCH = 0x03; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 44e3305a..237839d9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -85,14 +85,6 @@ public final class QwpConstants { * constant; used by the decoder to reject malformed wire bytes. */ public static final int MAX_TABLE_NAME_LENGTH = 127; - /** - * Schema mode: Full schema included. - */ - public static final byte SCHEMA_MODE_FULL = 0x00; - /** - * Schema mode: Schema reference (ID lookup). - */ - public static final byte SCHEMA_MODE_REFERENCE = 0x01; /** * Status byte on a {@code QUERY_ERROR} frame: the query was cancelled, * either by a client {@code CANCEL} frame or by explicit server-side @@ -208,29 +200,12 @@ public final class QwpConstants { */ public static final byte TYPE_VARCHAR = 0x0F; /** - * Current protocol version. - */ - public static final byte VERSION_1 = 1; - /** - * Protocol v2 adds an unsolicited {@code SERVER_INFO} control frame - * delivered as the first WebSocket frame after the 101 upgrade. Servers that - * advertise v2 send the frame automatically; v2 clients must consume it - * before submitting the first query and may use the role value it carries - * to route reads to primary vs replica. - */ - public static final byte VERSION_2 = 2; - /** - * Maximum protocol version the ingest path advertises / accepts. Pinned to - * v1 because the v2 bump only adds an egress control frame; bumping the - * ingest wire to v2 would silently round-trip a version byte that no v1 - * server can accept. - */ - public static final byte MAX_SUPPORTED_INGEST_VERSION = VERSION_1; - /** - * Maximum protocol version supported by this client on the egress path. - * Ingest pins to {@link #MAX_SUPPORTED_INGEST_VERSION}. + * The QWP protocol version. The wire carries it in the per-frame header at + * offset 4; both directions write {@code 1} and validate {@code == 1}. The + * {@code X-QWP-Max-Version} handshake header still negotiates the version, + * leaving room for a future bump. */ - public static final byte MAX_SUPPORTED_VERSION = VERSION_2; + public static final byte VERSION = 1; private QwpConstants() { // utility class diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 5375e8ed..0947e5e1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -72,7 +72,6 @@ public class QwpTableBuffer implements QuietCloseable { private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access private int inProgressColumnCount; private int rowCount; - private int schemaId; public QwpTableBuffer(String tableName) { this(tableName, null); @@ -90,7 +89,6 @@ public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { this.columns = new ObjList<>(); this.columnNameToIndex = new LowerCaseCharSequenceIntHashMap(); this.rowCount = 0; - this.schemaId = -1; this.columnDefsCacheValid = false; } @@ -132,7 +130,6 @@ public void clear() { committedColumnCount = 0; inProgressColumnCount = 0; rowCount = 0; - schemaId = -1; columnDefsCacheValid = false; cachedColumnDefs = null; } @@ -249,13 +246,6 @@ public int getRowCount() { return rowCount; } - /** - * Returns the schema ID assigned by the sender, or -1 if not yet assigned. - */ - public int getSchemaId() { - return schemaId; - } - /** * Returns the table name. */ @@ -363,14 +353,6 @@ public void rollbackUncommittedColumns() { rebuildColumnAccessStructures(); } - /** - * Assigns a schema ID. Called by the sender when this table's schema - * is first sent to the server. - */ - public void setSchemaId(int schemaId) { - this.schemaId = schemaId; - } - private static void assertColumnType(CharSequence name, byte type, ColumnBuffer column) { if (column.type != type) { throw new LineSenderException( @@ -408,7 +390,6 @@ private ColumnBuffer createColumn(CharSequence name, byte type, boolean useNullB for (int r = 0; r < rowCount; r++) { col.addNull(); } - schemaId = -1; columnDefsCacheValid = false; } catch (Throwable t) { col.close(); @@ -458,7 +439,6 @@ private void rebuildColumnAccessStructures() { columnNameToIndex.put(col.name, i); } - schemaId = -1; columnDefsCacheValid = false; cachedColumnDefs = null; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java index 3ce0e2f5..ffbc16b5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaSymbolDictionaryTest.java @@ -64,7 +64,7 @@ public void testEdgeCase_batchWithNoSymbols() throws Exception { int batchMaxId = -1; // Can still encode with delta dict (empty delta) - int size = encoder.encodeWithDeltaDict(batch, globalDict, -1, batchMaxId, false); + int size = encoder.encodeWithDeltaDict(batch, globalDict, -1, batchMaxId); Assert.assertTrue(size > 0); // Verify flag is set @@ -261,7 +261,7 @@ public void testMultipleBatches_encodeAndDecode() throws Exception { batch1.nextRow(); int batch1MaxId = 1; - int size1 = encoder.encodeWithDeltaDict(batch1, clientDict, maxSentSymbolId, batch1MaxId, false); + int size1 = encoder.encodeWithDeltaDict(batch1, clientDict, maxSentSymbolId, batch1MaxId); Assert.assertTrue(size1 > 0); maxSentSymbolId = batch1MaxId; @@ -285,7 +285,7 @@ public void testMultipleBatches_encodeAndDecode() throws Exception { batch2.nextRow(); int batch2MaxId = 2; - int size2 = encoder.encodeWithDeltaDict(batch2, clientDict, maxSentSymbolId, batch2MaxId, false); + int size2 = encoder.encodeWithDeltaDict(batch2, clientDict, maxSentSymbolId, batch2MaxId); Assert.assertTrue(size2 > 0); // Decode batch 2 @@ -370,7 +370,7 @@ public void testMultipleTables_encodedInSameBatch() throws Exception { int confirmedMaxId = -1; int batchMaxId = 2; // AAPL(0), GOOG(1), MSFT(2) - int size = encoder.encodeWithDeltaDict(table1, globalDict, confirmedMaxId, batchMaxId, false); + int size = encoder.encodeWithDeltaDict(table1, globalDict, confirmedMaxId, batchMaxId); Assert.assertTrue(size > 0); // Verify delta section contains all 3 symbols @@ -475,7 +475,7 @@ public void testReconnection_fullDeltaAfterReconnect() throws Exception { batch.nextRow(); // Encode - should send full delta (all symbols from 0) - int size = encoder.encodeWithDeltaDict(batch, clientDict, -1, 1, false); + int size = encoder.encodeWithDeltaDict(batch, clientDict, -1, 1); Assert.assertTrue(size > 0); // Verify deltaStart is 0 diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 2ba44c1e..6889b8c3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -557,31 +557,6 @@ public void testMaxNameLengthTooSmall_fails() { .maxNameLength(10)); } - @Test - public void testMaxSchemasPerConnection() { - Sender.LineSenderBuilder builder = Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .maxSchemasPerConnection(123); - Assert.assertNotNull(builder); - } - - @Test - public void testMaxSchemasPerConnectionDoubleSet_fails() { - assertThrows("already configured", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .maxSchemasPerConnection(123) - .maxSchemasPerConnection(456)); - } - - @Test - public void testMaxSchemasPerConnectionZero_fails() { - assertThrows("must be positive", - () -> Sender.builder(Sender.Transport.WEBSOCKET) - .address(LOCALHOST) - .maxSchemasPerConnection(0)); - } - @Test public void testMinRequestThroughput_notSupportedForWebSocket() { assertThrowsAny( @@ -847,9 +822,12 @@ public void testWsConfigString_withMaxBufSize_fails() { } @Test - public void testWsConfigString_withMaxSchemasPerConnection() { - Sender.LineSenderBuilder builder = Sender.builder("ws::addr=localhost:9000;max_schemas_per_connection=1024;"); - Assert.assertNotNull(builder); + public void testWsConfigString_withMaxSchemasPerConnection_fails() { + // max_schemas_per_connection was removed with the QWP schema-reference + // mechanism. The connection string used to accept it; it must now be + // rejected as an unknown key rather than silently swallowed. + assertBadConfig("ws::addr=localhost:9000;max_schemas_per_connection=1024;", + "unknown configuration key [key=max_schemas_per_connection]"); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java index a28b8300..b30e0b3d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpInPlaceDecodeAliasingTest.java @@ -223,7 +223,7 @@ private static int writeSingleVarcharBatch(long buf, String content) { byte[] bytes = content.getBytes(StandardCharsets.UTF_8); long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); // flags p = putByte(p, (byte) 1); // table_count @@ -234,8 +234,6 @@ private static int writeSingleVarcharBatch(long buf, String content) { p = putVarint(p, 0L); // table_name_len p = putVarint(p, 1L); // row_count = 1 p = putVarint(p, 1L); // column_count = 1 - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id p = putVarint(p, 1L); // column name length p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_VARCHAR); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java index 49fa7628..d18baf05 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientFromConfigTest.java @@ -655,7 +655,6 @@ public void testIngressOnlyKeysSilentlyAcceptedOnEgress() { "max_buf_size=100m", "max_datagram_size=1400", "max_name_len=127", - "max_schemas_per_connection=65535", "multicast_ttl=1", "pass=secret", "protocol_version=2", @@ -679,7 +678,7 @@ public void testIngressOnlyKeysSilentlyAcceptedOnEgress() { assertParses("ws::addr=db:9000;" + kv + ";"); all.append(kv).append(';'); } - // All 35 keys at once -- a typical shared-config connect string. + // All ingress-only keys at once -- a typical shared-config connect string. assertParses(all.toString()); // Out-of-range / malformed values are silently consumed too -- the @@ -687,7 +686,6 @@ public void testIngressOnlyKeysSilentlyAcceptedOnEgress() { assertParses("ws::addr=db:9000;auto_flush_rows=-1;"); assertParses("ws::addr=db:9000;init_buf_size=garbage;"); assertParses("ws::addr=db:9000;reconnect_max_duration_millis=banana;"); - assertParses("ws::addr=db:9000;max_schemas_per_connection=0;"); // Empty values are well-formed and silently consumed. assertParses("ws::addr=db:9000;auto_flush=;"); @@ -773,6 +771,16 @@ public void testMaxBatchRowsZeroRejected() { ); } + @Test + public void testMaxSchemasPerConnectionRejected() { + // max_schemas_per_connection was removed with the QWP schema-reference + // mechanism. It used to be silently accepted on egress (an ingress-only + // key); now the parser must surface it as an unknown-key error rather + // than swallowing it, matching the ingress Sender's rejection. + assertReject("ws::addr=db:9000;max_schemas_per_connection=1024;", + "unknown configuration key: max_schemas_per_connection"); + } + @Test public void testMinimalWsConfigAccepted() { assertParses("ws::addr=localhost:9000;"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUnitTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUnitTest.java index 4456949a..0c76f170 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUnitTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientUnitTest.java @@ -245,7 +245,7 @@ public void testFromConfigReturnsUsableClient() { @Test public void testGetServerInfoNullableBeforeConnect() { // The accessor is documented as returning null before connect(); a - // QwpServerInfo instance materialises only after the v2 SERVER_INFO + // QwpServerInfo instance materialises only after the SERVER_INFO // frame is decoded during connect(). try (QwpQueryClient c = QwpQueryClient.newPlainText("localhost", 9000)) { QwpServerInfo info = c.getServerInfo(); @@ -255,10 +255,10 @@ public void testGetServerInfoNullableBeforeConnect() { @Test public void testQwpMaxVersionConstant() { - // The negotiated-version cap is the v2 protocol; pin it so a future - // version bump that adds a new control frame must explicitly update - // both the constant and any callers asserting the cap. - Assert.assertEquals(2, QwpQueryClient.QWP_MAX_VERSION); + // QWP has collapsed to a single protocol version; pin it so a future + // version bump must explicitly update both the constant and any callers + // asserting the cap. + Assert.assertEquals(1, QwpQueryClient.QWP_MAX_VERSION); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientWalkTrackerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientWalkTrackerTest.java index 12f47d0d..acf434ed 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientWalkTrackerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpQueryClientWalkTrackerTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.http.client.HttpClientException; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; +import io.questdb.client.cutlass.qwp.client.QwpEgressMsgKind; import io.questdb.client.cutlass.qwp.client.QwpQueryClient; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; @@ -57,7 +58,7 @@ * from "all unreachable" ({@link HttpClientException}).

    • * *

    - * SERVER_INFO-driven role checks (target=primary against a v2 server + * SERVER_INFO-driven role checks (target=primary against a server * advertising REPLICA via the SERVER_INFO frame) belong to the parent * QuestDB egress integration suite -- TestWebSocketServer here only * covers the upgrade-time {@code X-QuestDB-Role} header path which is @@ -80,6 +81,7 @@ public void testWalk_404NotFoundIsTransportNotTerminal() throws Exception { TestWebSocketServer notFound = new TestWebSocketServer(port404, NOOP_HANDLER); notFound.setRejectWithStatus(404, "Not Found"); TestWebSocketServer ok = new TestWebSocketServer(portOk, NOOP_HANDLER); + ok.setSendServerInfo(true); try { notFound.start(); ok.start(); @@ -106,6 +108,7 @@ public void testWalk_426UpgradeRequiredIsTransportNotTerminal() throws Exception TestWebSocketServer rejecting = new TestWebSocketServer(port426, NOOP_HANDLER); rejecting.setRejectWithStatus(426, "Upgrade Required"); TestWebSocketServer ok = new TestWebSocketServer(portOk, NOOP_HANDLER); + ok.setSendServerInfo(true); try { rejecting.start(); ok.start(); @@ -258,14 +261,16 @@ public void testWalk_FallThroughResetRehabilitatesPriorTopologyRejects() throws // see every host TopologyReject (priority 5) and walk past the // now-healthy A only to fail. // - // target=any keeps this v1-friendly: the spec defines - // target=primary as requiring v2 SERVER_INFO, which the test - // server doesn't emit. A separate integration test in the - // parent QuestDB repo covers the SERVER_INFO path. + // target=any so the rehabilitated host A binds regardless of the + // role it advertises in its SERVER_INFO frame -- this test + // exercises the fall-through reset, not the role filter. The + // SERVER_INFO-driven role filter (target=primary/replica) is + // covered by a separate integration test in the parent QuestDB repo. int portA = TestPorts.findUnusedPort(); int portB = TestPorts.findUnusedPort(); TestWebSocketServer a = new TestWebSocketServer(portA, NOOP_HANDLER); a.setRejectWithRole("REPLICA"); + a.setSendServerInfo(true); TestWebSocketServer b = new TestWebSocketServer(portB, NOOP_HANDLER); b.setRejectWithRole("REPLICA"); try { @@ -305,6 +310,7 @@ public void testWalk_FirstReachablePrimaryWins() throws Exception { TestWebSocketServer rep = new TestWebSocketServer(portReplica, NOOP_HANDLER); rep.setRejectWithRole("REPLICA"); TestWebSocketServer prim = new TestWebSocketServer(portPrimary, NOOP_HANDLER, false, "PRIMARY"); + prim.setSendServerInfo(true); try { rep.start(); prim.start(); @@ -322,6 +328,110 @@ public void testWalk_FirstReachablePrimaryWins() throws Exception { } } + @Test + public void testWalk_ServerInfoReplicaRejectedForTargetPrimary() throws Exception { + // A node that completes a clean 101 upgrade and then advertises REPLICA + // only in its SERVER_INFO frame (not via the 421 X-QuestDB-Role header + // path the other role tests use) must be rejected by the role filter + // when target=primary. Pins matchesTarget(info.getRole(), target) where + // info is the decoded SERVER_INFO -- the branch that outlived the + // v1-mismatch removal. A clean 101 ignores the upgrade-time role header, + // so the rejection here is driven purely by the SERVER_INFO role. + int port = TestPorts.findUnusedPort(); + TestWebSocketServer replica = new TestWebSocketServer(port, NOOP_HANDLER); + replica.setAdvertisedRole("REPLICA"); + replica.setSendServerInfo(true); + try { + replica.start(); + Assert.assertTrue(replica.awaitStart(5, TimeUnit.SECONDS)); + + try (QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=localhost:" + port + ";target=primary;auth_timeout_ms=2000;")) { + try { + client.connect(); + Assert.fail("expected QwpRoleMismatchException for a REPLICA SERVER_INFO under target=primary"); + } catch (QwpRoleMismatchException expected) { + Assert.assertFalse("a role mismatch must not leave the client connected", client.isConnected()); + Assert.assertEquals("primary", expected.getTargetRole()); + // The rejected role is taken from the decoded SERVER_INFO frame. + Assert.assertNotNull("observed SERVER_INFO must be attached", expected.getLastObserved()); + Assert.assertEquals(QwpEgressMsgKind.ROLE_REPLICA, expected.getLastObserved().getRole()); + Assert.assertTrue("message must mention target=primary: " + expected.getMessage(), + expected.getMessage().contains("target=primary")); + } + } + } finally { + replica.close(); + } + } + + @Test + public void testWalk_ServerInfoRoleFilterSkipsReplicaBindsPrimary() throws Exception { + // Both endpoints complete a clean 101 and advertise their role only via + // the SERVER_INFO frame. With target=primary the walk must skip the + // REPLICA endpoint -- a SERVER_INFO role mismatch is a skip, not a + // terminal failure -- and bind the PRIMARY one. Exercises the + // walk-continues side of matchesTarget(info.getRole(), target). + int portReplica = TestPorts.findUnusedPort(); + int portPrimary = TestPorts.findUnusedPort(); + TestWebSocketServer replica = new TestWebSocketServer(portReplica, NOOP_HANDLER); + replica.setAdvertisedRole("REPLICA"); + replica.setSendServerInfo(true); + TestWebSocketServer primary = new TestWebSocketServer(portPrimary, NOOP_HANDLER); + primary.setAdvertisedRole("PRIMARY"); + primary.setSendServerInfo(true); + try { + replica.start(); + primary.start(); + Assert.assertTrue(replica.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue(primary.awaitStart(5, TimeUnit.SECONDS)); + + try (QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=localhost:" + portReplica + ",localhost:" + portPrimary + + ";target=primary;auth_timeout_ms=2000;")) { + client.connect(); + Assert.assertTrue("client must skip the REPLICA and bind the PRIMARY", client.isConnected()); + Assert.assertNotNull("bound connection must carry SERVER_INFO", client.getServerInfo()); + Assert.assertEquals(QwpEgressMsgKind.ROLE_PRIMARY, client.getServerInfo().getRole()); + } + } finally { + replica.close(); + primary.close(); + } + } + + @Test(timeout = 15_000) + public void testWalk_ServerInfoTimeoutIsTransportNotTerminal() throws Exception { + // A node that completes the 101 upgrade but never sends the mandatory + // SERVER_INFO frame must be treated as a transport error so the walk + // continues, not a terminal failure. receiveServerInfoSync() now runs on + // every connect, so a silent post-upgrade peer would otherwise stall the + // client until the server-info timeout; bound it short here and verify + // the walk falls through to a healthy node. + int portSilent = TestPorts.findUnusedPort(); + int portOk = TestPorts.findUnusedPort(); + TestWebSocketServer silent = new TestWebSocketServer(portSilent, NOOP_HANDLER); + // sendServerInfo left off: the 101 upgrade succeeds, then the node stays silent. + TestWebSocketServer ok = new TestWebSocketServer(portOk, NOOP_HANDLER); + ok.setSendServerInfo(true); + try { + silent.start(); + ok.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue(ok.awaitStart(5, TimeUnit.SECONDS)); + + try (QwpQueryClient client = QwpQueryClient.fromConfig( + "ws::addr=localhost:" + portSilent + ",localhost:" + portOk + ";auth_timeout_ms=2000;")) { + client.withServerInfoTimeout(300); + client.connect(); + Assert.assertTrue("client must walk past the SERVER_INFO-silent node", client.isConnected()); + } + } finally { + silent.close(); + ok.close(); + } + } + @Test public void testWalk_TransportFailureContinuesWalk() throws Exception { // First port has no server (TCP refused); second is reachable. @@ -330,6 +440,7 @@ public void testWalk_TransportFailureContinuesWalk() throws Exception { int portDead = TestPorts.findUnusedPort(); int portOk = TestPorts.findUnusedPort(); try (TestWebSocketServer ok = new TestWebSocketServer(portOk, NOOP_HANDLER)) { + ok.setSendServerInfo(true); ok.start(); Assert.assertTrue(ok.awaitStart(5, TimeUnit.SECONDS)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderColumnTypesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderColumnTypesTest.java index eed16251..a6872ed0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderColumnTypesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderColumnTypesTest.java @@ -47,7 +47,7 @@ * pins the rejection paths for malformed frames, this file pins the * happy-path decoders for every wire type plus the connection-scoped state * machines (delta SYMBOL dict, FLAG_GORILLA TIMESTAMP, FLAG_DELTA_SYMBOL_DICT, - * CACHE_RESET, SCHEMA_MODE_REFERENCE). Each test crafts a single-column or + * CACHE_RESET, batch_seq schema reuse). Each test crafts a single-column or * minimal multi-column RESULT_BATCH in native memory and verifies the values * surface correctly through {@link io.questdb.client.cutlass.qwp.client.QwpColumnBatch} * accessors, mirroring the production end-to-end flow without a server. @@ -137,8 +137,6 @@ public void testCacheResetDictMaskClearsConnectionDict() throws Exception { p = putVarint(p, 0L); p = putVarint(p, 1L); p = putVarint(p, 1L); - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); p = putVarint(p, 1L); p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_SYMBOL); @@ -157,7 +155,7 @@ public void testCacheResetDictMaskClearsConnectionDict() throws Exception { p = writeHeader(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); p = putByte(p, (byte) 0x11); p = putLong(p, 1L); - p = putVarint(p, 1L); + p = putVarint(p, 0L); // batch_seq = 0 (independent batch) p = putVarint(p, 0L); // deltaStart = 0 after reset p = putVarint(p, 1L); byte[] bbb = "bbb".getBytes(StandardCharsets.UTF_8); @@ -166,8 +164,6 @@ public void testCacheResetDictMaskClearsConnectionDict() throws Exception { p = putVarint(p, 0L); p = putVarint(p, 1L); p = putVarint(p, 1L); - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); p = putVarint(p, 1L); p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_SYMBOL); @@ -185,52 +181,10 @@ public void testCacheResetDictMaskClearsConnectionDict() throws Exception { }); } - @Test - public void testCacheResetSchemasMaskClearsRegistry() throws Exception { - TestUtils.assertMemoryLeak(() -> { - QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); - QwpBatchBuffer buffer = new QwpBatchBuffer(256); - long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); - try { - // Register schema_id=3 with one INT column, then CACHE_RESET schemas, - // then attempt to reference schema_id=3 -- decoder must reject. - long p = startSingleColumnFrame(staging, "n", QwpConstants.TYPE_INT, 1, 3L); - p = putByte(p, (byte) 0); - p = putInt(p, 1); - buffer.copyFromPayload(staging, (int) (p - staging)); - decoder.decode(buffer); - - decoder.applyCacheReset(QwpEgressMsgKind.RESET_MASK_SCHEMAS); - - p = staging; - p = writeHeader(p, (byte) 0); - p = putByte(p, (byte) 0x11); - p = putLong(p, 1L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 1L); - p = putByte(p, QwpConstants.SCHEMA_MODE_REFERENCE); - p = putVarint(p, 3L); - buffer.copyFromPayload(staging, (int) (p - staging)); - try { - decoder.decode(buffer); - Assert.fail("schema ref must be rejected after CACHE_RESET schemas mask"); - } catch (QwpDecodeException expected) { - Assert.assertTrue(expected.getMessage().contains("not registered")); - } - } finally { - Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); - buffer.close(); - decoder.close(); - } - }); - } - @Test public void testCharColumnDecodes() throws Exception { TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { - char[] values = {'a', 'Z', '0', '', '￿'}; + char[] values = {'a', 'Z', '0', '\u0000', '\uffff'}; long p = startSingleColumnFrame(staging, "c", QwpConstants.TYPE_CHAR, values.length); p = putByte(p, (byte) 0); for (char v : values) p = putShort(p, (short) v); @@ -292,8 +246,6 @@ public void testEmptyBatchDecodes() throws Exception { p = putVarint(p, 0L); // table_name_len p = putVarint(p, 0L); // row_count = 0 p = putVarint(p, 0L); // column_count = 0 - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id buffer.copyFromPayload(staging, (int) (p - staging)); decoder.decode(buffer); @@ -302,6 +254,39 @@ public void testEmptyBatchDecodes() throws Exception { })); } + @Test + public void testEmptyResultSetCarriesSchema() throws Exception { + TestUtils.assertMemoryLeak(() -> withDecoder(128, (decoder, buffer, staging) -> { + // Realistic empty result: batch_seq == 0, row_count == 0, but the schema + // is present (column_count > 0 with descriptors, no row bodies). The + // server always ships the schema on batch 0 even when the result is empty. + long p = staging; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); // RESULT_BATCH + p = putLong(p, 1L); // request_id + p = putVarint(p, 0L); // batch_seq = 0 + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 0L); // row_count = 0 + p = putVarint(p, 2L); // column_count = 2 + p = putVarint(p, 1L); + p = putByte(p, (byte) 'a'); + p = putByte(p, QwpConstants.TYPE_INT); + p = putVarint(p, 1L); + p = putByte(p, (byte) 'b'); + p = putByte(p, QwpConstants.TYPE_LONG); + // Each column still carries its null-flag byte even at row_count == 0 + // (no nulls -> 0x00); the decoder reads one per column. No row bodies. + p = putByte(p, (byte) 0); // col "a" null_flag + p = putByte(p, (byte) 0); // col "b" null_flag + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(0, batchOf(buffer).getRowCount()); + Assert.assertEquals(2, batchOf(buffer).getColumnCount()); + Assert.assertEquals("a", batchOf(buffer).getColumnName(0)); + Assert.assertEquals("b", batchOf(buffer).getColumnName(1)); + })); + } + @Test public void testGeohashColumnReadsPrecisionVarint() throws Exception { TestUtils.assertMemoryLeak(() -> withDecoder(256, (decoder, buffer, staging) -> { @@ -434,39 +419,45 @@ public void testNullBitmapPopulatesNonNullIdx() throws Exception { } @Test - public void testSchemaReferenceModeReusesEarlierSchema() throws Exception { + public void testResetQuerySchemaRejectsContinuationFromPriorQuery() throws Exception { TestUtils.assertMemoryLeak(() -> { QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); - QwpBatchBuffer buffer = new QwpBatchBuffer(512); - long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + QwpBatchBuffer buffer = new QwpBatchBuffer(256); + long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); try { - // Batch 1 registers schema_id=7 with one INT column. - long p = startSingleColumnFrame(staging, "n", QwpConstants.TYPE_INT, 1, /*schemaId=*/ 7L); + // Query A delivers its schema on batch_seq == 0. + long p = startSingleColumnFrame(staging, "n", QwpConstants.TYPE_INT, 1); p = putByte(p, (byte) 0); p = putInt(p, 99); buffer.copyFromPayload(staging, (int) (p - staging)); decoder.decode(buffer); Assert.assertEquals(99, batchOf(buffer).getIntValue(0, 0)); - // Batch 2 references schema_id=7 (REFERENCE mode) -- no schema bytes inline. + // The IoThread calls resetQuerySchema() when the next query starts. + // After that, query A's schema must no longer satisfy a continuation: + // a batch_seq > 0 arriving before the new query's batch_seq == 0 must + // be rejected, not bound to the prior query's schema. + decoder.resetQuerySchema(); + p = staging; - p = writeHeader(p, /*flags=*/ (byte) 0); + p = writeHeader(p, (byte) 0); p = putByte(p, (byte) 0x11); p = putLong(p, 1L); - p = putVarint(p, 1L); - p = putVarint(p, 0L); + p = putVarint(p, 1L); // batch_seq = 1 (continuation) + p = putVarint(p, 0L); // table_name_len p = putVarint(p, 1L); // row_count - p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_REFERENCE); - p = putVarint(p, 7L); // schema_id p = putByte(p, (byte) 0); // null_flag p = putInt(p, 1234); // value buffer.copyFromPayload(staging, (int) (p - staging)); - decoder.decode(buffer); - Assert.assertEquals(1234, batchOf(buffer).getIntValue(0, 0)); - Assert.assertEquals("n", batchOf(buffer).getColumnName(0)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject a continuation after resetQuerySchema()"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error mentions the missing schema batch: " + expected.getMessage(), + expected.getMessage().contains("batch_seq")); + } } finally { - Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); buffer.close(); decoder.close(); } @@ -474,29 +465,29 @@ public void testSchemaReferenceModeReusesEarlierSchema() throws Exception { } @Test - public void testSchemaReferenceModeUnregisteredRejected() throws Exception { + public void testSchemaMissingOnContinuationRejected() throws Exception { TestUtils.assertMemoryLeak(() -> { QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); QwpBatchBuffer buffer = new QwpBatchBuffer(256); long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); try { + // A continuation batch (batch_seq > 0) arriving before any + // schema-bearing batch_seq == 0 must be rejected, not bound to a + // stale schema. long p = staging; p = writeHeader(p, (byte) 0); p = putByte(p, (byte) 0x11); p = putLong(p, 1L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putByte(p, QwpConstants.SCHEMA_MODE_REFERENCE); - p = putVarint(p, 99L); // never registered + p = putVarint(p, 1L); // batch_seq = 1 (continuation) + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 0L); // row_count buffer.copyFromPayload(staging, (int) (p - staging)); try { decoder.decode(buffer); - Assert.fail("decoder must reject unregistered schema reference"); + Assert.fail("decoder must reject a continuation batch with no prior schema"); } catch (QwpDecodeException expected) { - Assert.assertTrue("error mentions schema id: " + expected.getMessage(), - expected.getMessage().contains("not registered")); + Assert.assertTrue("error mentions the missing schema batch: " + expected.getMessage(), + expected.getMessage().contains("batch_seq")); } } finally { Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); @@ -506,6 +497,170 @@ public void testSchemaReferenceModeUnregisteredRejected() throws Exception { }); } + @Test + public void testSchemaReusedAcrossContinuationBatches() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // Batch 1 (batch_seq == 0) carries the schema: one INT column "n". + long p = startSingleColumnFrame(staging, "n", QwpConstants.TYPE_INT, 1); + p = putByte(p, (byte) 0); + p = putInt(p, 99); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(99, batchOf(buffer).getIntValue(0, 0)); + + // Batch 2 (batch_seq == 1) carries rows only and reuses the schema + // parsed from batch 0 -- no column_count / column descriptors inline. + p = staging; + p = writeHeader(p, /*flags=*/ (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 1L); // batch_seq = 1 (continuation) + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 1L); // row_count + p = putByte(p, (byte) 0); // null_flag + p = putInt(p, 1234); // value + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(1234, batchOf(buffer).getIntValue(0, 0)); + Assert.assertEquals("n", batchOf(buffer).getColumnName(0)); + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testSchemaSlotsReusedAcrossQueriesWithDifferentTypes() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // Query A: one INT column "i". Seeds pooled schema slot 0 with + // name "i" / wire-type INT. + long p = startSingleColumnFrame(staging, "i", QwpConstants.TYPE_INT, 1); + p = putByte(p, (byte) 0); + p = putInt(p, 5); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(5, batchOf(buffer).getIntValue(0, 0)); + + decoder.resetQuerySchema(); + + // Query B batch_seq == 0: two columns LONG "p", INT "q". Slot 0 is + // reused from query A and must be fully overwritten (name "p", + // wire-type LONG), not retain A's "i"/INT; slot 1 is freshly grown. + p = staging; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq = 0 + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 1L); // row_count + p = putVarint(p, 2L); // column_count + p = putVarint(p, 1L); // col 0 name length + p = putByte(p, (byte) 'p'); + p = putByte(p, QwpConstants.TYPE_LONG); + p = putVarint(p, 1L); // col 1 name length + p = putByte(p, (byte) 'q'); + p = putByte(p, QwpConstants.TYPE_INT); + p = putByte(p, (byte) 0); // col 0 null_flag + p = putLong(p, 9_876_543_210L); // col 0 value (LONG) + p = putByte(p, (byte) 0); // col 1 null_flag + p = putInt(p, 42); // col 1 value (INT) + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + + Assert.assertEquals(2, batchOf(buffer).getColumnCount()); + Assert.assertEquals("p", batchOf(buffer).getColumnName(0)); + Assert.assertEquals(QwpConstants.TYPE_LONG, batchOf(buffer).getColumnWireType(0)); + Assert.assertEquals(9_876_543_210L, batchOf(buffer).getLongValue(0, 0)); + Assert.assertEquals("q", batchOf(buffer).getColumnName(1)); + Assert.assertEquals(42, batchOf(buffer).getIntValue(1, 0)); + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + + @Test + public void testSchemaSwitchesAcrossQueriesWithDifferentColumnCount() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(512); + long staging = Unsafe.malloc(512, MemoryTag.NATIVE_DEFAULT); + try { + // Query A batch_seq == 0: two INT columns "a", "b" -- grows the + // pooled schema to 2 slots. + long p = staging; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 0L); // batch_seq = 0 + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 1L); // row_count + p = putVarint(p, 2L); // column_count + p = putVarint(p, 1L); + p = putByte(p, (byte) 'a'); + p = putByte(p, QwpConstants.TYPE_INT); + p = putVarint(p, 1L); + p = putByte(p, (byte) 'b'); + p = putByte(p, QwpConstants.TYPE_INT); + p = putByte(p, (byte) 0); // col a null_flag + p = putInt(p, 10); + p = putByte(p, (byte) 0); // col b null_flag + p = putInt(p, 20); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(2, batchOf(buffer).getColumnCount()); + Assert.assertEquals(10, batchOf(buffer).getIntValue(0, 0)); + Assert.assertEquals(20, batchOf(buffer).getIntValue(1, 0)); + + decoder.resetQuerySchema(); + + // Query B batch_seq == 0: a single INT column "x" -- shrinks the + // pooled schema back to 1 slot. + p = startSingleColumnFrame(staging, "x", QwpConstants.TYPE_INT, 1); + p = putByte(p, (byte) 0); + p = putInt(p, 77); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(1, batchOf(buffer).getColumnCount()); + Assert.assertEquals(77, batchOf(buffer).getIntValue(0, 0)); + + // Query B continuation (batch_seq == 1): rows only. It binds to + // query B's 1-column schema -- columnCount derives from + // querySchema.size(), which must have shrunk to 1 (not stale at 2). + p = staging; + p = writeHeader(p, (byte) 0); + p = putByte(p, (byte) 0x11); + p = putLong(p, 1L); + p = putVarint(p, 1L); // batch_seq = 1 + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 1L); // row_count + p = putByte(p, (byte) 0); // null_flag + p = putInt(p, 88); + buffer.copyFromPayload(staging, (int) (p - staging)); + decoder.decode(buffer); + Assert.assertEquals(1, batchOf(buffer).getColumnCount()); + Assert.assertEquals("x", batchOf(buffer).getColumnName(0)); + Assert.assertEquals(88, batchOf(buffer).getIntValue(0, 0)); + } finally { + Unsafe.free(staging, 512, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + @Test public void testSymbolColumnDeltaModeAccumulatesAcrossBatches() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -532,8 +687,6 @@ public void testSymbolColumnDeltaModeAccumulatesAcrossBatches() throws Exception p = putVarint(p, 0L); // table_name_len p = putVarint(p, 2L); // row_count p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id p = putVarint(p, 1L); // colName length p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_SYMBOL); @@ -560,12 +713,6 @@ public void testSymbolColumnDeltaModeAccumulatesAcrossBatches() throws Exception for (byte b : gamma) p = putByte(p, b); p = putVarint(p, 0L); p = putVarint(p, 1L); - p = putVarint(p, 1L); - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); - p = putVarint(p, 1L); - p = putByte(p, (byte) 's'); - p = putByte(p, QwpConstants.TYPE_SYMBOL); p = putByte(p, (byte) 0); p = putVarint(p, 2L); // row 0 -> "gamma" (id=2) @@ -591,8 +738,6 @@ public void testSymbolColumnNonDeltaInlineDictionary() throws Exception { p = putVarint(p, 0L); p = putVarint(p, 3L); // row_count p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); p = putVarint(p, 1L); p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_SYMBOL); @@ -632,8 +777,6 @@ public void testSymbolIndexOutOfRangeRejected() throws Exception { p = putVarint(p, 0L); p = putVarint(p, 1L); p = putVarint(p, 1L); - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); p = putVarint(p, 1L); p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_SYMBOL); @@ -689,8 +832,6 @@ public void testTimestampColumnGorillaEncoding() throws Exception { p = putVarint(p, 0L); p = putVarint(p, values.length); p = putVarint(p, 1L); - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); p = putVarint(p, 1L); p = putByte(p, (byte) 't'); p = putByte(p, QwpConstants.TYPE_TIMESTAMP); @@ -745,8 +886,6 @@ public void testTimestampUnknownEncodingByteRejected() throws Exception { p = putVarint(p, 0L); p = putVarint(p, 3L); p = putVarint(p, 1L); - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); p = putVarint(p, 1L); p = putByte(p, (byte) 't'); p = putByte(p, QwpConstants.TYPE_TIMESTAMP); @@ -825,39 +964,6 @@ public void testTruncatedFixedColumnRejected() throws Exception { }); } - @Test - public void testUnknownSchemaModeRejected() throws Exception { - TestUtils.assertMemoryLeak(() -> { - QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); - QwpBatchBuffer buffer = new QwpBatchBuffer(256); - long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); - try { - long p = staging; - p = writeHeader(p, (byte) 0); - p = putByte(p, (byte) 0x11); - p = putLong(p, 1L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putByte(p, (byte) 0x42); // unknown schema_mode - p = putVarint(p, 0L); - buffer.copyFromPayload(staging, (int) (p - staging)); - try { - decoder.decode(buffer); - Assert.fail("decoder must reject unknown schema mode"); - } catch (QwpDecodeException expected) { - Assert.assertTrue("error mentions schema mode: " + expected.getMessage(), - expected.getMessage().contains("schema mode")); - } - } finally { - Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); - buffer.close(); - decoder.close(); - } - }); - } - @Test public void testUnsupportedVersionRejected() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -1043,10 +1149,6 @@ private static long putVarint(long p, long value) { * returned position points at where the column body starts. */ private static long startSingleColumnFrame(long buf, String colName, byte wireType, int rowCount) { - return startSingleColumnFrame(buf, colName, wireType, rowCount, 0L); - } - - private static long startSingleColumnFrame(long buf, String colName, byte wireType, int rowCount, long schemaId) { long p = buf; p = writeHeader(p, (byte) 0); p = putByte(p, (byte) 0x11); @@ -1055,8 +1157,6 @@ private static long startSingleColumnFrame(long buf, String colName, byte wireTy p = putVarint(p, 0L); // table_name_len p = putVarint(p, rowCount); p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, schemaId); byte[] nameBytes = colName.getBytes(StandardCharsets.UTF_8); p = putVarint(p, nameBytes.length); for (byte b : nameBytes) p = putByte(p, b); @@ -1085,9 +1185,9 @@ private static long writeHeader(long p, byte flags) { // the decoder see flags == 0 and silently take the non-delta / non-gorilla // / non-zstd path, leaving the delta/gorilla payload bytes at the position // the table-block parser then advances over -- producing inscrutable - // "unknown schema mode 0x..." errors mid-frame. + // table-block parse errors (bad varint, out-of-range column_count) mid-frame. p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, flags); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java index 41f8a2b6..073570b1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpResultBatchDecoderHardeningTest.java @@ -45,8 +45,8 @@ * Hardening tests for {@link QwpResultBatchDecoder} against malformed RESULT_BATCH * frames from a hostile or buggy server. Each test crafts a wire payload directly * in native memory and asserts that the decoder rejects it cleanly with a - * {@link QwpDecodeException} rather than reading out of bounds, growing the - * schema registry without bound, or returning negative offsets that propagate + * {@link QwpDecodeException} rather than reading out of bounds, accepting an + * out-of-range column_count, or returning negative offsets that propagate * into accessors. */ public class QwpResultBatchDecoderHardeningTest { @@ -110,6 +110,50 @@ public void testArrayValidDimensionsAreAccepted() throws Exception { }); } + /** + * Regression for the surviving table-block guard: a {@code column_count} + * above {@link QwpConstants#MAX_COLUMNS_PER_TABLE} on the schema-bearing + * batch_seq == 0 must be rejected before the decoder tries to parse that + * many column descriptors. This guard moved into the batch_seq == 0 branch + * when schema-reference mode was removed; the old schema_id range check that + * used to sit beside it is gone, so this is the only remaining bound here. + */ + @Test + public void testColumnCountOutOfRangeIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(64); + long staging = Unsafe.malloc(64, MemoryTag.NATIVE_DEFAULT); + try { + long p = staging; + p = putInt(p, QwpConstants.MAGIC_MESSAGE); + p = putByte(p, QwpConstants.VERSION); + p = putByte(p, (byte) 0); // flags + p = putByte(p, (byte) 0); // msg_kind in header (unused) + p = putByte(p, (byte) 1); // table_count + p = putInt(p, 0); // payload_length (unused) + p = putByte(p, (byte) 0x11); // RESULT_BATCH + p = putLong(p, 1L); // request_id + p = putVarint(p, 0L); // batch_seq = 0 (schema-bearing) + p = putVarint(p, 0L); // table_name_len + p = putVarint(p, 0L); // row_count + p = putVarint(p, 1_000_000_000L); // column_count: far above the cap + buffer.copyFromPayload(staging, (int) (p - staging)); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject an out-of-range column_count"); + } catch (QwpDecodeException expected) { + Assert.assertTrue("error must reference column_count: " + expected.getMessage(), + expected.getMessage().contains("column_count out of range")); + } + } finally { + Unsafe.free(staging, 64, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + @Test public void testUnsupportedVersionThrowsProtocolVersionException() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -117,7 +161,7 @@ public void testUnsupportedVersionThrowsProtocolVersionException() throws Except QwpBatchBuffer buffer = new QwpBatchBuffer(128); long staging = Unsafe.malloc(128, MemoryTag.NATIVE_DEFAULT); try { - int len = writeMinimalResultBatch(staging, 0L); + int len = writeMinimalResultBatch(staging); Unsafe.getUnsafe().putByte(staging + 4, (byte) 99); buffer.copyFromPayload(staging, len); try { @@ -137,6 +181,39 @@ public void testUnsupportedVersionThrowsProtocolVersionException() throws Except }); } + /** + * The protocol collapsed to a single version. Version 2 -- accepted by the + * pre-collapse range check ({@code >= VERSION_1 && <= MAX_SUPPORTED_VERSION}) + * -- must now be rejected by the flattened {@code != VERSION} check, the same + * as any other non-1 version byte. Pins the boundary that actually changed. + */ + @Test + public void testVersionTwoIsRejected() throws Exception { + TestUtils.assertMemoryLeak(() -> { + QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); + QwpBatchBuffer buffer = new QwpBatchBuffer(128); + long staging = Unsafe.malloc(128, MemoryTag.NATIVE_DEFAULT); + try { + int len = writeMinimalResultBatch(staging); + Unsafe.getUnsafe().putByte(staging + 4, (byte) 2); + buffer.copyFromPayload(staging, len); + try { + decoder.decode(buffer); + Assert.fail("decoder must reject protocol version 2"); + } catch (QwpProtocolVersionException expected) { + Assert.assertTrue("error must reference unsupported version: " + expected.getMessage(), + expected.getMessage().contains("unsupported version")); + Assert.assertTrue("must extend QwpDecodeException", + expected instanceof QwpDecodeException); + } + } finally { + Unsafe.free(staging, 128, MemoryTag.NATIVE_DEFAULT); + buffer.close(); + decoder.close(); + } + }); + } + /** * Regression: a delta SYMBOL dict entry whose length exceeds * {@link Integer#MAX_VALUE} must be rejected. Prior to the fix, the @@ -314,73 +391,6 @@ public void testGeohashPrecisionBelowMinIsRejected() throws Exception { }); } - /** - * Regression for C5: a server-supplied {@code schema_id} above the per-connection - * cap must be rejected. Without the fix, {@code ensureSchemaSlot} would happily - * append nulls until OOM (or AIOOBE for negative ids cast from a high varint). - */ - @Test - public void testHugeSchemaIdIsRejected() throws Exception { - TestUtils.assertMemoryLeak(() -> { - QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); - QwpBatchBuffer buffer = new QwpBatchBuffer(256); - long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); - try { - // schema_id = 1_000_000_000, well above the 65_535 cap. - int len = writeMinimalResultBatch(staging, /*schemaId=*/ 1_000_000_000L); - buffer.copyFromPayload(staging, len); - try { - decoder.decode(buffer); - Assert.fail("decoder must reject huge schema_id"); - } catch (QwpDecodeException expected) { - Assert.assertTrue("error message should mention schema_id: " + expected.getMessage(), - expected.getMessage().contains("schema_id")); - } - } finally { - Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); - buffer.close(); - decoder.close(); - } - }); - } - - /** - * Regression for C5: a varint that long-to-int casts to a negative value - * (a hostile high varint with the sign bit set after the cast) must be - * rejected, not silently passed to {@code getQuick(negativeIndex)}. - */ - @Test - public void testNegativeSchemaIdIsRejected() throws Exception { - TestUtils.assertMemoryLeak(() -> { - QwpResultBatchDecoder decoder = new QwpResultBatchDecoder(); - QwpBatchBuffer buffer = new QwpBatchBuffer(256); - long staging = Unsafe.malloc(256, MemoryTag.NATIVE_DEFAULT); - try { - // 5-byte varint encoding 0x80000000 (which casts to Integer.MIN_VALUE). - // varint bytes for 0x80000000: - // value bits 7..0: 0x00 -> byte: 0x80 (continuation) - // value bits 14..8: 0x00 -> byte: 0x80 - // value bits 21..15:0x00 -> byte: 0x80 - // value bits 28..22:0x00 -> byte: 0x80 - // value bits 35..29:0x08 -> byte: 0x08 (no continuation) - int len = writeMinimalResultBatchWithRawSchemaIdVarint( - staging, new byte[]{(byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x08}); - buffer.copyFromPayload(staging, len); - try { - decoder.decode(buffer); - Assert.fail("decoder must reject huge/negative schema_id"); - } catch (QwpDecodeException expected) { - Assert.assertTrue("error message should mention schema_id: " + expected.getMessage(), - expected.getMessage().contains("schema_id")); - } - } finally { - Unsafe.free(staging, 256, MemoryTag.NATIVE_DEFAULT); - buffer.close(); - decoder.close(); - } - }); - } - /** * Regression for C3: a hostile or buggy server can send a QUERY_ERROR frame * that claims a 65535-byte message but supplies a tiny payload. Without the @@ -822,9 +832,8 @@ private static void closePool(QwpEgressIoThread io) { * table-block: * name_len (varint), name bytes (none) * row_count (varint) - * column_count (varint) - * schema_mode (1 byte) + schema_id (varint) - * [if FULL] per column: name_len varint, name bytes, wire_type byte + * column_count (varint, batch_seq == 0 only) + * per column (batch_seq == 0 only): name_len varint, name bytes, wire_type byte * per column: null_flag byte (+optional bitmap), then column body * */ @@ -863,7 +872,7 @@ private static long putVarint(long p, long value) { private static int writeArrayResultBatchWithDims(long buf, int[] dims) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -874,8 +883,6 @@ private static int writeArrayResultBatchWithDims(long buf, int[] dims) { p = putVarint(p, 0L); // table_name_len p = putVarint(p, 1L); // row_count = 1 p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id p = putVarint(p, 1L); // column name length p = putByte(p, (byte) 'a'); p = putByte(p, QwpConstants.TYPE_DOUBLE_ARRAY); @@ -901,7 +908,7 @@ private static int writeArrayResultBatchWithDims(long buf, int[] dims) { private static int writeDeltaSymbolDictFrame(long buf, byte[] entryLenVarint) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); // Flags byte lives at HEADER_OFFSET_FLAGS (=5). Existing helpers with // flags=0 don't care about the exact slot; a delta-mode frame does. p = putByte(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); @@ -929,7 +936,7 @@ private static int writeDeltaSymbolDictFrame(long buf, byte[] entryLenVarint) { private static int writeDeltaSymbolDictFrameWithRange(long buf, long deltaStart, long deltaCount) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, QwpConstants.FLAG_DELTA_SYMBOL_DICT); // flags at HEADER_OFFSET_FLAGS p = putByte(p, (byte) 0); // reserved p = putByte(p, (byte) 1); // table_count @@ -951,7 +958,7 @@ private static int writeDeltaSymbolDictFrameWithRange(long buf, long deltaStart, private static int writeExecDoneTruncatedRowsAffected(long buf) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -973,7 +980,7 @@ private static int writeExecDoneTruncatedRowsAffected(long buf) { private static int writeGeohashResultBatch(long buf, long precisionBits) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); // header msg_kind (unused) p = putByte(p, (byte) 0); // flags (no delta dict) p = putByte(p, (byte) 1); // table_count @@ -984,8 +991,6 @@ private static int writeGeohashResultBatch(long buf, long precisionBits) { p = putVarint(p, 0L); // table_name_len p = putVarint(p, 0L); // row_count (no data rows) p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id // Schema: one column "g" of TYPE_GEOHASH. p = putVarint(p, 1L); p = putByte(p, (byte) 'g'); @@ -997,10 +1002,10 @@ private static int writeGeohashResultBatch(long buf, long precisionBits) { return (int) (p - buf); } - private static int writeMinimalResultBatch(long buf, long schemaId) { + private static int writeMinimalResultBatch(long buf) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -1011,8 +1016,6 @@ private static int writeMinimalResultBatch(long buf, long schemaId) { p = putVarint(p, 0L); // table_name_len p = putVarint(p, 0L); // row_count = 0 (no body needed) p = putVarint(p, 0L); // column_count = 0 - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, schemaId); return (int) (p - buf); } @@ -1024,7 +1027,7 @@ private static int writeMinimalResultBatch(long buf, long schemaId) { private static int writeMinimalResultBatchWithRawNameLenVarint(long buf, byte[] nameLenVarint) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -1036,30 +1039,6 @@ private static int writeMinimalResultBatchWithRawNameLenVarint(long buf, byte[] return (int) (p - buf); } - /** - * Variant that writes a custom raw varint sequence for schema_id. Lets us - * inject a multi-byte varint that decodes to a value with the int sign bit - * set after long-to-int truncation. - */ - private static int writeMinimalResultBatchWithRawSchemaIdVarint(long buf, byte[] schemaIdVarint) { - long p = buf; - p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); - p = putByte(p, (byte) 0); - p = putByte(p, (byte) 0); - p = putByte(p, (byte) 1); - p = putInt(p, 0); - p = putByte(p, (byte) 0x11); - p = putLong(p, 1L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putVarint(p, 0L); - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - for (byte b : schemaIdVarint) p = putByte(p, b); - return (int) (p - buf); - } - /** * Crafts a RESULT_BATCH frame whose flags byte advertises FLAG_ZSTD but * whose body is junk (not a valid zstd frame). Used by @@ -1068,7 +1047,7 @@ private static int writeMinimalResultBatchWithRawSchemaIdVarint(long buf, byte[] private static int writeResultBatchWithCorruptZstdBody(long buf) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, QwpConstants.FLAG_ZSTD); // flags byte (offset 5) -- FLAG_ZSTD set p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -1091,7 +1070,7 @@ private static int writeResultBatchWithCorruptZstdBody(long buf) { private static int writeResultEndTruncatedFinalSeq(long buf) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -1110,7 +1089,7 @@ private static int writeResultEndTruncatedFinalSeq(long buf) { private static int writeResultEndTruncatedTotalRows(long buf) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -1126,7 +1105,7 @@ private static int writeStringResultBatch(long buf, int nonNull, int totalBytes) long p = buf; // Header: magic + version + msg_kind + flags + table_count + payload_length p = putInt(p, QwpConstants.MAGIC_MESSAGE); // 4 - p = putByte(p, QwpConstants.VERSION_1); // 1 + p = putByte(p, QwpConstants.VERSION); // 1 p = putByte(p, (byte) 0); // msg_kind in header (unused by client) p = putByte(p, (byte) 0); // flags p = putByte(p, (byte) 1); // table_count @@ -1139,9 +1118,7 @@ private static int writeStringResultBatch(long buf, int nonNull, int totalBytes) p = putVarint(p, 0L); // table_name_len = 0 p = putVarint(p, nonNull); // row_count p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id - // Schema entries (full): one column "s" of TYPE_VARCHAR + // Schema entries: one column "s" of TYPE_VARCHAR p = putVarint(p, 1L); // column name length p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_VARCHAR); @@ -1173,7 +1150,7 @@ private static int writeStringResultBatch(long buf, int nonNull, int totalBytes) private static int writeStringResultBatchWithOffsets(long buf, int[] intOffsets, int totalBytes, int bytesLen) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); p = putByte(p, (byte) 1); @@ -1184,8 +1161,6 @@ private static int writeStringResultBatchWithOffsets(long buf, int[] intOffsets, p = putVarint(p, 0L); // table_name_len p = putVarint(p, intOffsets.length); // row_count = nonNull p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id p = putVarint(p, 1L); // column name length p = putByte(p, (byte) 's'); p = putByte(p, QwpConstants.TYPE_VARCHAR); @@ -1209,7 +1184,7 @@ private static int writeStringResultBatchWithOffsets(long buf, int[] intOffsets, private static int writeSymbolResultBatch(long buf, long rowCount, long dictSize) { long p = buf; p = putInt(p, QwpConstants.MAGIC_MESSAGE); - p = putByte(p, QwpConstants.VERSION_1); + p = putByte(p, QwpConstants.VERSION); p = putByte(p, (byte) 0); p = putByte(p, (byte) 0); // flags: no delta dict, so non-delta SYMBOL path p = putByte(p, (byte) 1); @@ -1220,8 +1195,6 @@ private static int writeSymbolResultBatch(long buf, long rowCount, long dictSize p = putVarint(p, 0L); // table_name_len p = putVarint(p, rowCount); p = putVarint(p, 1L); // column_count - p = putByte(p, QwpConstants.SCHEMA_MODE_FULL); - p = putVarint(p, 0L); // schema_id // Schema: one column "s" of TYPE_SYMBOL. p = putVarint(p, 1L); p = putByte(p, (byte) 's'); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java index 98bbbdeb..067a8a97 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpUdpSenderTest.java @@ -2325,7 +2325,7 @@ private DatagramDecoder(byte[] packet) { private List decode() { Assert.assertEquals(MAGIC_MESSAGE, reader.readIntLE()); - Assert.assertEquals(VERSION_1, reader.readByte()); + Assert.assertEquals(VERSION, reader.readByte()); reader.readByte(); int tableCount = reader.readUnsignedShortLE(); int payloadLength = reader.readIntLE(); @@ -2548,8 +2548,6 @@ private List decodeTable() { String tableName = reader.readString(); int rowCount = (int) reader.readVarint(); int columnCount = (int) reader.readVarint(); - Assert.assertEquals(SCHEMA_MODE_FULL, reader.readByte()); - reader.readVarint(); // schemaId QwpColumnDef[] defs = new QwpColumnDef[columnCount]; for (int i = 0; i < columnCount; i++) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 591b27f3..271501b5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -54,7 +54,7 @@ public void testBufferResetAndReuse() throws Exception { col.addLong(i); buffer.nextRow(); } - int size1 = encoder.encode(buffer, false); + int size1 = encoder.encode(buffer); // Reset and second batch buffer.reset(); @@ -63,7 +63,7 @@ public void testBufferResetAndReuse() throws Exception { col.addLong(i * 2); buffer.nextRow(); } - int size2 = encoder.encode(buffer, false); + int size2 = encoder.encode(buffer); Assert.assertTrue(size1 > size2); // More rows = larger Assert.assertEquals(50, buffer.getRowCount()); @@ -81,7 +81,7 @@ public void testEncode2DDoubleArray() throws Exception { col.addDoubleArray(new double[][]{{1.0, 2.0}, {3.0, 4.0}}); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -97,7 +97,7 @@ public void testEncode2DLongArray() throws Exception { col.addLongArray(new long[][]{{1L, 2L}, {3L, 4L}}); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -116,7 +116,7 @@ public void testEncode3DDoubleArray() throws Exception { }); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -141,7 +141,7 @@ public void testEncodeAllBasicTypesInOneRow() throws Exception { buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(1, buffer.getRowCount()); } @@ -160,7 +160,7 @@ public void testEncodeAllBooleanValues() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(100, buffer.getRowCount()); } @@ -177,7 +177,7 @@ public void testEncodeDecimal128() throws Exception { col.addDecimal128(io.questdb.client.std.Decimal128.fromLong(123456789012345L, 4)); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -193,7 +193,7 @@ public void testEncodeDecimal256() throws Exception { col.addDecimal256(io.questdb.client.std.Decimal256.fromLong(Long.MAX_VALUE, 6)); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -209,7 +209,7 @@ public void testEncodeDecimal64() throws Exception { col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(12345L, 2)); // 123.45 buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -225,7 +225,7 @@ public void testEncodeDoubleArray() throws Exception { col.addDoubleArray(new double[]{1.0, 2.0, 3.0}); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -241,7 +241,7 @@ public void testEncodeEmptyString() throws Exception { col.addString(""); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -257,7 +257,7 @@ public void testEncodeEmptyTableName() throws Exception { col.addLong(1L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 0); } }); @@ -279,7 +279,7 @@ public void testEncodeLargeArray() throws Exception { col.addDoubleArray(largeArray); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 8000); // At least 8 bytes per double } }); @@ -297,7 +297,7 @@ public void testEncodeLargeRowCount() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(10_000, buffer.getRowCount()); } @@ -314,7 +314,7 @@ public void testEncodeLongArray() throws Exception { col.addLongArray(new long[]{1L, 2L, 3L}); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -332,7 +332,7 @@ public void testEncodeLongString() throws Exception { col.addString(sb); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 10_000); } }); @@ -351,7 +351,7 @@ public void testEncodeMaxMinLong() throws Exception { col.addLong(Long.MIN_VALUE); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(2, buffer.getRowCount()); } @@ -385,7 +385,7 @@ public void testEncodeMixedColumnTypes() throws Exception { buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -413,7 +413,7 @@ public void testEncodeMixedColumnsMultipleRows() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(50, buffer.getRowCount()); } @@ -438,7 +438,7 @@ public void testEncodeMultipleColumns() throws Exception { buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); // Verify header @@ -468,7 +468,7 @@ public void testEncodeMultipleDecimal64() throws Exception { col.addDecimal64(io.questdb.client.std.Decimal64.fromLong(11111L, 2)); // 111.11 buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(3, buffer.getRowCount()); } @@ -491,7 +491,7 @@ public void testEncodeMultipleRows() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(100, buffer.getRowCount()); } @@ -517,7 +517,7 @@ public void testEncodeMultipleSymbolsSameDictionary() throws Exception { col.addSymbol("server1"); // Back to first buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(4, buffer.getRowCount()); } @@ -536,7 +536,7 @@ public void testEncodeMultipleUuids() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(10, buffer.getRowCount()); } @@ -553,7 +553,7 @@ public void testEncodeNaNDouble() throws Exception { col.addDouble(Double.NaN); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -569,7 +569,7 @@ public void testEncodeNegativeLong() throws Exception { col.addLong(-123456789L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -586,7 +586,7 @@ public void testEncodeNullableColumnWithNull() throws Exception { col.addString(null); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -603,7 +603,7 @@ public void testEncodeNullableColumnWithValue() throws Exception { col.addString("hello"); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -625,7 +625,7 @@ public void testEncodeNullableSymbolWithNull() throws Exception { col.addSymbol("server2"); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(3, buffer.getRowCount()); } @@ -642,7 +642,7 @@ public void testEncodeSingleRowWithBoolean() throws Exception { col.addBoolean(true); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -658,7 +658,7 @@ public void testEncodeSingleRowWithDouble() throws Exception { col.addDouble(23.5); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -675,7 +675,7 @@ public void testEncodeSingleRowWithLong() throws Exception { col.addLong(12345L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); // At least header size QwpBufferWriter buf = encoder.getBuffer(); @@ -688,7 +688,7 @@ public void testEncodeSingleRowWithLong() throws Exception { Assert.assertEquals((byte) '1', Unsafe.getUnsafe().getByte(ptr + 3)); // Version - Assert.assertEquals(VERSION_1, Unsafe.getUnsafe().getByte(ptr + 4)); + Assert.assertEquals(VERSION, Unsafe.getUnsafe().getByte(ptr + 4)); // Table count (little-endian short) Assert.assertEquals((short) 1, Unsafe.getUnsafe().getShort(ptr + 6)); @@ -706,7 +706,7 @@ public void testEncodeSingleRowWithString() throws Exception { col.addString("hello"); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -723,7 +723,7 @@ public void testEncodeSingleRowWithTimestamp() throws Exception { col.addLong(1000000L); // Micros buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -739,7 +739,7 @@ public void testEncodeSingleSymbol() throws Exception { col.addSymbol("server1"); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -764,7 +764,7 @@ public void testEncodeSpecialDoubles() throws Exception { col.addDouble(Double.NEGATIVE_INFINITY); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(4, buffer.getRowCount()); } @@ -783,7 +783,7 @@ public void testEncodeSymbolWithManyDistinctValues() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); Assert.assertEquals(100, buffer.getRowCount()); } @@ -800,7 +800,7 @@ public void testEncodeUnicodeString() throws Exception { col.addString("Hello 世界 🌍"); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -816,7 +816,7 @@ public void testEncodeUuid() throws Exception { col.addUuid(0x123456789ABCDEF0L, 0xFEDCBA9876543210L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -844,7 +844,7 @@ public void testEncodeWithDeltaDict_freshConnection_sendsAllSymbols() throws Exc int confirmedMaxId = -1; int batchMaxId = 1; - int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId); Assert.assertTrue(size > 12); QwpBufferWriter buf = encoder.getBuffer(); @@ -879,7 +879,7 @@ public void testEncodeWithDeltaDict_noNewSymbols_sendsEmptyDelta() throws Except int confirmedMaxId = 1; int batchMaxId = 1; - int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId); Assert.assertTrue(size > 12); QwpBufferWriter buf = encoder.getBuffer(); @@ -919,11 +919,10 @@ public void testEncodeWithDeltaDict_readsGlobalIdsFromDataBuffer() throws Except buffer.nextRow(); col.addSymbolWithGlobalId("SYM_7", 7); buffer.nextRow(); - buffer.setSchemaId(0); Assert.assertEquals(0, col.getAuxDataAddress()); - int size = encoder.encodeWithDeltaDict(buffer, globalDict, 7, 7, false); + int size = encoder.encodeWithDeltaDict(buffer, globalDict, 7, 7); Assert.assertTrue(size > 12); Cursor cursor = new Cursor(encoder.getBuffer().getBufferPtr() + HEADER_SIZE); @@ -933,8 +932,6 @@ public void testEncodeWithDeltaDict_readsGlobalIdsFromDataBuffer() throws Except Assert.assertEquals("test_table", cursor.readString()); Assert.assertEquals(2, cursor.readVarint()); Assert.assertEquals(1, cursor.readVarint()); - Assert.assertEquals(SCHEMA_MODE_FULL, cursor.readByte()); - Assert.assertEquals(0, cursor.readVarint()); // schemaId Assert.assertEquals("ticker", cursor.readString()); Assert.assertEquals(TYPE_SYMBOL, cursor.readByte()); Assert.assertEquals(0, cursor.readByte()); // no nulls @@ -969,7 +966,7 @@ public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() throws Exceptio int confirmedMaxId = 1; int batchMaxId = 3; - int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId, false); + int size = encoder.encodeWithDeltaDict(buffer, globalDict, confirmedMaxId, batchMaxId); Assert.assertTrue(size > 12); QwpBufferWriter buf = encoder.getBuffer(); @@ -989,23 +986,6 @@ public void testEncodeWithDeltaDict_withConfirmed_sendsOnlyNew() throws Exceptio }); } - @Test - public void testEncodeWithSchemaRef() throws Exception { - assertMemoryLeak(() -> { - try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); - QwpTableBuffer buffer = new QwpTableBuffer("test_table")) { - - QwpTableBuffer.ColumnBuffer col = buffer.getOrCreateColumn("x", TYPE_LONG, false); - col.addLong(42L); - buffer.nextRow(); - buffer.setSchemaId(0); - - int size = encoder.encode(buffer, true); // Use schema reference - Assert.assertTrue(size > 12); - } - }); - } - @Test public void testEncodeZeroLong() throws Exception { assertMemoryLeak(() -> { @@ -1016,7 +996,7 @@ public void testEncodeZeroLong() throws Exception { col.addLong(0L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -1032,13 +1012,13 @@ public void testEncoderReusability() throws Exception { QwpTableBuffer.ColumnBuffer col1 = buffer1.getOrCreateColumn("x", TYPE_LONG, false); col1.addLong(1L); buffer1.nextRow(); - int size1 = encoder.encode(buffer1, false); + int size1 = encoder.encode(buffer1); // Encode second message (encoder should reset internally) QwpTableBuffer.ColumnBuffer col2 = buffer2.getOrCreateColumn("y", TYPE_DOUBLE, false); col2.addDouble(2.0); buffer2.nextRow(); - int size2 = encoder.encode(buffer2, false); + int size2 = encoder.encode(buffer2); // Both should succeed Assert.assertTrue(size1 > 12); @@ -1085,7 +1065,7 @@ public void testGorillaEncoding_compressionRatio() throws Exception { buffer.nextRow(); } - int sizeWithGorilla = encoder.encode(buffer, false); + int sizeWithGorilla = encoder.encode(buffer); // Calculate theoretical minimum size for Gorilla: // - Header: 12 bytes @@ -1103,7 +1083,7 @@ public void testGorillaEncoding_compressionRatio() throws Exception { buffer.nextRow(); } - int sizeWithoutGorilla = encoder.encode(buffer, false); + int sizeWithoutGorilla = encoder.encode(buffer); // For constant delta, Gorilla should achieve significant compression double compressionRatio = (double) sizeWithGorilla / sizeWithoutGorilla; @@ -1131,7 +1111,7 @@ public void testGorillaEncoding_multipleTimestampColumns() throws Exception { buffer.nextRow(); } - int sizeWithGorilla = encoder.encode(buffer, false); + int sizeWithGorilla = encoder.encode(buffer); // Compare with uncompressed encoder.setGorillaEnabled(false); @@ -1146,7 +1126,7 @@ public void testGorillaEncoding_multipleTimestampColumns() throws Exception { buffer.nextRow(); } - int sizeWithoutGorilla = encoder.encode(buffer, false); + int sizeWithoutGorilla = encoder.encode(buffer); Assert.assertTrue("Gorilla should compress multiple timestamp columns", sizeWithGorilla < sizeWithoutGorilla); @@ -1168,7 +1148,7 @@ public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() throws buffer.nextRow(); } - int sizeWithGorilla = encoder.encode(buffer, false); + int sizeWithGorilla = encoder.encode(buffer); // Now encode without Gorilla encoder.setGorillaEnabled(false); @@ -1179,7 +1159,7 @@ public void testGorillaEncoding_multipleTimestamps_usesGorillaEncoding() throws buffer.nextRow(); } - int sizeWithoutGorilla = encoder.encode(buffer, false); + int sizeWithoutGorilla = encoder.encode(buffer); // Gorilla should produce smaller output for constant-delta timestamps Assert.assertTrue("Gorilla encoding should be smaller", @@ -1202,7 +1182,7 @@ public void testGorillaEncoding_nanosTimestamps() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); // Verify header has Gorilla flag @@ -1225,7 +1205,7 @@ public void testGorillaEncoding_singleTimestamp_usesUncompressed() throws Except col.addLong(1000000L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); } }); @@ -1245,7 +1225,7 @@ public void testGorillaEncoding_twoTimestamps_usesUncompressed() throws Exceptio col.addLong(2000000L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); // Verify header has Gorilla flag set @@ -1279,7 +1259,7 @@ public void testGorillaEncoding_varyingDelta() throws Exception { buffer.nextRow(); } - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 12); // Verify header has Gorilla flag @@ -1302,7 +1282,7 @@ public void testGorillaFlagDisabled() throws Exception { col.addLong(1000000L); buffer.nextRow(); - encoder.encode(buffer, false); + encoder.encode(buffer); // Check flags byte doesn't have Gorilla bit set QwpBufferWriter buf = encoder.getBuffer(); @@ -1324,7 +1304,7 @@ public void testGorillaFlagEnabled() throws Exception { col.addLong(1000000L); buffer.nextRow(); - encoder.encode(buffer, false); + encoder.encode(buffer); // Check flags byte has Gorilla bit set QwpBufferWriter buf = encoder.getBuffer(); @@ -1343,7 +1323,7 @@ public void testPayloadLengthPatched() throws Exception { col.addLong(42L); buffer.nextRow(); - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); // Payload length is at offset 8 (4 magic + 1 version + 1 flags + 2 tablecount) QwpBufferWriter buf = encoder.getBuffer(); @@ -1365,7 +1345,7 @@ public void testReset() throws Exception { col.addLong(1L); buffer.nextRow(); - int size1 = encoder.encode(buffer, false); + int size1 = encoder.encode(buffer); // Reset and encode again buffer.reset(); @@ -1373,7 +1353,7 @@ public void testReset() throws Exception { col.addLong(2L); buffer.nextRow(); - int size2 = encoder.encode(buffer, false); + int size2 = encoder.encode(buffer); // Sizes should be similar (same schema) Assert.assertEquals(size1, size2); @@ -1392,7 +1372,7 @@ public void testVersionByteInHeader() throws Exception { buffer.nextRow(); // Default version - int size = encoder.encode(buffer, false); + int size = encoder.encode(buffer); Assert.assertTrue(size > 0); long ptr = encoder.getBuffer().getBufferPtr(); Assert.assertEquals(1, Unsafe.getUnsafe().getByte(ptr + 4)); @@ -1403,7 +1383,7 @@ public void testVersionByteInHeader() throws Exception { col.addLong(42); buffer.nextRow(); encoder.setVersion((byte) 3); - encoder.encode(buffer, false); + encoder.encode(buffer); Assert.assertEquals(3, Unsafe.getUnsafe().getByte(ptr + 4)); } }); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 2d61ad9d..d8976f41 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -346,7 +346,7 @@ public void testFlushAppendFailureDoesNotLeaveMicrobatchBufferInUse() throws Exc CursorSendEngine engine = new CursorSendEngine(null, 33, 33, 1L); try (QwpWebSocketSender sender = QwpWebSocketSender.connect( "localhost", port, null, Integer.MAX_VALUE, 0, 0L, null, - QwpWebSocketSender.DEFAULT_MAX_SCHEMAS_PER_CONNECTION, false, engine, 0L)) { + false, engine, 0L)) { sender.table("t").longColumn("v", 1L).atNow(); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SegmentedNativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SegmentedNativeBufferWriterTest.java index 52e13cee..c53eea63 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SegmentedNativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SegmentedNativeBufferWriterTest.java @@ -121,7 +121,7 @@ public void testGorillaEncodingWithSegmentedBuffer() throws Exception { setBuffer.setAccessible(true); Method encodeTable = cwClass.getDeclaredMethod("encodeTable", - QwpTableBuffer.class, boolean.class, boolean.class, boolean.class); + QwpTableBuffer.class, boolean.class, boolean.class); encodeTable.setAccessible(true); try (QwpTableBuffer tableBuffer = new QwpTableBuffer("test")) { @@ -148,7 +148,7 @@ public void testGorillaEncodingWithSegmentedBuffer() throws Exception { // Encode table with Gorilla enabled into the segmented buffer. // Before the fix, this would compute a wrong write address and corrupt memory. setBuffer.invoke(columnWriter, writer); - encodeTable.invoke(columnWriter, tableBuffer, false, false, true); + encodeTable.invoke(columnWriter, tableBuffer, false, true); // Verify encoding produced data beyond the initial 32 bytes Assert.assertTrue("Encoding should produce data", writer.getPosition() > 32); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 6e045dd6..ac2e0d54 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -36,17 +36,19 @@ import java.util.concurrent.atomic.AtomicLong; /** - * Pins down the "every frame on disk is self-sufficient" rule. + * Pins down the "every frame on disk is self-sufficient" rule for the symbol + * dictionary. *

    - * The cursor SF path used to elide schema definitions and previously-sent - * symbols on subsequent batches over the same connection — emitting refs - * + delta-dicts. That's wrong for SF: the bytes survive process restarts - * and are replayed against fresh server connections (post-reconnect, or - * via a background drainer adopting an orphan slot). A frame with a - * schema-ref to an ID the new server has never seen is unrecoverable. + * The cursor SF path used to elide previously-sent symbols on subsequent + * batches over the same connection, emitting a delta-dict that carried only + * the new entries. That's wrong for SF: the bytes survive process restarts and + * replay against fresh server connections (post-reconnect, or via a background + * drainer adopting an orphan slot). A delta that references symbol ids the new + * server has never seen is unrecoverable. *

    - * Today every frame must carry its full schema and a complete symbol-dict - * delta starting at id 0. This test asserts both invariants on the wire. + * Today every frame must carry a complete symbol-dict delta starting at id 0 + * (column schemas travel inline on the first batch too). This test asserts the + * symbol-dict invariant on the wire. */ public class SelfSufficientFramesTest { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 3da4f829..fadc3eed 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -66,6 +66,12 @@ public class TestWebSocketServer implements Closeable { // The server emits the header on both the 101 success path and (when // rejectingRole != null) the 421 misdirected-request path. private volatile String advertisedRole; + // When true, the server sends a SERVER_INFO frame right after a successful + // 101 upgrade, mirroring the egress server contract (which always emits it). + // The advertised role follows advertisedRole (STANDALONE when unset). Egress + // QwpQueryClient tests enable this; ingress sender tests leave it off so their + // connections carry only ACK frames. + private volatile boolean sendServerInfo; // When non-null the next handshake responds with HTTP 421 Misdirected // Request + X-QuestDB-Role: , mimicking a server whose // QwpServerInfoProvider reports REPLICA / PRIMARY_CATCHUP. Set after @@ -173,6 +179,54 @@ public void setRejectWithStatus(int statusCode, String reasonPhrase) { this.rejectingStatusReason = reasonPhrase; } + /** + * When enabled, the server sends a {@code SERVER_INFO} frame immediately + * after a successful 101 upgrade, the way a real egress endpoint does. The + * advertised role follows {@link #setAdvertisedRole}, defaulting to + * {@code STANDALONE}. Leave disabled for ingress (Sender) tests. + */ + public void setSendServerInfo(boolean sendServerInfo) { + this.sendServerInfo = sendServerInfo; + } + + private static byte[] buildServerInfoFrame(byte role) { + byte[] clusterId = "questdb".getBytes(StandardCharsets.UTF_8); + byte[] nodeId = "test-node".getBytes(StandardCharsets.UTF_8); + int bodyLen = 1 + 1 + 8 + 4 + 8 + 2 + clusterId.length + 2 + nodeId.length; + ByteBuffer bb = ByteBuffer.allocate(12 + bodyLen).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 'Q').put((byte) 'W').put((byte) 'P').put((byte) '1'); + bb.put((byte) 1); // version + bb.put((byte) 0); // flags + bb.putShort((short) 0); // table_count (unused for control frames) + bb.putInt(bodyLen); // payload_length + bb.put((byte) 0x18); // SERVER_INFO msg_kind + bb.put(role); + bb.putLong(0L); // epoch + bb.putInt(0); // capabilities (no CAP_ZONE -> no zone_id trailer) + bb.putLong(1L); // server_wall_ns (positive) + bb.putShort((short) clusterId.length); + bb.put(clusterId); + bb.putShort((short) nodeId.length); + bb.put(nodeId); + return bb.array(); + } + + private static byte roleByte(String role) { + if (role == null) { + return 0; // ROLE_STANDALONE + } + switch (role) { + case "PRIMARY": + return 1; + case "REPLICA": + return 2; + case "PRIMARY_CATCHUP": + return 3; + default: + return 0; // STANDALONE + } + } + public void start() throws IOException { if (running.getAndSet(true)) { return; @@ -476,6 +530,10 @@ void start() { return; } + if (sendServerInfo) { + sendBinary(buildServerInfoFrame(roleByte(advertisedRole))); + } + byte[] readBuf = new byte[8192]; while (running.get() && !isClosed) {