From f15c69cae26f9f041c4947c018f733236dc375c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Feb 2026 01:15:28 +0000 Subject: [PATCH 1/3] Filter out empty messages from LLM prompt construction Empty messages (with no content and no tool calls) could be sent to the LLM, which is wasteful and potentially causes issues. This change: - Skips assistant messages with empty body and no tool calls during history construction (matching the existing guard on user messages) - Adds a safety filter (hasMessageContent) on the final messages array to catch any empty messages from any source, while preserving assistant messages with tool calls and tool response messages CS-10116 https://claude.ai/code/session_01JMzG85yR6uQUXVfnUTegXz --- .../ai-bot/tests/prompt-construction-test.ts | 262 ++++++++++++++++++ packages/runtime-common/ai/prompt.ts | 38 ++- 2 files changed, 292 insertions(+), 8 deletions(-) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 569b0c6026a..99bc943c0f5 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -5153,6 +5153,268 @@ new ); }); + + test('excludes assistant messages with empty body and no tool calls', async () => { + const history: DiscreteMatrixEvent[] = [ + { + type: 'm.room.message', + event_id: '1', + origin_server_ts: 1, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Hello', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '1', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '2', + origin_server_ts: 2, + content: { + body: '', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + }, + sender: '@aibot:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '2', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '3', + origin_server_ts: 3, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Can you help me?', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '3', + }, + status: EventStatus.SENT, + }, + ]; + + const result = await buildPromptForModel( + history, + '@aibot:localhost', + undefined, + undefined, + [], + fakeMatrixClient, + ); + + const assistantMessages = result.filter( + (message) => message.role === 'assistant', + ); + assert.equal( + assistantMessages.length, + 0, + 'Empty assistant message should not be included', + ); + + const userMessages = result.filter((message) => message.role === 'user'); + assert.equal( + userMessages.length, + 2, + 'Both user messages should be included', + ); + }); + + test('keeps assistant messages with empty body when they have tool calls', async () => { + const history: DiscreteMatrixEvent[] = [ + { + type: 'm.room.message', + event_id: '1', + origin_server_ts: 1, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Update my card', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '1', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '2', + origin_server_ts: 2, + content: { + body: '', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_COMMAND_REQUESTS_KEY]: [ + { + toolCallId: 'call_1', + name: 'patchCardInstance', + arguments: JSON.stringify({ + card_id: 'http://localhost/card/1', + attributes: { title: 'Updated' }, + }), + }, + ], + }, + sender: '@aibot:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '2', + }, + status: EventStatus.SENT, + }, + { + type: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, + event_id: '3', + origin_server_ts: 3, + content: { + msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE, + body: 'Command result', + commandRequestId: 'call_1', + 'm.relates_to': { + rel_type: APP_BOXEL_COMMAND_RESULT_REL_TYPE, + event_id: '2', + key: 'applied', + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '3', + }, + status: EventStatus.SENT, + }, + ]; + + const result = await buildPromptForModel( + history, + '@aibot:localhost', + undefined, + undefined, + [], + fakeMatrixClient, + ); + + const assistantMessages = result.filter( + (message) => message.role === 'assistant', + ); + assert.equal( + assistantMessages.length, + 1, + 'Assistant message with tool calls should be kept even with empty body', + ); + assert.ok( + assistantMessages[0].tool_calls?.length, + 'Assistant message should have tool calls', + ); + }); + + test('excludes user messages with empty body', async () => { + const history: DiscreteMatrixEvent[] = [ + { + type: 'm.room.message', + event_id: '1', + origin_server_ts: 1, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: '', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '1', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '2', + origin_server_ts: 2, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Hello', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '2', + }, + status: EventStatus.SENT, + }, + ]; + + const result = await buildPromptForModel( + history, + '@aibot:localhost', + undefined, + undefined, + [], + fakeMatrixClient, + ); + + const userMessages = result.filter((message) => message.role === 'user'); + assert.equal( + userMessages.length, + 1, + 'Only the non-empty user message should be included', + ); + assert.equal(userMessages[0].content, 'Hello'); + }); test('only the most recent message attachments include file content in the prompt', async () => { // Policy: files attached to older messages should show metadata only, // even if they are NOT re-attached in later messages. diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index c6be316bcad..deab74c95f1 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -1340,15 +1340,18 @@ export async function buildPromptForModel( event as CardMessageEvent, history, ); - let historicalMessage: OpenAIPromptMessage = { - role: 'assistant', - content: elideCodeBlocks(body, codePatchResults), - }; + let content = elideCodeBlocks(body, codePatchResults); let toolCalls = toToolCalls(event as CardMessageEvent); - if (toolCalls.length) { - historicalMessage.tool_calls = toolCalls; + if (content || toolCalls.length) { + let historicalMessage: OpenAIPromptMessage = { + role: 'assistant', + content, + }; + if (toolCalls.length) { + historicalMessage.tool_calls = toolCalls; + } + historicalMessages.push(historicalMessage); } - historicalMessages.push(historicalMessage); let commandResults = getCommandResults( event as CardMessageEvent, history, @@ -1464,7 +1467,26 @@ export async function buildPromptForModel( } } - return messages; + return messages.filter(hasMessageContent); +} + +function hasMessageContent(message: OpenAIPromptMessage): boolean { + // Assistant messages with tool calls are valid even without text content + if (message.role === 'assistant' && message.tool_calls?.length) { + return true; + } + // Tool messages are required responses to tool calls + if (message.role === 'tool') { + return true; + } + let content = message.content; + if (typeof content === 'string') { + return content.length > 0; + } + if (Array.isArray(content)) { + return content.length > 0; + } + return false; } function collectPendingCodePatchCorrectnessCheck( From f3c72fba57424bd9fd095fafd7ce23a41eaab4a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Feb 2026 01:24:07 +0000 Subject: [PATCH 2/3] Remove over-engineered safety filter, fix test field name - Remove hasMessageContent filter: it was redundant with the inline guards already present for each message type and would silently mask construction bugs rather than surfacing them - Fix test: use correct field name `id` instead of `toolCallId` for encoded command requests (matching all other tests in the file) - Add tool result assertions to verify the assistant+tool message pairing is preserved when assistant body is empty https://claude.ai/code/session_01JMzG85yR6uQUXVfnUTegXz --- .../ai-bot/tests/prompt-construction-test.ts | 14 ++++++++++++- packages/runtime-common/ai/prompt.ts | 21 +------------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 99bc943c0f5..9f3bb314842 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -5284,7 +5284,7 @@ new isStreamingFinished: true, [APP_BOXEL_COMMAND_REQUESTS_KEY]: [ { - toolCallId: 'call_1', + id: 'call_1', name: 'patchCardInstance', arguments: JSON.stringify({ card_id: 'http://localhost/card/1', @@ -5346,6 +5346,18 @@ new assistantMessages[0].tool_calls?.length, 'Assistant message should have tool calls', ); + + const toolMessages = result.filter((message) => message.role === 'tool'); + assert.equal( + toolMessages.length, + 1, + 'Tool result message should be present alongside the assistant tool call', + ); + assert.equal( + toolMessages[0].tool_call_id, + 'call_1', + 'Tool result should reference the correct tool call id', + ); }); test('excludes user messages with empty body', async () => { diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index deab74c95f1..ab277501bfe 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -1467,26 +1467,7 @@ export async function buildPromptForModel( } } - return messages.filter(hasMessageContent); -} - -function hasMessageContent(message: OpenAIPromptMessage): boolean { - // Assistant messages with tool calls are valid even without text content - if (message.role === 'assistant' && message.tool_calls?.length) { - return true; - } - // Tool messages are required responses to tool calls - if (message.role === 'tool') { - return true; - } - let content = message.content; - if (typeof content === 'string') { - return content.length > 0; - } - if (Array.isArray(content)) { - return content.length > 0; - } - return false; + return messages; } function collectPendingCodePatchCorrectnessCheck( From 717e725e3f52ab2eeb9bee755dcd0203c028f808 Mon Sep 17 00:00:00 2001 From: Luke Melia Date: Mon, 30 Mar 2026 16:44:33 +0000 Subject: [PATCH 3/3] fix: add required data field and remove invalid body field in test events --- packages/ai-bot/tests/prompt-construction-test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 9f3bb314842..cfc9691e159 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -5153,7 +5153,6 @@ new ); }); - test('excludes assistant messages with empty body and no tool calls', async () => { const history: DiscreteMatrixEvent[] = [ { @@ -5188,6 +5187,7 @@ new msgtype: APP_BOXEL_MESSAGE_MSGTYPE, format: 'org.matrix.custom.html', isStreamingFinished: true, + data: {}, }, sender: '@aibot:localhost', room_id: 'room1', @@ -5292,6 +5292,7 @@ new }), }, ], + data: {}, }, sender: '@aibot:localhost', room_id: 'room1', @@ -5307,13 +5308,13 @@ new origin_server_ts: 3, content: { msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE, - body: 'Command result', commandRequestId: 'call_1', 'm.relates_to': { rel_type: APP_BOXEL_COMMAND_RESULT_REL_TYPE, event_id: '2', key: 'applied', }, + data: {}, }, sender: '@user:localhost', room_id: 'room1',