Skip to content

Filter out empty messages from LLM prompt construction #3967

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lukemelia merged 3 commits into from
Mar 30, 2026
Merged

Filter out empty messages from LLM prompt construction #3967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f15c69c
Filter out empty messages from LLM prompt construction
claude Feb 10, 2026
f3c72fb
Remove over-engineered safety filter, fix test field name
claude Feb 10, 2026
717e725
fix: add required data field and remove invalid body field in test ev…
Mar 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
275 changes: 275 additions & 0 deletions packages/ai-bot/tests/prompt-construction-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5153,6 +5153,281 @@ new
);
});

test('excludes assistant messages with empty body and no tool calls', async () => {
const history: DiscreteMatrixEvent[] = [
{
type: 'm.room.message',
event_id: '1',
origin_server_ts: 1,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Hello',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '1',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '2',
origin_server_ts: 2,
content: {
body: '',
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
isStreamingFinished: true,
data: {},
},
sender: '@aibot:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '2',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '3',
origin_server_ts: 3,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Can you help me?',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '3',
},
status: EventStatus.SENT,
},
];

const result = await buildPromptForModel(
history,
'@aibot:localhost',
undefined,
undefined,
[],
fakeMatrixClient,
);

const assistantMessages = result.filter(
(message) => message.role === 'assistant',
);
assert.equal(
assistantMessages.length,
0,
'Empty assistant message should not be included',
);

const userMessages = result.filter((message) => message.role === 'user');
assert.equal(
userMessages.length,
2,
'Both user messages should be included',
);
});

test('keeps assistant messages with empty body when they have tool calls', async () => {
const history: DiscreteMatrixEvent[] = [
{
type: 'm.room.message',
event_id: '1',
origin_server_ts: 1,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Update my card',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '1',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '2',
origin_server_ts: 2,
content: {
body: '',
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
isStreamingFinished: true,
[APP_BOXEL_COMMAND_REQUESTS_KEY]: [
{
id: 'call_1',
name: 'patchCardInstance',
arguments: JSON.stringify({
card_id: 'http://localhost/card/1',
attributes: { title: 'Updated' },
}),
},
],
data: {},
},
sender: '@aibot:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '2',
},
status: EventStatus.SENT,
},
{
type: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
event_id: '3',
origin_server_ts: 3,
content: {
msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
commandRequestId: 'call_1',
'm.relates_to': {
rel_type: APP_BOXEL_COMMAND_RESULT_REL_TYPE,
event_id: '2',
key: 'applied',
},
data: {},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '3',
},
status: EventStatus.SENT,
},
];

const result = await buildPromptForModel(
history,
'@aibot:localhost',
undefined,
undefined,
[],
fakeMatrixClient,
);

const assistantMessages = result.filter(
(message) => message.role === 'assistant',
);
assert.equal(
assistantMessages.length,
1,
'Assistant message with tool calls should be kept even with empty body',
);
assert.ok(
assistantMessages[0].tool_calls?.length,
'Assistant message should have tool calls',
);

const toolMessages = result.filter((message) => message.role === 'tool');
assert.equal(
toolMessages.length,
1,
'Tool result message should be present alongside the assistant tool call',
);
assert.equal(
toolMessages[0].tool_call_id,
'call_1',
'Tool result should reference the correct tool call id',
);
});

test('excludes user messages with empty body', async () => {
const history: DiscreteMatrixEvent[] = [
{
type: 'm.room.message',
event_id: '1',
origin_server_ts: 1,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: '',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '1',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '2',
origin_server_ts: 2,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Hello',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '2',
},
status: EventStatus.SENT,
},
];

const result = await buildPromptForModel(
history,
'@aibot:localhost',
undefined,
undefined,
[],
fakeMatrixClient,
);

const userMessages = result.filter((message) => message.role === 'user');
assert.equal(
userMessages.length,
1,
'Only the non-empty user message should be included',
);
assert.equal(userMessages[0].content, 'Hello');
});
test('only the most recent message attachments include file content in the prompt', async () => {
// Policy: files attached to older messages should show metadata only,
// even if they are NOT re-attached in later messages.
Expand Down
17 changes: 10 additions & 7 deletions packages/runtime-common/ai/prompt.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1340,15 +1340,18 @@ export async function buildPromptForModel(
event as CardMessageEvent,
history,
);
let historicalMessage: OpenAIPromptMessage = {
role: 'assistant',
content: elideCodeBlocks(body, codePatchResults),
};
let content = elideCodeBlocks(body, codePatchResults);
let toolCalls = toToolCalls(event as CardMessageEvent);
if (toolCalls.length) {
historicalMessage.tool_calls = toolCalls;
if (content || toolCalls.length) {
let historicalMessage: OpenAIPromptMessage = {
role: 'assistant',
content,
};
if (toolCalls.length) {
historicalMessage.tool_calls = toolCalls;
}
historicalMessages.push(historicalMessage);
}
historicalMessages.push(historicalMessage);
let commandResults = getCommandResults(
event as CardMessageEvent,
history,
Expand Down
Loading