diff --git a/api.go b/api.go index acc789ef..e0486d77 100644 --- a/api.go +++ b/api.go @@ -30,6 +30,7 @@ type ( TokenUsageRecord = recorder.TokenUsageRecord PromptUsageRecord = recorder.PromptUsageRecord ToolUsageRecord = recorder.ToolUsageRecord + ModelThoughtRecord = recorder.ModelThoughtRecord Recorder = recorder.Recorder Metadata = recorder.Metadata diff --git a/fixtures/anthropic/multi_thinking_builtin_tool.txtar b/fixtures/anthropic/multi_thinking_builtin_tool.txtar new file mode 100644 index 00000000..d27ad63f --- /dev/null +++ b/fixtures/anthropic/multi_thinking_builtin_tool.txtar @@ -0,0 +1,152 @@ +Claude Code has builtin tools to (e.g.) explore the filesystem. +This fixture has two thinking blocks before the tool_use block. + +-- request -- +{ + "model": "claude-sonnet-4-20250514", + "max_tokens": 1024, + "tools": [ + { + "name": "Read", + "description": "Read the contents of a file at the given path.", + "input_schema": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "The absolute path to the file to read" + } + }, + "required": ["file_path"] + } + } + ], + "messages": [ + { + "role": "user", + "content": "read the foo file" + } + ] +} + +-- streaming -- +event: message_start +data: {"type":"message_start","message":{"id":"msg_015SQewixvT9s4cABCVvUE6g","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":22,"cache_read_input_tokens":13993,"output_tokens":5,"service_tier":"standard"}} } + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user wants me to read a file called \"foo\". Let me find and read it."}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"Eu8BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ=="}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"thinking","thinking":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"thinking_delta","thinking":"I should use the Read tool to access the file contents."}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"signature_delta","signature":"Aa1BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ=="}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01RX68weRSquLx6HUTj65iBo","name":"Read","input":{}}} + +event: ping +data: {"type": "ping"} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":""} } + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"file_path\": \"/tmp/blah/foo"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"\"}"} } + +event: content_block_stop +data: {"type":"content_block_stop","index":2 } + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":61} } + +event: message_stop +data: {"type":"message_stop" } + + +-- non-streaming -- +{ + "id": "msg_01JHKqEmh7wYuPXqUWUvusfL", + "container": { + "id": "", + "expires_at": "0001-01-01T00:00:00Z" + }, + "content": [ + { + "type": "thinking", + "thinking": "The user wants me to read a file called \"foo\". Let me find and read it.", + "signature": "Eu8BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ==" + }, + { + "type": "thinking", + "thinking": "I should use the Read tool to access the file contents.", + "signature": "Aa1BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ==" + }, + { + "citations": null, + "text": "", + "type": "tool_use", + "id": "toolu_01AusGgY5aKFhzWrFBv9JfHq", + "input": { + "file_path": "/tmp/blah/foo" + }, + "name": "Read", + "content": { + "OfWebSearchResultBlockArray": null, + "OfString": "", + "OfMCPToolResultBlockContent": null, + "error_code": "", + "type": "", + "content": null, + "return_code": 0, + "stderr": "", + "stdout": "" + }, + "tool_use_id": "", + "server_name": "", + "is_error": false, + "file_id": "", + "signature": "", + "thinking": "", + "data": "" + } + ], + "model": "claude-sonnet-4-20250514", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": "", + "type": "message", + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 23490, + "input_tokens": 5, + "output_tokens": 84, + "server_tool_use": { + "web_search_requests": 0 + }, + "service_tier": "standard" + } +} + diff --git a/fixtures/anthropic/simple.txtar b/fixtures/anthropic/simple.txtar index f1300b7b..235138cc 100644 --- a/fixtures/anthropic/simple.txtar +++ b/fixtures/anthropic/simple.txtar @@ -23,91 +23,100 @@ event: message_start data: {"type":"message_start","message":{"id":"msg_01Pvyf26bY17RcjmWfJsXGBn","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":18,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1,"service_tier":"standard"}} } event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"This is a classic philosophical question about medieval scholasticism. I'll give a thoughtful answer."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""} } event: ping data: {"type": "ping"} event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"This"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"This"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" is a famous philosophical question often used to illustrate medieval"}} +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" is a famous philosophical question often used to illustrate medieval"}} event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" scholastic debates that seem pointless or ov"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" scholastic debates that seem pointless or ov"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"erly abstract. The question \"How many angels can dance on the head of"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"erly abstract. The question \"How many angels can dance on the head of"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" a pin?\" is typically cited as an example of us"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" a pin?\" is typically cited as an example of us"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"eless speculation.\n\nHistorically, medieval theolog"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"eless speculation.\n\nHistorically, medieval theolog"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ians did debate the nature of angels -"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"ians did debate the nature of angels -"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" whether they were incorporeal beings, how"}} +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" whether they were incorporeal beings, how"}} event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" they occupied space, and whether multiple angels could exist"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" they occupied space, and whether multiple angels could exist"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" in the same location. However, there"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" in the same location. However, there"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s little evidence they literally"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"'s little evidence they literally"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" debated dancing angels on pinheads.\n\nThe question has"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" debated dancing angels on pinheads.\n\nThe question has"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" no factual answer since it depends on assumptions about:"}} +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" no factual answer since it depends on assumptions about:"}} event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n- The existence and nature of angels\n- Whether"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"\n- The existence and nature of angels\n- Whether"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" incorporeal beings occupy physical space\n- What"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" incorporeal beings occupy physical space\n- What"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" constitutes \"dancing\" for a spiritual"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" constitutes \"dancing\" for a spiritual"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" entity\n- The size of both the"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" entity\n- The size of both the"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" pin and the angels\n\nIt's become a metaph"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" pin and the angels\n\nIt's become a metaph"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"or for overthinking trivial matters"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"or for overthinking trivial matters"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" or getting lost in theoretical discussions disconnected from practical reality."} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" or getting lost in theoretical discussions disconnected from practical reality."} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Some use it to critique certain types of academic"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" Some use it to critique certain types of academic"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" or theological debate, while others defen"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" or theological debate, while others defen"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d the value of exploring fundamental questions about existence an"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"d the value of exploring fundamental questions about existence an"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d metaphysics.\n\nSo while u"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"d metaphysics.\n\nSo while u"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"nanswerable literally, it serves as an interesting lens"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"nanswerable literally, it serves as an interesting lens"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" for discussing the nature of philosophical inquiry itself."} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":" for discussing the nature of philosophical inquiry itself."} } event: content_block_stop -data: {"type":"content_block_stop","index":0 } +data: {"type":"content_block_stop","index":1 } event: message_delta data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":240} } @@ -122,6 +131,10 @@ data: {"type":"message_stop" } "role": "assistant", "model": "claude-sonnet-4-20250514", "content": [ + { + "type": "thinking", + "thinking": "This is a classic philosophical question about medieval scholasticism. I'll give a thoughtful answer." + }, { "type": "text", "text": "This is a famous philosophical question, often called \"How many angels can dance on the head of a pin?\" It's typically used to represent pointless or overly abstract theological debates.\n\nThe question doesn't have a literal answer because:\n\n1. **Historical context**: It's often attributed to medieval scholastic philosophers, though there's little evidence they actually debated this exact question. It became a popular way to mock what some saw as useless academic arguments.\n\n2. **Philosophical purpose**: The question highlights the difficulty of discussing non-physical beings (angels) in physical terms (space on a pinhead).\n\n3. **Different interpretations**: \n - If angels are purely spiritual, they might not take up physical space at all\n - If they do occupy space, we'd need to know their \"size\"\n - The question might be asking about the nature of space, matter, and spirit\n\nSo the real answer is that it's not meant to be answered literally - it's a thought experiment about the limits of rational inquiry and the sometimes absurd directions theological speculation can take.\n\nWould you like to explore the philosophical implications behind this question, or were you thinking about it in a different context?" diff --git a/fixtures/anthropic/single_builtin_tool.txtar b/fixtures/anthropic/single_builtin_tool.txtar index 50ca93f1..c271cb7c 100644 --- a/fixtures/anthropic/single_builtin_tool.txtar +++ b/fixtures/anthropic/single_builtin_tool.txtar @@ -33,22 +33,55 @@ event: message_start data: {"type":"message_start","message":{"id":"msg_015SQewixvT9s4cABCVvUE6g","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":22,"cache_read_input_tokens":13993,"output_tokens":5,"service_tier":"standard"}} } event: content_block_start -data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"toolu_01RX68weRSquLx6HUTj65iBo","name":"Read","input":{}} } +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user wants me to read"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" a"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" file called \""} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"foo\"."} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" Let me find"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" and"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":" read it."} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":""} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"Eu8BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ=="}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01RX68weRSquLx6HUTj65iBo","name":"Read","input":{}}} event: ping data: {"type": "ping"} event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":""} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"file_path\": \"/tmp/blah/foo"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"file_path\": \"/tmp/blah/foo"} } event: content_block_delta -data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\"}"} } +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"}"} } event: content_block_stop -data: {"type":"content_block_stop","index":0 } +data: {"type":"content_block_stop","index":1 } event: message_delta data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":61} } @@ -65,6 +98,11 @@ data: {"type":"message_stop" } "expires_at": "0001-01-01T00:00:00Z" }, "content": [ + { + "type": "thinking", + "thinking": "The user wants me to read a file called \"foo\". Let me find and read it.", + "signature": "Eu8BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ==" + }, { "citations": null, "text": "I can see there's a file named `foo` in the `/tmp/blah` directory. Let me read it.", diff --git a/fixtures/anthropic/single_builtin_tool_parallel.txtar b/fixtures/anthropic/single_builtin_tool_parallel.txtar new file mode 100644 index 00000000..9c53ed2c --- /dev/null +++ b/fixtures/anthropic/single_builtin_tool_parallel.txtar @@ -0,0 +1,175 @@ +Claude Code has builtin tools to (e.g.) explore the filesystem. +This fixture has a single thinking block followed by two parallel tool_use blocks. +The thinking should only be attributed to the first tool_use. + +-- request -- +{ + "model": "claude-sonnet-4-20250514", + "max_tokens": 1024, + "tools": [ + { + "name": "Read", + "description": "Read the contents of a file at the given path.", + "input_schema": { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "The absolute path to the file to read" + } + }, + "required": ["file_path"] + } + } + ], + "messages": [ + { + "role": "user", + "content": "read the foo and bar files" + } + ] +} + +-- streaming -- +event: message_start +data: {"type":"message_start","message":{"id":"msg_01ParallelToolStream","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":22,"cache_read_input_tokens":13993,"output_tokens":5,"service_tier":"standard"}} } + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"The user wants me to read two files: \"foo\" and \"bar\". I'll read both of them."}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"Eu8BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ=="}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01ParallelFirst000000000","name":"Read","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""} } + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"file_path\": \"/tmp/blah/foo"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"\"}"} } + +event: content_block_stop +data: {"type":"content_block_stop","index":1 } + +event: content_block_start +data: {"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"toolu_01ParallelSecond00000000","name":"Read","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":""} } + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"file_path\": \"/tmp/blah/bar"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"\"}"} } + +event: content_block_stop +data: {"type":"content_block_stop","index":2 } + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":72} } + +event: message_stop +data: {"type":"message_stop" } + + +-- non-streaming -- +{ + "id": "msg_01ParallelToolBlocking", + "container": { + "id": "", + "expires_at": "0001-01-01T00:00:00Z" + }, + "content": [ + { + "type": "thinking", + "thinking": "The user wants me to read two files: \"foo\" and \"bar\". I'll read both of them.", + "signature": "Eu8BCkYICxgCKkBR++kFr7Za2JhF/9OCpjEc46/EcipL75RK+MEbxJ/VBJPWQTWrNGfwb5khWYJtKEpjjkH07cR/MQvThfb7t7CkEgwU4pKwL7NuZXd1/wgaDILyd0bYMqQovWo3dyIw95Ny7yZPljNBDLsvMBdBr7w+RtbU+AlSftjBuBZHp0VzI54/W+9u6f7qfx0JXsVBKldqqOjFvewT8Xm6Qp/77g6/j0zBiuAQABj/6vS1qATjd8KSIFDg9G/tCtzwmV/T/egmzswWd5CBiAhW6lgJgEDRr+gRUrFSOB7o3hypW8FUnUrr1JtzzwMYAQ==" + }, + { + "citations": null, + "text": "", + "type": "tool_use", + "id": "toolu_01ParallelBlockFirst0000", + "input": { + "file_path": "/tmp/blah/foo" + }, + "name": "Read", + "content": { + "OfWebSearchResultBlockArray": null, + "OfString": "", + "OfMCPToolResultBlockContent": null, + "error_code": "", + "type": "", + "content": null, + "return_code": 0, + "stderr": "", + "stdout": "" + }, + "tool_use_id": "", + "server_name": "", + "is_error": false, + "file_id": "", + "signature": "", + "thinking": "", + "data": "" + }, + { + "citations": null, + "text": "", + "type": "tool_use", + "id": "toolu_01ParallelBlockSecond000", + "input": { + "file_path": "/tmp/blah/bar" + }, + "name": "Read", + "content": { + "OfWebSearchResultBlockArray": null, + "OfString": "", + "OfMCPToolResultBlockContent": null, + "error_code": "", + "type": "", + "content": null, + "return_code": 0, + "stderr": "", + "stdout": "" + }, + "tool_use_id": "", + "server_name": "", + "is_error": false, + "file_id": "", + "signature": "", + "thinking": "", + "data": "" + } + ], + "model": "claude-sonnet-4-20250514", + "role": "assistant", + "stop_reason": "tool_use", + "stop_sequence": "", + "type": "message", + "usage": { + "cache_creation": { + "ephemeral_1h_input_tokens": 0, + "ephemeral_5m_input_tokens": 0 + }, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 23490, + "input_tokens": 5, + "output_tokens": 95, + "server_tool_use": { + "web_search_requests": 0 + }, + "service_tier": "standard" + } +} diff --git a/fixtures/fixtures.go b/fixtures/fixtures.go index 3c150471..8aaeef15 100644 --- a/fixtures/fixtures.go +++ b/fixtures/fixtures.go @@ -15,6 +15,12 @@ var ( //go:embed anthropic/single_builtin_tool.txtar AntSingleBuiltinTool []byte + //go:embed anthropic/multi_thinking_builtin_tool.txtar + AntMultiThinkingBuiltinTool []byte + + //go:embed anthropic/single_builtin_tool_parallel.txtar + AntSingleBuiltinToolParallel []byte + //go:embed anthropic/single_injected_tool.txtar AntSingleInjectedTool []byte @@ -61,6 +67,15 @@ var ( //go:embed openai/responses/blocking/single_builtin_tool.txtar OaiResponsesBlockingSingleBuiltinTool []byte + //go:embed openai/responses/blocking/multi_reasoning_builtin_tool.txtar + OaiResponsesBlockingMultiReasoningBuiltinTool []byte + + //go:embed openai/responses/blocking/commentary_builtin_tool.txtar + OaiResponsesBlockingCommentaryBuiltinTool []byte + + //go:embed openai/responses/blocking/summary_and_commentary_builtin_tool.txtar + OaiResponsesBlockingSummaryAndCommentaryBuiltinTool []byte + //go:embed openai/responses/blocking/cached_input_tokens.txtar OaiResponsesBlockingCachedInputTokens []byte @@ -76,6 +91,9 @@ var ( //go:embed openai/responses/blocking/prev_response_id.txtar OaiResponsesBlockingPrevResponseID []byte + //go:embed openai/responses/blocking/single_builtin_tool_parallel.txtar + OaiResponsesBlockingSingleBuiltinToolParallel []byte + //go:embed openai/responses/blocking/single_injected_tool.txtar OaiResponsesBlockingSingleInjectedTool []byte @@ -96,6 +114,15 @@ var ( //go:embed openai/responses/streaming/builtin_tool.txtar OaiResponsesStreamingBuiltinTool []byte + //go:embed openai/responses/streaming/multi_reasoning_builtin_tool.txtar + OaiResponsesStreamingMultiReasoningBuiltinTool []byte + + //go:embed openai/responses/streaming/commentary_builtin_tool.txtar + OaiResponsesStreamingCommentaryBuiltinTool []byte + + //go:embed openai/responses/streaming/summary_and_commentary_builtin_tool.txtar + OaiResponsesStreamingSummaryAndCommentaryBuiltinTool []byte + //go:embed openai/responses/streaming/cached_input_tokens.txtar OaiResponsesStreamingCachedInputTokens []byte @@ -111,6 +138,9 @@ var ( //go:embed openai/responses/streaming/prev_response_id.txtar OaiResponsesStreamingPrevResponseID []byte + //go:embed openai/responses/streaming/single_builtin_tool_parallel.txtar + OaiResponsesStreamingSingleBuiltinToolParallel []byte + //go:embed openai/responses/streaming/single_injected_tool.txtar OaiResponsesStreamingSingleInjectedTool []byte diff --git a/fixtures/openai/responses/blocking/commentary_builtin_tool.txtar b/fixtures/openai/responses/blocking/commentary_builtin_tool.txtar new file mode 100644 index 00000000..d0e83dd7 --- /dev/null +++ b/fixtures/openai/responses/blocking/commentary_builtin_tool.txtar @@ -0,0 +1,139 @@ +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Use the add function to calculate the sum." + } + ], + "model": "gpt-5.4", + "stream": false, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- non-streaming -- +{ + "id": "resp_0aba2ac43dc240b30169b15720243c819ebb64977365d42cf5", + "object": "response", + "created_at": 1773229856, + "status": "completed", + "background": false, + "completed_at": 1773229861, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5.4-2026-03-05", + "output": [ + { + "id": "rs_0aba2ac43dc240b30169b157208c88819e8238a91b5f7a919b", + "type": "reasoning", + "status": "completed", + "encrypted_content": "gAAAAA==", + "summary": [] + }, + { + "id": "msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "text": "Checking whether 3 + 5 is prime by calling the add function first." + } + ], + "phase": "commentary", + "role": "assistant" + }, + { + "id": "fc_0aba2ac43dc240b30169b157255604819e8a108124efc1635c", + "type": "function_call", + "status": "completed", + "arguments": "{\"a\":3,\"b\":5}", + "call_id": "call_A8TkZmIcKtw2Zw952Wc5QVe7", + "name": "add" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "xhigh", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "low" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "description": "Add two numbers together.", + "name": "add", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ], + "additionalProperties": false + }, + "strict": true + } + ], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 58, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 30, + "output_tokens_details": { + "reasoning_tokens": 10 + }, + "total_tokens": 88 + }, + "user": null, + "metadata": {} +} diff --git a/fixtures/openai/responses/blocking/multi_reasoning_builtin_tool.txtar b/fixtures/openai/responses/blocking/multi_reasoning_builtin_tool.txtar new file mode 100644 index 00000000..022b433e --- /dev/null +++ b/fixtures/openai/responses/blocking/multi_reasoning_builtin_tool.txtar @@ -0,0 +1,142 @@ +Two reasoning output items before a function_call. + +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Use the add function to calculate the sum." + } + ], + "model": "gpt-4.1", + "stream": false, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- non-streaming -- +{ + "id": "resp_0da6045a8b68fa5200695fa23dcc2c81a19c849f627abf8a31", + "object": "response", + "created_at": 1767875133, + "status": "completed", + "background": false, + "completed_at": 1767875134, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "rs_0da6045a8b68fa5200695fa23e100081a19bf68887d47ae93d", + "type": "reasoning", + "status": "completed", + "summary": [ + { + "type": "summary_text", + "text": "The user wants to add 3 and 5. Let me call the add function." + } + ] + }, + { + "id": "rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e", + "type": "reasoning", + "status": "completed", + "summary": [ + { + "type": "summary_text", + "text": "After adding, I will check if the result is prime." + } + ] + }, + { + "id": "fc_0da6045a8b68fa5200695fa23e198081a19bf68887d47ae93d", + "type": "function_call", + "status": "completed", + "arguments": "{\"a\":3,\"b\":5}", + "call_id": "call_CJSaa2u51JG996575oVljuNq", + "name": "add" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "description": "Add two numbers together.", + "name": "add", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ], + "additionalProperties": false + }, + "strict": true + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 58, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 18, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 76 + }, + "user": null, + "metadata": {} +} diff --git a/fixtures/openai/responses/blocking/single_builtin_tool.txtar b/fixtures/openai/responses/blocking/single_builtin_tool.txtar index f41bd3cc..14299ff3 100644 --- a/fixtures/openai/responses/blocking/single_builtin_tool.txtar +++ b/fixtures/openai/responses/blocking/single_builtin_tool.txtar @@ -50,6 +50,17 @@ "max_tool_calls": null, "model": "gpt-4.1-2025-04-14", "output": [ + { + "id": "rs_0da6045a8b68fa5200695fa23e100081a19bf68887d47ae93d", + "type": "reasoning", + "status": "completed", + "summary": [ + { + "type": "summary_text", + "text": "The user wants to add 3 and 5. Let me call the add function." + } + ] + }, { "id": "fc_0da6045a8b68fa5200695fa23e198081a19bf68887d47ae93d", "type": "function_call", diff --git a/fixtures/openai/responses/blocking/single_builtin_tool_parallel.txtar b/fixtures/openai/responses/blocking/single_builtin_tool_parallel.txtar new file mode 100644 index 00000000..4be0d240 --- /dev/null +++ b/fixtures/openai/responses/blocking/single_builtin_tool_parallel.txtar @@ -0,0 +1,140 @@ +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Also add 10 + 20. Use the add function for both." + } + ], + "model": "gpt-4.1", + "stream": false, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- non-streaming -- +{ + "id": "resp_parallel_blocking_001", + "object": "response", + "created_at": 1767875133, + "status": "completed", + "background": false, + "billing": { + "payer": "developer" + }, + "completed_at": 1767875134, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4.1-2025-04-14", + "output": [ + { + "id": "rs_parallel_blocking_reasoning_001", + "type": "reasoning", + "status": "completed", + "summary": [ + { + "type": "summary_text", + "text": "The user wants two additions: 3+5 and 10+20. I'll call add for both." + } + ] + }, + { + "id": "fc_parallel_blocking_first_001", + "type": "function_call", + "status": "completed", + "arguments": "{\"a\":3,\"b\":5}", + "call_id": "call_ParallelBlockingFirst001", + "name": "add" + }, + { + "id": "fc_parallel_blocking_second_001", + "type": "function_call", + "status": "completed", + "arguments": "{\"a\":10,\"b\":20}", + "call_id": "call_ParallelBlockingSecond01", + "name": "add" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "description": "Add two numbers together.", + "name": "add", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ], + "additionalProperties": false + }, + "strict": true + } + ], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 65, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 30, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 95 + }, + "user": null, + "metadata": {} +} diff --git a/fixtures/openai/responses/blocking/summary_and_commentary_builtin_tool.txtar b/fixtures/openai/responses/blocking/summary_and_commentary_builtin_tool.txtar new file mode 100644 index 00000000..15082c36 --- /dev/null +++ b/fixtures/openai/responses/blocking/summary_and_commentary_builtin_tool.txtar @@ -0,0 +1,146 @@ +Both a reasoning summary and a commentary message before a function_call. + +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Use the add function to calculate the sum." + } + ], + "model": "gpt-5.4", + "stream": false, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- non-streaming -- +{ + "id": "resp_1bba3bc54ed351c41270c26831354d920fcc75088476e53de6", + "object": "response", + "created_at": 1773229900, + "status": "completed", + "background": false, + "completed_at": 1773229905, + "error": null, + "incomplete_details": null, + "instructions": null, + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-5.4-2026-03-05", + "output": [ + { + "id": "rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6", + "type": "reasoning", + "status": "completed", + "encrypted_content": "gAAAAA==", + "summary": [ + { + "type": "summary_text", + "text": "I need to add 3 and 5 to check primality." + } + ] + }, + { + "id": "msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "text": "Let me calculate the sum first using the add function." + } + ], + "phase": "commentary", + "role": "assistant" + }, + { + "id": "fc_1bba3bc54ed351c41270c26831b0ad920fee97200698074f08", + "type": "function_call", + "status": "completed", + "arguments": "{\"a\":3,\"b\":5}", + "call_id": "call_B9UjYX01Lvvv1XwjDsdmRW3f", + "name": "add" + } + ], + "parallel_tool_calls": true, + "previous_response_id": null, + "prompt_cache_key": null, + "prompt_cache_retention": null, + "reasoning": { + "effort": "xhigh", + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": false, + "temperature": 1.0, + "text": { + "format": { + "type": "text" + }, + "verbosity": "low" + }, + "tool_choice": "auto", + "tools": [ + { + "type": "function", + "description": "Add two numbers together.", + "name": "add", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ], + "additionalProperties": false + }, + "strict": true + } + ], + "top_logprobs": 0, + "top_p": 0.98, + "truncation": "disabled", + "usage": { + "input_tokens": 58, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 35, + "output_tokens_details": { + "reasoning_tokens": 10 + }, + "total_tokens": 93 + }, + "user": null, + "metadata": {} +} diff --git a/fixtures/openai/responses/streaming/builtin_tool.txtar b/fixtures/openai/responses/streaming/builtin_tool.txtar index b6a7a0a5..98793f3b 100644 --- a/fixtures/openai/responses/streaming/builtin_tool.txtar +++ b/fixtures/openai/responses/streaming/builtin_tool.txtar @@ -40,41 +40,59 @@ event: response.in_progress data: {"type":"response.in_progress","response":{"id":"resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458","object":"response","created_at":1767875312,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} event: response.output_item.added -data: {"type":"response.output_item.added","item":{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"in_progress","arguments":"","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"},"output_index":0,"sequence_number":2} +data: {"type":"response.output_item.added","item":{"id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","type":"reasoning","status":"in_progress","summary":[]},"output_index":0,"sequence_number":2} + +event: response.reasoning_summary_part.added +data: {"type":"response.reasoning_summary_part.added","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"part":{"type":"summary_text","text":""},"summary_index":0,"sequence_number":3} + +event: response.reasoning_summary_text.delta +data: {"type":"response.reasoning_summary_text.delta","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"summary_index":0,"delta":"The user wants to add 3 and 5. Let me call the add function.","sequence_number":4} + +event: response.reasoning_summary_text.done +data: {"type":"response.reasoning_summary_text.done","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"summary_index":0,"text":"The user wants to add 3 and 5. Let me call the add function.","sequence_number":5} + +event: response.reasoning_summary_part.done +data: {"type":"response.reasoning_summary_part.done","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"part":{"type":"summary_text","text":"The user wants to add 3 and 5. Let me call the add function."},"summary_index":0,"sequence_number":6} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"The user wants to add 3 and 5. Let me call the add function."}]},"output_index":0,"sequence_number":7} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"in_progress","arguments":"","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"},"output_index":1,"sequence_number":8} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"gWZHP8i4lSgQYT","output_index":0,"sequence_number":3} +data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"gWZHP8i4lSgQYT","output_index":1,"sequence_number":9} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"a","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"yC1iubuqc098ZSH","output_index":0,"sequence_number":4} +data: {"type":"response.function_call_arguments.delta","delta":"a","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"yC1iubuqc098ZSH","output_index":1,"sequence_number":10} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"G17nNbWUcJkqA2","output_index":0,"sequence_number":5} +data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"G17nNbWUcJkqA2","output_index":1,"sequence_number":11} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"3","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"Mj71L4eeLZbIEFU","output_index":0,"sequence_number":6} +data: {"type":"response.function_call_arguments.delta","delta":"3","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"Mj71L4eeLZbIEFU","output_index":1,"sequence_number":12} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":",\"","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"ZchcCauvlPtVc7","output_index":0,"sequence_number":7} +data: {"type":"response.function_call_arguments.delta","delta":",\"","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"ZchcCauvlPtVc7","output_index":1,"sequence_number":13} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"b","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"gWLYMrsBI3ZHKVP","output_index":0,"sequence_number":8} +data: {"type":"response.function_call_arguments.delta","delta":"b","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"gWLYMrsBI3ZHKVP","output_index":1,"sequence_number":14} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"n4iUzpnbPE4DnO","output_index":0,"sequence_number":9} +data: {"type":"response.function_call_arguments.delta","delta":"\":","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"n4iUzpnbPE4DnO","output_index":1,"sequence_number":15} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"5","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"23mO3rxkXqDOi6g","output_index":0,"sequence_number":10} +data: {"type":"response.function_call_arguments.delta","delta":"5","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"23mO3rxkXqDOi6g","output_index":1,"sequence_number":16} event: response.function_call_arguments.delta -data: {"type":"response.function_call_arguments.delta","delta":"}","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"AQnBsNz7GqkdylH","output_index":0,"sequence_number":11} +data: {"type":"response.function_call_arguments.delta","delta":"}","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"AQnBsNz7GqkdylH","output_index":1,"sequence_number":17} event: response.function_call_arguments.done -data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":3,\"b\":5}","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","output_index":0,"sequence_number":12} +data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":3,\"b\":5}","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","output_index":1,"sequence_number":18} event: response.output_item.done -data: {"type":"response.output_item.done","item":{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"},"output_index":0,"sequence_number":13} +data: {"type":"response.output_item.done","item":{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"},"output_index":1,"sequence_number":19} event: response.completed -data: {"type":"response.completed","response":{"id":"resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458","object":"response","created_at":1767875312,"status":"completed","background":false,"completed_at":1767875312,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":18,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":76},"user":null,"metadata":{}},"sequence_number":14} +data: {"type":"response.completed","response":{"id":"resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458","object":"response","created_at":1767875312,"status":"completed","background":false,"completed_at":1767875312,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[{"id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"The user wants to add 3 and 5. Let me call the add function."}]},{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":18,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":76},"user":null,"metadata":{}},"sequence_number":20} diff --git a/fixtures/openai/responses/streaming/commentary_builtin_tool.txtar b/fixtures/openai/responses/streaming/commentary_builtin_tool.txtar new file mode 100644 index 00000000..2f090f62 --- /dev/null +++ b/fixtures/openai/responses/streaming/commentary_builtin_tool.txtar @@ -0,0 +1,80 @@ +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Use the add function to calculate the sum." + } + ], + "model": "gpt-5.4", + "stream": true, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- streaming -- +event: response.created +data: {"type":"response.created","response":{"id":"resp_0aba2ac43dc240b30169b15720243c819ebb64977365d42cf5","object":"response","created_at":1773229856,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-2026-03-05","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"xhigh","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"low"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_0aba2ac43dc240b30169b15720243c819ebb64977365d42cf5","object":"response","created_at":1773229856,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-2026-03-05","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"xhigh","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"low"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"rs_0aba2ac43dc240b30169b157208c88819e8238a91b5f7a919b","type":"reasoning","status":"in_progress","summary":[]},"output_index":0,"sequence_number":2} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"rs_0aba2ac43dc240b30169b157208c88819e8238a91b5f7a919b","type":"reasoning","status":"completed","encrypted_content":"gAAAAA==","summary":[]},"output_index":0,"sequence_number":3} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f","type":"message","status":"in_progress","content":[],"phase":"commentary","role":"assistant"},"output_index":1,"sequence_number":4} + +event: response.content_part.added +data: {"type":"response.content_part.added","item_id":"msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f","output_index":1,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]},"sequence_number":5} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","item_id":"msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f","output_index":1,"content_index":0,"delta":"Checking whether 3 + 5 is prime by calling the add function first.","sequence_number":6} + +event: response.output_text.done +data: {"type":"response.output_text.done","item_id":"msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f","output_index":1,"content_index":0,"text":"Checking whether 3 + 5 is prime by calling the add function first.","sequence_number":7} + +event: response.content_part.done +data: {"type":"response.content_part.done","item_id":"msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f","output_index":1,"content_index":0,"part":{"type":"output_text","text":"Checking whether 3 + 5 is prime by calling the add function first.","annotations":[]},"sequence_number":8} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Checking whether 3 + 5 is prime by calling the add function first."}],"phase":"commentary","role":"assistant"},"output_index":1,"sequence_number":9} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_0aba2ac43dc240b30169b157255604819e8a108124efc1635c","type":"function_call","status":"in_progress","arguments":"","call_id":"call_A8TkZmIcKtw2Zw952Wc5QVe7","name":"add"},"output_index":2,"sequence_number":10} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"{\"a\":3,\"b\":5}","item_id":"fc_0aba2ac43dc240b30169b157255604819e8a108124efc1635c","output_index":2,"sequence_number":11} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":3,\"b\":5}","item_id":"fc_0aba2ac43dc240b30169b157255604819e8a108124efc1635c","output_index":2,"sequence_number":12} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"fc_0aba2ac43dc240b30169b157255604819e8a108124efc1635c","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_A8TkZmIcKtw2Zw952Wc5QVe7","name":"add"},"output_index":2,"sequence_number":13} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_0aba2ac43dc240b30169b15720243c819ebb64977365d42cf5","object":"response","created_at":1773229856,"status":"completed","background":false,"completed_at":1773229861,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-2026-03-05","output":[{"id":"rs_0aba2ac43dc240b30169b157208c88819e8238a91b5f7a919b","type":"reasoning","status":"completed","encrypted_content":"gAAAAA==","summary":[]},{"id":"msg_0aba2ac43dc240b30169b1572286d0819eb24b1d0f84c8fb3f","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Checking whether 3 + 5 is prime by calling the add function first."}],"phase":"commentary","role":"assistant"},{"id":"fc_0aba2ac43dc240b30169b157255604819e8a108124efc1635c","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_A8TkZmIcKtw2Zw952Wc5QVe7","name":"add"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"xhigh","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"low"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":30,"output_tokens_details":{"reasoning_tokens":10},"total_tokens":88},"user":null,"metadata":{}},"sequence_number":14} + diff --git a/fixtures/openai/responses/streaming/multi_reasoning_builtin_tool.txtar b/fixtures/openai/responses/streaming/multi_reasoning_builtin_tool.txtar new file mode 100644 index 00000000..b54ebc7a --- /dev/null +++ b/fixtures/openai/responses/streaming/multi_reasoning_builtin_tool.txtar @@ -0,0 +1,94 @@ +Two reasoning output items before a function_call. + +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Use the add function to calculate the sum." + } + ], + "model": "gpt-4.1", + "stream": true, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- streaming -- +event: response.created +data: {"type":"response.created","response":{"id":"resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458","object":"response","created_at":1767875312,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458","object":"response","created_at":1767875312,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","type":"reasoning","status":"in_progress","summary":[]},"output_index":0,"sequence_number":2} + +event: response.reasoning_summary_part.added +data: {"type":"response.reasoning_summary_part.added","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"part":{"type":"summary_text","text":""},"summary_index":0,"sequence_number":3} + +event: response.reasoning_summary_text.delta +data: {"type":"response.reasoning_summary_text.delta","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"summary_index":0,"delta":"The user wants to add 3 and 5. Let me call the add function.","sequence_number":4} + +event: response.reasoning_summary_text.done +data: {"type":"response.reasoning_summary_text.done","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"summary_index":0,"text":"The user wants to add 3 and 5. Let me call the add function.","sequence_number":5} + +event: response.reasoning_summary_part.done +data: {"type":"response.reasoning_summary_part.done","item_id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","output_index":0,"part":{"type":"summary_text","text":"The user wants to add 3 and 5. Let me call the add function."},"summary_index":0,"sequence_number":6} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"The user wants to add 3 and 5. Let me call the add function."}]},"output_index":0,"sequence_number":7} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e","type":"reasoning","status":"in_progress","summary":[]},"output_index":1,"sequence_number":8} + +event: response.reasoning_summary_part.added +data: {"type":"response.reasoning_summary_part.added","item_id":"rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e","output_index":1,"part":{"type":"summary_text","text":""},"summary_index":0,"sequence_number":9} + +event: response.reasoning_summary_text.delta +data: {"type":"response.reasoning_summary_text.delta","item_id":"rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e","output_index":1,"summary_index":0,"delta":"After adding, I will check if the result is prime.","sequence_number":10} + +event: response.reasoning_summary_text.done +data: {"type":"response.reasoning_summary_text.done","item_id":"rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e","output_index":1,"summary_index":0,"text":"After adding, I will check if the result is prime.","sequence_number":11} + +event: response.reasoning_summary_part.done +data: {"type":"response.reasoning_summary_part.done","item_id":"rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e","output_index":1,"part":{"type":"summary_text","text":"After adding, I will check if the result is prime."},"summary_index":0,"sequence_number":12} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"After adding, I will check if the result is prime."}]},"output_index":1,"sequence_number":13} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"in_progress","arguments":"","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"},"output_index":2,"sequence_number":14} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"{\"a\":3,\"b\":5}","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","obfuscation":"gWZHP8i4lSgQYT","output_index":2,"sequence_number":15} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":3,\"b\":5}","item_id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","output_index":2,"sequence_number":16} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"},"output_index":2,"sequence_number":17} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458","object":"response","created_at":1767875312,"status":"completed","background":false,"completed_at":1767875312,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[{"id":"rs_0c3fb28cfcf463a500695fa2f0a0a881a0890103ba88b0628e","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"The user wants to add 3 and 5. Let me call the add function."}]},{"id":"rs_1aa7045a8b68fa5200695fa23e200082b29cf79998e58bf94e","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"After adding, I will check if the result is prime."}]},{"id":"fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_7VaiUXZYuuuwWwviCrckxq6t","name":"add"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":18,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":76},"user":null,"metadata":{}},"sequence_number":18} + diff --git a/fixtures/openai/responses/streaming/single_builtin_tool_parallel.txtar b/fixtures/openai/responses/streaming/single_builtin_tool_parallel.txtar new file mode 100644 index 00000000..0319cab0 --- /dev/null +++ b/fixtures/openai/responses/streaming/single_builtin_tool_parallel.txtar @@ -0,0 +1,86 @@ +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Also add 10 + 20. Use the add function for both." + } + ], + "model": "gpt-4.1", + "stream": true, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- streaming -- +event: response.created +data: {"type":"response.created","response":{"id":"resp_parallel_streaming_001","object":"response","created_at":1767875312,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_parallel_streaming_001","object":"response","created_at":1767875312,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"rs_parallel_streaming_reasoning_001","type":"reasoning","status":"in_progress","summary":[]},"output_index":0,"sequence_number":2} + +event: response.reasoning_summary_part.added +data: {"type":"response.reasoning_summary_part.added","item_id":"rs_parallel_streaming_reasoning_001","output_index":0,"part":{"type":"summary_text","text":""},"summary_index":0,"sequence_number":3} + +event: response.reasoning_summary_text.delta +data: {"type":"response.reasoning_summary_text.delta","item_id":"rs_parallel_streaming_reasoning_001","output_index":0,"summary_index":0,"delta":"The user wants two additions: 3+5 and 10+20. I'll call add for both.","sequence_number":4} + +event: response.reasoning_summary_text.done +data: {"type":"response.reasoning_summary_text.done","item_id":"rs_parallel_streaming_reasoning_001","output_index":0,"summary_index":0,"text":"The user wants two additions: 3+5 and 10+20. I'll call add for both.","sequence_number":5} + +event: response.reasoning_summary_part.done +data: {"type":"response.reasoning_summary_part.done","item_id":"rs_parallel_streaming_reasoning_001","output_index":0,"part":{"type":"summary_text","text":"The user wants two additions: 3+5 and 10+20. I'll call add for both."},"summary_index":0,"sequence_number":6} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"rs_parallel_streaming_reasoning_001","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"The user wants two additions: 3+5 and 10+20. I'll call add for both."}]},"output_index":0,"sequence_number":7} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_parallel_streaming_first_001","type":"function_call","status":"in_progress","arguments":"","call_id":"call_ParallelStreamFirst001","name":"add"},"output_index":1,"sequence_number":8} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"{\"a\":3,\"b\":5}","item_id":"fc_parallel_streaming_first_001","output_index":1,"sequence_number":9} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":3,\"b\":5}","item_id":"fc_parallel_streaming_first_001","output_index":1,"sequence_number":10} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"fc_parallel_streaming_first_001","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_ParallelStreamFirst001","name":"add"},"output_index":1,"sequence_number":11} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_parallel_streaming_second_001","type":"function_call","status":"in_progress","arguments":"","call_id":"call_ParallelStreamSecond01","name":"add"},"output_index":2,"sequence_number":12} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"{\"a\":10,\"b\":20}","item_id":"fc_parallel_streaming_second_001","output_index":2,"sequence_number":13} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":10,\"b\":20}","item_id":"fc_parallel_streaming_second_001","output_index":2,"sequence_number":14} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"fc_parallel_streaming_second_001","type":"function_call","status":"completed","arguments":"{\"a\":10,\"b\":20}","call_id":"call_ParallelStreamSecond01","name":"add"},"output_index":2,"sequence_number":15} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_parallel_streaming_001","object":"response","created_at":1767875312,"status":"completed","background":false,"completed_at":1767875312,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-4.1-2025-04-14","output":[{"id":"rs_parallel_streaming_reasoning_001","type":"reasoning","status":"completed","summary":[{"type":"summary_text","text":"The user wants two additions: 3+5 and 10+20. I'll call add for both."}]},{"id":"fc_parallel_streaming_first_001","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_ParallelStreamFirst001","name":"add"},{"id":"fc_parallel_streaming_second_001","type":"function_call","status":"completed","arguments":"{\"a\":10,\"b\":20}","call_id":"call_ParallelStreamSecond01","name":"add"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":65,"input_tokens_details":{"cached_tokens":0},"output_tokens":30,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":95},"user":null,"metadata":{}},"sequence_number":16} + diff --git a/fixtures/openai/responses/streaming/summary_and_commentary_builtin_tool.txtar b/fixtures/openai/responses/streaming/summary_and_commentary_builtin_tool.txtar new file mode 100644 index 00000000..172b0065 --- /dev/null +++ b/fixtures/openai/responses/streaming/summary_and_commentary_builtin_tool.txtar @@ -0,0 +1,94 @@ +Both a reasoning summary and a commentary message before a function_call. + +-- request -- +{ + "input": [ + { + "role": "user", + "content": "Is 3 + 5 a prime number? Use the add function to calculate the sum." + } + ], + "model": "gpt-5.4", + "stream": true, + "tools": [ + { + "type": "function", + "name": "add", + "description": "Add two numbers together.", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "number" + }, + "b": { + "type": "number" + } + }, + "required": [ + "a", + "b" + ] + } + } + ] +} + +-- streaming -- +event: response.created +data: {"type":"response.created","response":{"id":"resp_1bba3bc54ed351c41270c26831354d920fcc75088476e53de6","object":"response","created_at":1773229900,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-2026-03-05","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"xhigh","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"low"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + +event: response.in_progress +data: {"type":"response.in_progress","response":{"id":"resp_1bba3bc54ed351c41270c26831354d920fcc75088476e53de6","object":"response","created_at":1773229900,"status":"in_progress","background":false,"completed_at":null,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-2026-03-05","output":[],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"xhigh","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"low"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6","type":"reasoning","status":"in_progress","summary":[]},"output_index":0,"sequence_number":2} + +event: response.reasoning_summary_part.added +data: {"type":"response.reasoning_summary_part.added","item_id":"rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6","output_index":0,"part":{"type":"summary_text","text":""},"summary_index":0,"sequence_number":3} + +event: response.reasoning_summary_text.delta +data: {"type":"response.reasoning_summary_text.delta","item_id":"rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6","output_index":0,"summary_index":0,"delta":"I need to add 3 and 5 to check primality.","sequence_number":4} + +event: response.reasoning_summary_text.done +data: {"type":"response.reasoning_summary_text.done","item_id":"rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6","output_index":0,"summary_index":0,"text":"I need to add 3 and 5 to check primality.","sequence_number":5} + +event: response.reasoning_summary_part.done +data: {"type":"response.reasoning_summary_part.done","item_id":"rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6","output_index":0,"part":{"type":"summary_text","text":"I need to add 3 and 5 to check primality."},"summary_index":0,"sequence_number":6} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6","type":"reasoning","status":"completed","encrypted_content":"gAAAAA==","summary":[{"type":"summary_text","text":"I need to add 3 and 5 to check primality."}]},"output_index":0,"sequence_number":7} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7","type":"message","status":"in_progress","content":[],"phase":"commentary","role":"assistant"},"output_index":1,"sequence_number":8} + +event: response.content_part.added +data: {"type":"response.content_part.added","item_id":"msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7","output_index":1,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]},"sequence_number":9} + +event: response.output_text.delta +data: {"type":"response.output_text.delta","item_id":"msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7","output_index":1,"content_index":0,"delta":"Let me calculate the sum first using the add function.","sequence_number":10} + +event: response.output_text.done +data: {"type":"response.output_text.done","item_id":"msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7","output_index":1,"content_index":0,"text":"Let me calculate the sum first using the add function.","sequence_number":11} + +event: response.content_part.done +data: {"type":"response.content_part.done","item_id":"msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7","output_index":1,"content_index":0,"part":{"type":"output_text","text":"Let me calculate the sum first using the add function.","annotations":[]},"sequence_number":12} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Let me calculate the sum first using the add function."}],"phase":"commentary","role":"assistant"},"output_index":1,"sequence_number":13} + +event: response.output_item.added +data: {"type":"response.output_item.added","item":{"id":"fc_1bba3bc54ed351c41270c26831b0ad920fee97200698074f08","type":"function_call","status":"in_progress","arguments":"","call_id":"call_B9UjYX01Lvvv1XwjDsdmRW3f","name":"add"},"output_index":2,"sequence_number":14} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","delta":"{\"a\":3,\"b\":5}","item_id":"fc_1bba3bc54ed351c41270c26831b0ad920fee97200698074f08","output_index":2,"sequence_number":15} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","arguments":"{\"a\":3,\"b\":5}","item_id":"fc_1bba3bc54ed351c41270c26831b0ad920fee97200698074f08","output_index":2,"sequence_number":16} + +event: response.output_item.done +data: {"type":"response.output_item.done","item":{"id":"fc_1bba3bc54ed351c41270c26831b0ad920fee97200698074f08","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_B9UjYX01Lvvv1XwjDsdmRW3f","name":"add"},"output_index":2,"sequence_number":17} + +event: response.completed +data: {"type":"response.completed","response":{"id":"resp_1bba3bc54ed351c41270c26831354d920fcc75088476e53de6","object":"response","created_at":1773229900,"status":"completed","background":false,"completed_at":1773229905,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-2026-03-05","output":[{"id":"rs_1bba3bc54ed351c41270c26831908d920fcc75088476e53de6","type":"reasoning","status":"completed","encrypted_content":"gAAAAA==","summary":[{"type":"summary_text","text":"I need to add 3 and 5 to check primality."}]},{"id":"msg_1bba3bc54ed351c41270c26831a09d920fdd86199587f64ef7","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"text":"Let me calculate the sum first using the add function."}],"phase":"commentary","role":"assistant"},{"id":"fc_1bba3bc54ed351c41270c26831b0ad920fee97200698074f08","type":"function_call","status":"completed","arguments":"{\"a\":3,\"b\":5}","call_id":"call_B9UjYX01Lvvv1XwjDsdmRW3f","name":"add"}],"parallel_tool_calls":true,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":null,"reasoning":{"effort":"xhigh","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"low"},"tool_choice":"auto","tools":[{"type":"function","description":"Add two numbers together.","name":"add","parameters":{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":58,"input_tokens_details":{"cached_tokens":0},"output_tokens":35,"output_tokens_details":{"reasoning_tokens":10},"total_tokens":93},"user":null,"metadata":{}},"sequence_number":18} + diff --git a/intercept/messages/base.go b/intercept/messages/base.go index 37522380..f1a123ad 100644 --- a/intercept/messages/base.go +++ b/intercept/messages/base.go @@ -168,6 +168,29 @@ func (i *interceptionBase) disableParallelToolCalls() { } } +// extractModelThoughts returns any thinking blocks that were returned in the response. +func (i *interceptionBase) extractModelThoughts(msg *anthropic.Message) []*recorder.ModelThoughtRecord { + if msg == nil { + return nil + } + + var thoughtRecords []*recorder.ModelThoughtRecord + for _, block := range msg.Content { + switch variant := block.AsAny().(type) { + case anthropic.ThinkingBlock: + if variant.Thinking == "" { + continue + } + thoughtRecords = append(thoughtRecords, &recorder.ModelThoughtRecord{ + Content: variant.Thinking, + Metadata: recorder.Metadata{"source": recorder.ThoughtSourceThinking}, + }) + } + // anthropic.RedactedThinkingBlock also exists, but there's nothing useful we can capture. + } + return thoughtRecords +} + // IsSmallFastModel checks if the model is a small/fast model (Haiku 3.5). // These models are optimized for tasks like code autocomplete and other small, quick operations. // See `ANTHROPIC_SMALL_FAST_MODEL`: https://docs.anthropic.com/en/docs/claude-code/settings#environment-variables diff --git a/intercept/messages/blocking.go b/intercept/messages/blocking.go index e22b97f8..6d2ed6f8 100644 --- a/intercept/messages/blocking.go +++ b/intercept/messages/blocking.go @@ -135,7 +135,16 @@ func (i *BlockingInterception) ProcessRequest(w http.ResponseWriter, r *http.Req accumulateUsage(&cumulativeUsage, resp.Usage) - // Handle tool calls for non-streaming. + // Capture any thinking blocks that were returned. + for _, t := range i.extractModelThoughts(resp) { + _ = i.recorder.RecordModelThought(ctx, &recorder.ModelThoughtRecord{ + InterceptionID: i.ID().String(), + Content: t.Content, + Metadata: t.Metadata, + }) + } + + // Handle tool calls. var pendingToolCalls []anthropic.ToolUseBlock for _, c := range resp.Content { toolUse := c.AsToolUse() @@ -157,7 +166,6 @@ func (i *BlockingInterception) ProcessRequest(w http.ResponseWriter, r *http.Req Args: toolUse.Input, Injected: false, }) - } // If no injected tool calls, we're done. diff --git a/intercept/messages/streaming.go b/intercept/messages/streaming.go index 4e87fd85..595cfe44 100644 --- a/intercept/messages/streaming.go +++ b/intercept/messages/streaming.go @@ -252,6 +252,17 @@ newStream: // Don't send message_stop until all tools have been called. case string(constant.ValueOf[constant.MessageStop]()): + + // Capture any thinking blocks that were returned. + for _, t := range i.extractModelThoughts(&message) { + _ = i.recorder.RecordModelThought(ctx, &recorder.ModelThoughtRecord{ + InterceptionID: i.ID().String(), + Content: t.Content, + Metadata: t.Metadata, + }) + } + + // Process injected tools. if len(pendingToolCalls) > 0 { // Append the whole message from this stream as context since we'll be sending a new request with the tool results. messages.Messages = append(messages.Messages, message.ToParam()) diff --git a/intercept/responses/base.go b/intercept/responses/base.go index 69db3878..59a14b02 100644 --- a/intercept/responses/base.go +++ b/intercept/responses/base.go @@ -254,6 +254,16 @@ func (i *responsesInterceptionBase) recordUserPrompt(ctx context.Context, respon } } +func (i *responsesInterceptionBase) recordModelThoughts(ctx context.Context, response *responses.Response) { + for _, t := range i.extractModelThoughts(response) { + _ = i.recorder.RecordModelThought(ctx, &recorder.ModelThoughtRecord{ + InterceptionID: i.ID().String(), + Content: t.Content, + Metadata: t.Metadata, + }) + } +} + func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Context, response *responses.Response) { if response == nil { i.logger.Warn(ctx, "got empty response, skipping tool usage recording") @@ -326,6 +336,58 @@ func (i *responsesInterceptionBase) recordTokenUsage(ctx context.Context, respon } } +// extractModelThoughts extracts model thoughts from response output items. +// It captures both reasoning summary items and commentary messages (message +// output items with "phase": "commentary") as model thoughts. +func (i *responsesInterceptionBase) extractModelThoughts(response *responses.Response) []*recorder.ModelThoughtRecord { + if response == nil { + return nil + } + + var thoughts []*recorder.ModelThoughtRecord + for _, item := range response.Output { + switch item.Type { + case string(constant.ValueOf[constant.Reasoning]()): + reasoning := item.AsReasoning() + for _, summary := range reasoning.Summary { + if summary.Text == "" { + continue + } + thoughts = append(thoughts, &recorder.ModelThoughtRecord{ + Content: summary.Text, + Metadata: recorder.Metadata{"source": recorder.ThoughtSourceReasoningSummary}, + }) + } + + case string(constant.ValueOf[constant.Message]()): + // The API sometimes returns commentary messages instead of reasoning + // summaries. These are assistant message output items with "phase": "commentary". + // The SDK doesn't expose a Phase field, so we extract it from raw JSON. + // TODO: revisit when the OpenAI SDK adds a proper Phase field. + raw := item.RawJSON() + if gjson.Get(raw, "role").String() != string(constant.ValueOf[constant.Assistant]()) || + gjson.Get(raw, "phase").String() != "commentary" { + continue + } + msg := item.AsMessage() + for _, part := range msg.Content { + if part.Type != string(constant.ValueOf[constant.OutputText]()) { + continue + } + if part.Text == "" { + continue + } + thoughts = append(thoughts, &recorder.ModelThoughtRecord{ + Content: part.Text, + Metadata: recorder.Metadata{"source": recorder.ThoughtSourceCommentary}, + }) + } + } + } + + return thoughts +} + func (i *responsesInterceptionBase) hasInjectableTools() bool { return i.mcpProxy != nil && len(i.mcpProxy.ListTools()) > 0 } diff --git a/intercept/responses/blocking.go b/intercept/responses/blocking.go index 0c11a541..48491e43 100644 --- a/intercept/responses/blocking.go +++ b/intercept/responses/blocking.go @@ -94,6 +94,7 @@ func (i *BlockingResponsesInterceptor) ProcessRequest(w http.ResponseWriter, r * } i.recordTokenUsage(ctx, response) + i.recordModelThoughts(ctx, response) // Check if there any injected tools to invoke. pending := i.getPendingInjectedToolCalls(response) diff --git a/intercept/responses/injected_tools.go b/intercept/responses/injected_tools.go index c3934fa3..e3720230 100644 --- a/intercept/responses/injected_tools.go +++ b/intercept/responses/injected_tools.go @@ -171,7 +171,7 @@ func (i *responsesInterceptionBase) prepareRequestForAgenticLoop(ctx context.Con return nil } -// getPendingInjectedToolCalls extracts function calls from the response that are managed by MCP proxy +// getPendingInjectedToolCalls extracts function calls from the response that are managed by MCP proxy. func (i *responsesInterceptionBase) getPendingInjectedToolCalls(response *responses.Response) []responses.ResponseFunctionToolCall { var calls []responses.ResponseFunctionToolCall diff --git a/intercept/responses/streaming.go b/intercept/responses/streaming.go index 38f5771b..32ee1f02 100644 --- a/intercept/responses/streaming.go +++ b/intercept/responses/streaming.go @@ -172,6 +172,8 @@ func (i *StreamingResponsesInterceptor) ProcessRequest(w http.ResponseWriter, r // Record token usage for each inner loop iteration i.recordTokenUsage(ctx, completedResponse) } + + i.recordModelThoughts(ctx, completedResponse) } if promptFound { diff --git a/internal/integrationtest/bridge_test.go b/internal/integrationtest/bridge_test.go index 01eb5815..a2a746e3 100644 --- a/internal/integrationtest/bridge_test.go +++ b/internal/integrationtest/bridge_test.go @@ -124,6 +124,107 @@ func TestAnthropicMessages(t *testing.T) { }) } +func TestAnthropicMessagesModelThoughts(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + streaming bool + fixture []byte + expectedThoughts []recorder.ModelThoughtRecord // nil means no model thoughts expected + }{ + { + name: "single thinking block/streaming", + streaming: true, + fixture: fixtures.AntSingleBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants me to read", recorder.ThoughtSourceThinking)}, + }, + { + name: "single thinking block/blocking", + streaming: false, + fixture: fixtures.AntSingleBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants me to read", recorder.ThoughtSourceThinking)}, + }, + { + name: "multiple thinking blocks/streaming", + streaming: true, + fixture: fixtures.AntMultiThinkingBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{ + newModelThought("The user wants me to read", recorder.ThoughtSourceThinking), + newModelThought("I should use the Read tool", recorder.ThoughtSourceThinking), + }, + }, + { + name: "multiple thinking blocks/blocking", + streaming: false, + fixture: fixtures.AntMultiThinkingBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{ + newModelThought("The user wants me to read", recorder.ThoughtSourceThinking), + newModelThought("I should use the Read tool", recorder.ThoughtSourceThinking), + }, + }, + { + name: "parallel tool calls/streaming", + streaming: true, + fixture: fixtures.AntSingleBuiltinToolParallel, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants me to read two files", recorder.ThoughtSourceThinking)}, + }, + { + name: "parallel tool calls/blocking", + streaming: false, + fixture: fixtures.AntSingleBuiltinToolParallel, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants me to read two files", recorder.ThoughtSourceThinking)}, + }, + { + name: "thoughts without tool calls/streaming", + streaming: true, + fixture: fixtures.AntSimple, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("This is a classic philosophical question about medieval scholasticism", recorder.ThoughtSourceThinking)}, + }, + { + name: "thoughts without tool calls/blocking", + streaming: false, + fixture: fixtures.AntSimple, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("This is a classic philosophical question about medieval scholasticism", recorder.ThoughtSourceThinking)}, + }, + { + name: "no thoughts captured", + streaming: false, + fixture: fixtures.AntSingleInjectedTool, + expectedThoughts: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) + t.Cleanup(cancel) + + fix := fixtures.Parse(t, tc.fixture) + upstream := newMockUpstream(t, ctx, newFixtureResponse(fix)) + + bridgeServer := newBridgeTestServer(t, ctx, upstream.URL) + + reqBody, err := sjson.SetBytes(fix.Request(), "stream", tc.streaming) + require.NoError(t, err) + resp := bridgeServer.makeRequest(t, http.MethodPost, pathAnthropicMessages, reqBody) + require.Equal(t, http.StatusOK, resp.StatusCode) + + if tc.streaming { + sp := aibridge.NewSSEParser() + require.NoError(t, sp.Parse(resp.Body)) + assert.Contains(t, sp.AllEvents(), "message_start") + assert.Contains(t, sp.AllEvents(), "message_stop") + } + + bridgeServer.Recorder.VerifyModelThoughtsRecorded(t, tc.expectedThoughts) + bridgeServer.Recorder.VerifyAllInterceptionsEnded(t) + }) + } +} + func TestAWSBedrockIntegration(t *testing.T) { t.Parallel() diff --git a/internal/integrationtest/helpers.go b/internal/integrationtest/helpers.go index 84bd64d5..038e6335 100644 --- a/internal/integrationtest/helpers.go +++ b/internal/integrationtest/helpers.go @@ -6,6 +6,7 @@ import ( "cdr.dev/slog/v3" "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/aibridge/config" + "github.com/coder/aibridge/recorder" ) // anthropicCfg creates a minimal Anthropic config for testing. @@ -53,3 +54,12 @@ func newLogger(t *testing.T) slog.Logger { t.Helper() return slogtest.Make(t, &slogtest.Options{}).Leveled(slog.LevelDebug) } + +func newModelThought(content, source string) recorder.ModelThoughtRecord { + return recorder.ModelThoughtRecord{ + Content: content, + Metadata: recorder.Metadata{ + "source": source, + }, + } +} diff --git a/internal/integrationtest/responses_test.go b/internal/integrationtest/responses_test.go index eee1235f..43e09023 100644 --- a/internal/integrationtest/responses_test.go +++ b/internal/integrationtest/responses_test.go @@ -936,6 +936,107 @@ func TestResponsesInjectedTool(t *testing.T) { } } +func TestResponsesModelThoughts(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + fixture []byte + expectedThoughts []recorder.ModelThoughtRecord // nil means no tool usages expected at all + }{ + { + name: "single reasoning/blocking", + fixture: fixtures.OaiResponsesBlockingSingleBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants to add 3 and 5", recorder.ThoughtSourceReasoningSummary)}, + }, + { + name: "single reasoning/streaming", + fixture: fixtures.OaiResponsesStreamingBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants to add 3 and 5", recorder.ThoughtSourceReasoningSummary)}, + }, + { + name: "multiple reasoning items/blocking", + fixture: fixtures.OaiResponsesBlockingMultiReasoningBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{ + newModelThought("The user wants to add 3 and 5", recorder.ThoughtSourceReasoningSummary), + newModelThought("After adding, I will check if the result is prime", recorder.ThoughtSourceReasoningSummary), + }, + }, + { + name: "multiple reasoning items/streaming", + fixture: fixtures.OaiResponsesStreamingMultiReasoningBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{ + newModelThought("The user wants to add 3 and 5", recorder.ThoughtSourceReasoningSummary), + newModelThought("After adding, I will check if the result is prime", recorder.ThoughtSourceReasoningSummary), + }, + }, + { + name: "commentary/blocking", + fixture: fixtures.OaiResponsesBlockingCommentaryBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("Checking whether 3 + 5 is prime by calling the add function first.", recorder.ThoughtSourceCommentary)}, + }, + { + name: "commentary/streaming", + fixture: fixtures.OaiResponsesStreamingCommentaryBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("Checking whether 3 + 5 is prime by calling the add function first.", recorder.ThoughtSourceCommentary)}, + }, + { + name: "summary and commentary/blocking", + fixture: fixtures.OaiResponsesBlockingSummaryAndCommentaryBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{ + newModelThought("I need to add 3 and 5 to check primality.", recorder.ThoughtSourceReasoningSummary), + newModelThought("Let me calculate the sum first using the add function.", recorder.ThoughtSourceCommentary), + }, + }, + { + name: "summary and commentary/streaming", + fixture: fixtures.OaiResponsesStreamingSummaryAndCommentaryBuiltinTool, + expectedThoughts: []recorder.ModelThoughtRecord{ + newModelThought("I need to add 3 and 5 to check primality.", recorder.ThoughtSourceReasoningSummary), + newModelThought("Let me calculate the sum first using the add function.", recorder.ThoughtSourceCommentary), + }, + }, + { + name: "parallel tool calls/blocking", + fixture: fixtures.OaiResponsesBlockingSingleBuiltinToolParallel, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants two additions", recorder.ThoughtSourceReasoningSummary)}, + }, + { + name: "parallel tool calls/streaming", + fixture: fixtures.OaiResponsesStreamingSingleBuiltinToolParallel, + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("The user wants two additions", recorder.ThoughtSourceReasoningSummary)}, + }, + { + name: "thoughts without tool calls", + fixture: fixtures.OaiResponsesStreamingCodex, // This fixture contains reasoning, but it's not associated with tool calls. + expectedThoughts: []recorder.ModelThoughtRecord{newModelThought("Preparing simple response", recorder.ThoughtSourceReasoningSummary)}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) + t.Cleanup(cancel) + + fix := fixtures.Parse(t, tc.fixture) + upstream := newMockUpstream(t, ctx, newFixtureResponse(fix)) + + bridgeServer := newBridgeTestServer(t, ctx, upstream.URL) + + resp := bridgeServer.makeRequest(t, http.MethodPost, pathOpenAIResponses, fix.Request()) + require.Equal(t, http.StatusOK, resp.StatusCode) + + _, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + bridgeServer.Recorder.VerifyModelThoughtsRecorded(t, tc.expectedThoughts) + bridgeServer.Recorder.VerifyAllInterceptionsEnded(t) + }) + } +} + func requireResponsesError(t *testing.T, code int, message string, body []byte) { var respErr responses.Error err := json.Unmarshal(body, &respErr) diff --git a/internal/integrationtest/trace_test.go b/internal/integrationtest/trace_test.go index 88bec31c..e9b27d64 100644 --- a/internal/integrationtest/trace_test.go +++ b/internal/integrationtest/trace_test.go @@ -51,6 +51,7 @@ func TestTraceAnthropic(t *testing.T) { {"Intercept.RecordPromptUsage", 1, codes.Unset}, {"Intercept.RecordTokenUsage", 1, codes.Unset}, {"Intercept.RecordToolUsage", 1, codes.Unset}, + {"Intercept.RecordModelThought", 1, codes.Unset}, {"Intercept.ProcessRequest.Upstream", 1, codes.Unset}, } @@ -63,41 +64,76 @@ func TestTraceAnthropic(t *testing.T) { {"Intercept.RecordPromptUsage", 1, codes.Unset}, {"Intercept.RecordTokenUsage", 2, codes.Unset}, {"Intercept.RecordToolUsage", 1, codes.Unset}, + {"Intercept.RecordModelThought", 1, codes.Unset}, {"Intercept.ProcessRequest.Upstream", 1, codes.Unset}, } cases := []struct { name string + fixture []byte streaming bool bedrock bool expect []expectTrace }{ { - name: "trace_anthr_non_streaming", - expect: expectNonStreaming, + name: "trace_anthr_non_streaming", + expect: expectNonStreaming, + fixture: fixtures.AntSingleBuiltinTool, }, { name: "trace_bedrock_non_streaming", bedrock: true, expect: expectNonStreaming, + fixture: fixtures.AntSingleBuiltinTool, }, { name: "trace_anthr_streaming", streaming: true, expect: expectStreaming, + fixture: fixtures.AntSingleBuiltinTool, }, { name: "trace_bedrock_streaming", streaming: true, bedrock: true, expect: expectStreaming, + fixture: fixtures.AntSingleBuiltinTool, + }, + { + name: "trace_multi_thinking_non_streaming", + fixture: fixtures.AntMultiThinkingBuiltinTool, + expect: []expectTrace{ + {"Intercept", 1, codes.Unset}, + {"Intercept.CreateInterceptor", 1, codes.Unset}, + {"Intercept.RecordInterception", 1, codes.Unset}, + {"Intercept.ProcessRequest", 1, codes.Unset}, + {"Intercept.RecordInterceptionEnded", 1, codes.Unset}, + {"Intercept.RecordPromptUsage", 1, codes.Unset}, + {"Intercept.RecordTokenUsage", 1, codes.Unset}, + {"Intercept.RecordToolUsage", 1, codes.Unset}, + {"Intercept.RecordModelThought", 2, codes.Unset}, + {"Intercept.ProcessRequest.Upstream", 1, codes.Unset}, + }, + }, + { + name: "trace_multi_thinking_streaming", + fixture: fixtures.AntMultiThinkingBuiltinTool, + streaming: true, + expect: []expectTrace{ + {"Intercept", 1, codes.Unset}, + {"Intercept.CreateInterceptor", 1, codes.Unset}, + {"Intercept.RecordInterception", 1, codes.Unset}, + {"Intercept.ProcessRequest", 1, codes.Unset}, + {"Intercept.RecordInterceptionEnded", 1, codes.Unset}, + {"Intercept.RecordPromptUsage", 1, codes.Unset}, + {"Intercept.RecordTokenUsage", 2, codes.Unset}, + {"Intercept.RecordToolUsage", 1, codes.Unset}, + {"Intercept.RecordModelThought", 2, codes.Unset}, + {"Intercept.ProcessRequest.Upstream", 1, codes.Unset}, + }, }, } - fix := fixtures.Parse(t, fixtures.AntSingleBuiltinTool) - - fixtureReqBody := fix.Request() - for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), time.Second*30) @@ -105,6 +141,7 @@ func TestTraceAnthropic(t *testing.T) { sr, tracer := setupTracer(t) + fix := fixtures.Parse(t, tc.fixture) upstream := newMockUpstream(t, ctx, newFixtureResponse(fix)) opts := []bridgeOption{ @@ -115,7 +152,7 @@ func TestTraceAnthropic(t *testing.T) { } bridgeServer := newBridgeTestServer(t, ctx, upstream.URL, opts...) - reqBody, err := sjson.SetBytes(fixtureReqBody, "stream", tc.streaming) + reqBody, err := sjson.SetBytes(fix.Request(), "stream", tc.streaming) require.NoError(t, err) resp := bridgeServer.makeRequest(t, http.MethodPost, pathAnthropicMessages, reqBody) require.Equal(t, http.StatusOK, resp.StatusCode) @@ -451,6 +488,42 @@ func TestTraceOpenAI(t *testing.T) { {"Intercept.ProcessRequest.Upstream", 1, codes.Unset}, }, }, + { + name: "trace_openai_responses_streaming_with_reasoning", + fixture: fixtures.OaiResponsesStreamingMultiReasoningBuiltinTool, + streaming: true, + path: pathOpenAIResponses, + expect: []expectTrace{ + {"Intercept", 1, codes.Unset}, + {"Intercept.CreateInterceptor", 1, codes.Unset}, + {"Intercept.RecordInterception", 1, codes.Unset}, + {"Intercept.ProcessRequest", 1, codes.Unset}, + {"Intercept.RecordInterceptionEnded", 1, codes.Unset}, + {"Intercept.RecordPromptUsage", 1, codes.Unset}, + {"Intercept.RecordTokenUsage", 1, codes.Unset}, + {"Intercept.RecordToolUsage", 1, codes.Unset}, + {"Intercept.RecordModelThought", 2, codes.Unset}, + {"Intercept.ProcessRequest.Upstream", 1, codes.Unset}, + }, + }, + { + name: "trace_openai_responses_blocking_with_reasoning", + fixture: fixtures.OaiResponsesBlockingMultiReasoningBuiltinTool, + streaming: false, + path: pathOpenAIResponses, + expect: []expectTrace{ + {"Intercept", 1, codes.Unset}, + {"Intercept.CreateInterceptor", 1, codes.Unset}, + {"Intercept.RecordInterception", 1, codes.Unset}, + {"Intercept.ProcessRequest", 1, codes.Unset}, + {"Intercept.RecordInterceptionEnded", 1, codes.Unset}, + {"Intercept.RecordPromptUsage", 1, codes.Unset}, + {"Intercept.RecordTokenUsage", 1, codes.Unset}, + {"Intercept.RecordToolUsage", 1, codes.Unset}, + {"Intercept.RecordModelThought", 2, codes.Unset}, + {"Intercept.ProcessRequest.Upstream", 1, codes.Unset}, + }, + }, } for _, tc := range cases { diff --git a/internal/testutil/mock_recorder.go b/internal/testutil/mock_recorder.go index 09bcac39..991d0904 100644 --- a/internal/testutil/mock_recorder.go +++ b/internal/testutil/mock_recorder.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "slices" + "strings" "sync" "testing" @@ -20,6 +21,7 @@ type MockRecorder struct { tokenUsages []*recorder.TokenUsageRecord userPrompts []*recorder.PromptUsageRecord toolUsages []*recorder.ToolUsageRecord + modelThoughts []*recorder.ModelThoughtRecord interceptionsEnd map[string]*recorder.InterceptionRecordEnded } @@ -64,6 +66,13 @@ func (m *MockRecorder) RecordToolUsage(ctx context.Context, req *recorder.ToolUs return nil } +func (m *MockRecorder) RecordModelThought(ctx context.Context, req *recorder.ModelThoughtRecord) error { + m.mu.Lock() + defer m.mu.Unlock() + m.modelThoughts = append(m.modelThoughts, req) + return nil +} + // RecordedTokenUsages returns a copy of recorded token usages in a thread-safe manner. // Note: This is a shallow clone - the slice is copied but the pointers reference the // same underlying records. This is sufficient for our test assertions which only read @@ -112,6 +121,14 @@ func (m *MockRecorder) RecordedToolUsages() []*recorder.ToolUsageRecord { return slices.Clone(m.toolUsages) } +// RecordedModelThoughts returns a copy of recorded model thoughts in a thread-safe manner. +// Note: This is a shallow clone (see RecordedTokenUsages for details). +func (m *MockRecorder) RecordedModelThoughts() []*recorder.ModelThoughtRecord { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Clone(m.modelThoughts) +} + // RecordedInterceptions returns a copy of recorded interceptions in a thread-safe manner. // Note: This is a shallow clone (see RecordedTokenUsages for details). func (m *MockRecorder) RecordedInterceptions() []*recorder.InterceptionRecord { @@ -147,3 +164,28 @@ func (m *MockRecorder) VerifyAllInterceptionsEnded(t *testing.T) { require.Containsf(t, m.interceptionsEnd, intc.ID, "interception with id: %v has not been ended", intc.ID) } } + +func (m *MockRecorder) VerifyModelThoughtsRecorded(t *testing.T, expected []recorder.ModelThoughtRecord) { + thoughts := m.RecordedModelThoughts() + if expected == nil { + require.Empty(t, thoughts) + return + } + + require.Len(t, thoughts, len(expected), "unexpected number of model thoughts") + + // We can't guarantee the order of model thoughts since they're recorded separately, so + // we have to scan all thoughts for a match. + + for _, exp := range expected { + var matched *recorder.ModelThoughtRecord + for _, thought := range thoughts { + if strings.Contains(thought.Content, exp.Content) { + matched = thought + } + } + + require.NotNil(t, matched, "could not find thought matching %q", exp.Content) + require.EqualValues(t, exp.Metadata, matched.Metadata) + } +} diff --git a/recorder/recorder.go b/recorder/recorder.go index 6e37b632..c4f427c5 100644 --- a/recorder/recorder.go +++ b/recorder/recorder.go @@ -116,6 +116,24 @@ func (r *RecorderWrapper) RecordToolUsage(ctx context.Context, req *ToolUsageRec return err } +func (r *RecorderWrapper) RecordModelThought(ctx context.Context, req *ModelThoughtRecord) (outErr error) { + ctx, span := r.tracer.Start(ctx, "Intercept.RecordModelThought", trace.WithAttributes(tracing.InterceptionAttributesFromContext(ctx)...)) + defer tracing.EndSpanErr(span, &outErr) + + client, err := r.clientFn() + if err != nil { + return fmt.Errorf("acquire client: %w", err) + } + + req.CreatedAt = time.Now() + if err = client.RecordModelThought(ctx, req); err == nil { + return nil + } + + r.logger.Warn(ctx, "failed to record model thought", slog.Error(err), slog.F("interception_id", req.InterceptionID)) + return err +} + func NewRecorder(logger slog.Logger, tracer trace.Tracer, clientFn func() (Recorder, error)) *RecorderWrapper { return &RecorderWrapper{ logger: logger, @@ -259,6 +277,22 @@ func (a *AsyncRecorder) RecordToolUsage(ctx context.Context, req *ToolUsageRecor return nil // Caller is not interested in error. } +func (a *AsyncRecorder) RecordModelThought(ctx context.Context, req *ModelThoughtRecord) error { + a.wg.Add(1) + go func() { + defer a.wg.Done() + timedCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), a.timeout) + defer cancel() + + err := a.wrapped.RecordModelThought(timedCtx, req) + if err != nil { + a.logger.Warn(timedCtx, "failed to record model thought", slog.F("type", "model_thought"), slog.Error(err), slog.F("payload", req)) + } + }() + + return nil // Caller is not interested in error. +} + func (a *AsyncRecorder) Wait() { a.wg.Wait() } diff --git a/recorder/types.go b/recorder/types.go index b33494d4..20e735f4 100644 --- a/recorder/types.go +++ b/recorder/types.go @@ -19,6 +19,8 @@ type Recorder interface { RecordPromptUsage(ctx context.Context, req *PromptUsageRecord) error // RecordToolUsage records the tools used in an interception with an upstream AI provider. RecordToolUsage(ctx context.Context, req *ToolUsageRecord) error + // RecordModelThought records model thoughts produced in an interception with an upstream AI provider. + RecordModelThought(ctx context.Context, req *ModelThoughtRecord) error } type ToolArgs any @@ -73,3 +75,17 @@ type ToolUsageRecord struct { Metadata Metadata CreatedAt time.Time } + +// Model thought source constants. +const ( + ThoughtSourceThinking = "thinking" + ThoughtSourceReasoningSummary = "reasoning_summary" + ThoughtSourceCommentary = "commentary" +) + +type ModelThoughtRecord struct { + InterceptionID string + Content string + Metadata Metadata + CreatedAt time.Time +}