diff --git a/src/claude_agent_sdk/_internal/message_parser.py b/src/claude_agent_sdk/_internal/message_parser.py index 06cf6366..b34098ac 100644 --- a/src/claude_agent_sdk/_internal/message_parser.py +++ b/src/claude_agent_sdk/_internal/message_parser.py @@ -1,5 +1,6 @@ """Message parser for Claude Code SDK responses.""" +import json import logging from typing import Any @@ -20,6 +21,79 @@ logger = logging.getLogger(__name__) +# Common wrapper keys that models may add around structured output +# See: https://github.com/anthropics/claude-agent-sdk-python/issues/502 +_WRAPPER_KEYS = frozenset({"output", "response", "json", "data", "result"}) + + +def _normalize_structured_output(value: Any) -> Any: + """Normalize structured output by unwrapping common wrapper keys and parsing stringified JSON. + + This handles two common issues with model-generated structured output: + + 1. Wrapper keys (#502): Model wraps data in {"output": {...}}, {"response": {...}}, etc. + We unwrap these to return just the inner data. + + 2. Stringified JSON (#510): Model serializes arrays/objects as JSON strings like + "[{\\"field\\": ...}]" instead of native arrays. We parse these back to native types. + + Args: + value: The raw structured_output value from the CLI + + Returns: + Normalized structured output with wrappers removed and strings parsed + """ + if value is None: + return None + + # Handle wrapper keys: {"output": {...}} -> {...} + if isinstance(value, dict) and len(value) == 1: + key = next(iter(value.keys())) + if key.lower() in _WRAPPER_KEYS: + logger.debug(f"Unwrapping structured_output from '{key}' wrapper") + value = value[key] + + # Recursively normalize the value (handles nested stringified JSON) + return _parse_stringified_json(value) + + +def _parse_stringified_json(value: Any) -> Any: + """Recursively parse stringified JSON values back to native Python types. + + Handles cases where the model serializes arrays/objects as strings: + - "[{\\"field\\": \\"value\\"}]" -> [{"field": "value"}] + - "{\\"key\\": \\"value\\"}" -> {"key": "value"} + + Args: + value: Any value that may contain stringified JSON + + Returns: + Value with stringified JSON parsed to native types + """ + if isinstance(value, str): + # Try to parse strings that look like JSON arrays or objects + stripped = value.strip() + if (stripped.startswith("[") and stripped.endswith("]")) or ( + stripped.startswith("{") and stripped.endswith("}") + ): + try: + parsed = json.loads(value) + logger.debug("Parsed stringified JSON in structured_output") + # Recursively normalize the parsed value + return _parse_stringified_json(parsed) + except json.JSONDecodeError: + # Not valid JSON, return as-is + pass + return value + + if isinstance(value, dict): + return {k: _parse_stringified_json(v) for k, v in value.items()} + + if isinstance(value, list): + return [_parse_stringified_json(item) for item in value] + + return value + def parse_message(data: dict[str, Any]) -> Message: """ @@ -146,6 +220,12 @@ def parse_message(data: dict[str, Any]) -> Message: case "result": try: + # Normalize structured_output to handle wrapper keys and stringified JSON + # See: https://github.com/anthropics/claude-agent-sdk-python/issues/502 + # See: https://github.com/anthropics/claude-agent-sdk-python/issues/510 + raw_structured_output = data.get("structured_output") + normalized_output = _normalize_structured_output(raw_structured_output) + return ResultMessage( subtype=data["subtype"], duration_ms=data["duration_ms"], @@ -156,7 +236,7 @@ def parse_message(data: dict[str, Any]) -> Message: total_cost_usd=data.get("total_cost_usd"), usage=data.get("usage"), result=data.get("result"), - structured_output=data.get("structured_output"), + structured_output=normalized_output, ) except KeyError as e: raise MessageParseError( diff --git a/tests/test_structured_output_normalization.py b/tests/test_structured_output_normalization.py new file mode 100644 index 00000000..83e75408 --- /dev/null +++ b/tests/test_structured_output_normalization.py @@ -0,0 +1,251 @@ +"""Tests for structured output normalization. + +Tests the fixes for: +- Issue #502: Model wraps data in 'output'/'response'/'json' field +- Issue #510: Model serializes arrays as JSON strings +""" + +import pytest + +from claude_agent_sdk._internal.message_parser import ( + _normalize_structured_output, + _parse_stringified_json, + parse_message, +) +from claude_agent_sdk.types import ResultMessage + + +class TestNormalizeStructuredOutput: + """Tests for _normalize_structured_output function.""" + + def test_returns_none_for_none_input(self): + """Test that None input returns None.""" + assert _normalize_structured_output(None) is None + + def test_unwraps_output_wrapper(self): + """Test unwrapping {'output': {...}} wrapper (Issue #502).""" + wrapped = {"output": {"actions": [1, 2, 3]}} + result = _normalize_structured_output(wrapped) + assert result == {"actions": [1, 2, 3]} + + def test_unwraps_response_wrapper(self): + """Test unwrapping {'response': {...}} wrapper (Issue #502).""" + wrapped = {"response": {"actions": [1, 2, 3]}} + result = _normalize_structured_output(wrapped) + assert result == {"actions": [1, 2, 3]} + + def test_unwraps_json_wrapper(self): + """Test unwrapping {'json': {...}} wrapper (Issue #502).""" + wrapped = {"json": {"actions": [1, 2, 3]}} + result = _normalize_structured_output(wrapped) + assert result == {"actions": [1, 2, 3]} + + def test_unwraps_data_wrapper(self): + """Test unwrapping {'data': {...}} wrapper.""" + wrapped = {"data": {"actions": [1, 2, 3]}} + result = _normalize_structured_output(wrapped) + assert result == {"actions": [1, 2, 3]} + + def test_unwraps_result_wrapper(self): + """Test unwrapping {'result': {...}} wrapper.""" + wrapped = {"result": {"actions": [1, 2, 3]}} + result = _normalize_structured_output(wrapped) + assert result == {"actions": [1, 2, 3]} + + def test_case_insensitive_wrapper_keys(self): + """Test that wrapper key detection is case-insensitive.""" + wrapped = {"OUTPUT": {"actions": [1, 2, 3]}} + result = _normalize_structured_output(wrapped) + assert result == {"actions": [1, 2, 3]} + + def test_does_not_unwrap_multi_key_dict(self): + """Test that dicts with multiple keys are not unwrapped.""" + not_wrapped = {"output": {"a": 1}, "other": {"b": 2}} + result = _normalize_structured_output(not_wrapped) + assert result == {"output": {"a": 1}, "other": {"b": 2}} + + def test_does_not_unwrap_non_wrapper_single_key(self): + """Test that single-key dicts with non-wrapper keys are not unwrapped.""" + not_wrapped = {"actions": [1, 2, 3]} + result = _normalize_structured_output(not_wrapped) + assert result == {"actions": [1, 2, 3]} + + def test_preserves_valid_structured_output(self): + """Test that already-valid structured output is preserved.""" + valid = {"actions": [{"type": "click", "target": "button"}]} + result = _normalize_structured_output(valid) + assert result == valid + + +class TestParseStringifiedJson: + """Tests for _parse_stringified_json function.""" + + def test_parses_stringified_array(self): + """Test parsing stringified JSON array (Issue #510).""" + stringified = '[{"field": "value"}, {"field": "other"}]' + result = _parse_stringified_json(stringified) + assert result == [{"field": "value"}, {"field": "other"}] + + def test_parses_stringified_object(self): + """Test parsing stringified JSON object.""" + stringified = '{"key": "value", "number": 42}' + result = _parse_stringified_json(stringified) + assert result == {"key": "value", "number": 42} + + def test_parses_nested_stringified_json(self): + """Test parsing nested stringified JSON in a dict value (Issue #510).""" + data = { + "name": "Section", + "items": '[{"field": {"text": "hello"}}]', + } + result = _parse_stringified_json(data) + assert result == { + "name": "Section", + "items": [{"field": {"text": "hello"}}], + } + + def test_parses_deeply_nested_stringified_json(self): + """Test parsing deeply nested stringified JSON.""" + data = { + "level1": { + "level2": '{"level3": [1, 2, 3]}', + } + } + result = _parse_stringified_json(data) + assert result == { + "level1": { + "level2": {"level3": [1, 2, 3]}, + } + } + + def test_preserves_regular_strings(self): + """Test that regular strings are preserved.""" + assert _parse_stringified_json("hello world") == "hello world" + assert _parse_stringified_json("not json") == "not json" + + def test_preserves_invalid_json_strings(self): + """Test that invalid JSON-looking strings are preserved.""" + invalid = "[this is not valid json]" + assert _parse_stringified_json(invalid) == invalid + + def test_handles_empty_array_string(self): + """Test parsing empty array string.""" + assert _parse_stringified_json("[]") == [] + + def test_handles_empty_object_string(self): + """Test parsing empty object string.""" + assert _parse_stringified_json("{}") == {} + + def test_handles_whitespace_around_json(self): + """Test parsing JSON with surrounding whitespace.""" + result = _parse_stringified_json(' [1, 2, 3] ') + assert result == [1, 2, 3] + + +class TestCombinedNormalization: + """Tests for combined wrapper unwrapping and JSON parsing.""" + + def test_unwraps_and_parses_combined(self): + """Test unwrapping wrapper AND parsing stringified JSON inside.""" + data = { + "output": { + "name": "Section", + "items": '[{"field": "value"}]', + } + } + result = _normalize_structured_output(data) + assert result == { + "name": "Section", + "items": [{"field": "value"}], + } + + def test_complex_real_world_case(self): + """Test a complex real-world case from Issue #510.""" + # This simulates what the model sometimes returns + data = { + "output": { + "name": "Example Section", + "items": '[{"field": {"text": "Sample", "field_id": "F1"}, "metadata": [{"id": "M1"}]}]', + } + } + result = _normalize_structured_output(data) + assert result == { + "name": "Example Section", + "items": [ + { + "field": {"text": "Sample", "field_id": "F1"}, + "metadata": [{"id": "M1"}], + } + ], + } + + +class TestResultMessageParsing: + """Tests for ResultMessage parsing with structured output normalization.""" + + def test_parse_result_message_with_wrapped_output(self): + """Test that ResultMessage parsing normalizes wrapped structured_output.""" + data = { + "type": "result", + "subtype": "success", + "duration_ms": 1000, + "duration_api_ms": 800, + "is_error": False, + "num_turns": 3, + "session_id": "test-session", + "structured_output": {"output": {"actions": [1, 2, 3]}}, + } + result = parse_message(data) + assert isinstance(result, ResultMessage) + assert result.structured_output == {"actions": [1, 2, 3]} + + def test_parse_result_message_with_stringified_json(self): + """Test that ResultMessage parsing handles stringified JSON in structured_output.""" + data = { + "type": "result", + "subtype": "success", + "duration_ms": 1000, + "duration_api_ms": 800, + "is_error": False, + "num_turns": 3, + "session_id": "test-session", + "structured_output": { + "items": '[{"field": "value"}]', + }, + } + result = parse_message(data) + assert isinstance(result, ResultMessage) + assert result.structured_output == {"items": [{"field": "value"}]} + + def test_parse_result_message_preserves_valid_output(self): + """Test that valid structured_output is preserved unchanged.""" + expected_output = {"actions": [{"type": "click"}]} + data = { + "type": "result", + "subtype": "success", + "duration_ms": 1000, + "duration_api_ms": 800, + "is_error": False, + "num_turns": 3, + "session_id": "test-session", + "structured_output": expected_output, + } + result = parse_message(data) + assert isinstance(result, ResultMessage) + assert result.structured_output == expected_output + + def test_parse_result_message_with_none_output(self): + """Test that None structured_output remains None.""" + data = { + "type": "result", + "subtype": "success", + "duration_ms": 1000, + "duration_api_ms": 800, + "is_error": False, + "num_turns": 3, + "session_id": "test-session", + "structured_output": None, + } + result = parse_message(data) + assert isinstance(result, ResultMessage) + assert result.structured_output is None