diff --git a/pyproject.toml b/pyproject.toml index 2c0d3af..9daef5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sentienceapi" -version = "0.90.7" +version = "0.90.8" description = "Python SDK for Sentience AI Agent Browser Automation" readme = "README.md" requires-python = ">=3.11" diff --git a/sentience/__init__.py b/sentience/__init__.py index a6a1d3f..7f849a2 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -70,7 +70,7 @@ ) from .wait import wait_for -__version__ = "0.90.7" +__version__ = "0.90.8" __all__ = [ # Core SDK diff --git a/sentience/cloud_tracing.py b/sentience/cloud_tracing.py index 984f48f..1ec836e 100644 --- a/sentience/cloud_tracing.py +++ b/sentience/cloud_tracing.py @@ -145,6 +145,9 @@ def close( # Close file first self._trace_file.close() + # Generate index after closing file + self._generate_index() + if not blocking: # Fire-and-forget background upload thread = threading.Thread( @@ -231,6 +234,16 @@ def _do_upload(self, on_progress: Callable[[int, int], None] | None = None) -> N print(f" Local trace preserved at: {self._path}") # Don't raise - preserve trace locally even if upload fails + def _generate_index(self) -> None: + """Generate trace index file (automatic on close).""" + try: + from .trace_indexing import write_trace_index + + write_trace_index(str(self._path)) + except Exception as e: + # Non-fatal: log but don't crash + print(f"⚠️ Failed to generate trace index: {e}") + def _complete_trace(self) -> None: """ Call /v1/traces/complete to report file sizes to gateway. diff --git a/sentience/trace_indexing/__init__.py b/sentience/trace_indexing/__init__.py new file mode 100644 index 0000000..dfe9066 --- /dev/null +++ b/sentience/trace_indexing/__init__.py @@ -0,0 +1,27 @@ +""" +Trace indexing module for Sentience SDK. +""" + +from .indexer import build_trace_index, write_trace_index, read_step_events +from .index_schema import ( + TraceIndex, + StepIndex, + TraceSummary, + TraceFileInfo, + SnapshotInfo, + ActionInfo, + StepCounters, +) + +__all__ = [ + "build_trace_index", + "write_trace_index", + "read_step_events", + "TraceIndex", + "StepIndex", + "TraceSummary", + "TraceFileInfo", + "SnapshotInfo", + "ActionInfo", + "StepCounters", +] diff --git a/sentience/trace_indexing/index_schema.py b/sentience/trace_indexing/index_schema.py new file mode 100644 index 0000000..4439d82 --- /dev/null +++ b/sentience/trace_indexing/index_schema.py @@ -0,0 +1,111 @@ +""" +Type definitions for trace index schema using concrete classes. +""" + +from dataclasses import dataclass, field, asdict +from typing import Optional, List, Literal + + +@dataclass +class TraceFileInfo: + """Metadata about the trace file.""" + + path: str + size_bytes: int + sha256: str + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass +class TraceSummary: + """High-level summary of the trace.""" + + first_ts: str + last_ts: str + event_count: int + step_count: int + error_count: int + final_url: Optional[str] + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass +class SnapshotInfo: + """Snapshot metadata for index.""" + + snapshot_id: Optional[str] = None + digest: Optional[str] = None + url: Optional[str] = None + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass +class ActionInfo: + """Action metadata for index.""" + + type: Optional[str] = None + target_element_id: Optional[int] = None + args_digest: Optional[str] = None + success: Optional[bool] = None + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass +class StepCounters: + """Event counters per step.""" + + events: int = 0 + snapshots: int = 0 + actions: int = 0 + llm_calls: int = 0 + + def to_dict(self) -> dict: + return asdict(self) + + +@dataclass +class StepIndex: + """Index entry for a single step.""" + + step_index: int + step_id: str + goal: Optional[str] + status: Literal["ok", "error", "partial"] + ts_start: str + ts_end: str + offset_start: int + offset_end: int + url_before: Optional[str] + url_after: Optional[str] + snapshot_before: SnapshotInfo + snapshot_after: SnapshotInfo + action: ActionInfo + counters: StepCounters + + def to_dict(self) -> dict: + result = asdict(self) + return result + + +@dataclass +class TraceIndex: + """Complete trace index schema.""" + + version: int + run_id: str + created_at: str + trace_file: TraceFileInfo + summary: TraceSummary + steps: List[StepIndex] = field(default_factory=list) + + def to_dict(self) -> dict: + """Convert to dictionary for JSON serialization.""" + return asdict(self) diff --git a/sentience/trace_indexing/indexer.py b/sentience/trace_indexing/indexer.py new file mode 100644 index 0000000..e793f7c --- /dev/null +++ b/sentience/trace_indexing/indexer.py @@ -0,0 +1,363 @@ +""" +Trace indexing for fast timeline rendering and step drill-down. +""" + +import hashlib +import json +import os +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List + +from .index_schema import ( + TraceIndex, + StepIndex, + TraceSummary, + TraceFileInfo, + SnapshotInfo, + ActionInfo, + StepCounters, +) + + +def _normalize_text(text: str | None, max_len: int = 80) -> str: + """Normalize text for digest: trim, collapse whitespace, lowercase, cap length.""" + if not text: + return "" + # Trim and collapse whitespace + normalized = " ".join(text.split()) + # Lowercase + normalized = normalized.lower() + # Cap length + if len(normalized) > max_len: + normalized = normalized[:max_len] + return normalized + + +def _round_bbox(bbox: Dict[str, float], precision: int = 2) -> Dict[str, int]: + """Round bbox coordinates to reduce noise (default: 2px precision).""" + return { + "x": round(bbox.get("x", 0) / precision) * precision, + "y": round(bbox.get("y", 0) / precision) * precision, + "width": round(bbox.get("width", 0) / precision) * precision, + "height": round(bbox.get("height", 0) / precision) * precision, + } + + +def _compute_snapshot_digest(snapshot_data: Dict[str, Any]) -> str: + """ + Compute stable digest of snapshot for diffing. + + Includes: url, viewport, canonicalized elements (id, role, text_norm, bbox_rounded). + Excludes: importance, style fields, transient attributes. + """ + url = snapshot_data.get("url", "") + viewport = snapshot_data.get("viewport", {}) + elements = snapshot_data.get("elements", []) + + # Canonicalize elements + canonical_elements = [] + for elem in elements: + canonical_elem = { + "id": elem.get("id"), + "role": elem.get("role", ""), + "text_norm": _normalize_text(elem.get("text")), + "bbox": _round_bbox( + elem.get("bbox", {"x": 0, "y": 0, "width": 0, "height": 0}) + ), + "is_primary": elem.get("is_primary", False), + "is_clickable": elem.get("is_clickable", False), + } + canonical_elements.append(canonical_elem) + + # Sort by element id for determinism + canonical_elements.sort(key=lambda e: e.get("id", 0)) + + # Build canonical object + canonical = { + "url": url, + "viewport": { + "width": viewport.get("width", 0), + "height": viewport.get("height", 0), + }, + "elements": canonical_elements, + } + + # Hash + canonical_json = json.dumps(canonical, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(canonical_json.encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +def _compute_action_digest(action_data: Dict[str, Any]) -> str: + """ + Compute digest of action args for privacy + determinism. + + For TYPE: includes text_len + text_sha256 (not raw text) + For CLICK/PRESS: includes only non-sensitive fields + """ + action_type = action_data.get("type", "") + target_id = action_data.get("target_element_id") + + canonical = { + "type": action_type, + "target_element_id": target_id, + } + + # Type-specific canonicalization + if action_type == "TYPE": + text = action_data.get("text", "") + canonical["text_len"] = len(text) + canonical["text_sha256"] = hashlib.sha256(text.encode("utf-8")).hexdigest() + elif action_type == "PRESS": + canonical["key"] = action_data.get("key", "") + # CLICK has no extra args + + # Hash + canonical_json = json.dumps(canonical, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(canonical_json.encode("utf-8")).hexdigest() + return f"sha256:{digest}" + + +def _compute_file_sha256(file_path: str) -> str: + """Compute SHA256 hash of entire file.""" + sha256 = hashlib.sha256() + with open(file_path, "rb") as f: + while chunk := f.read(8192): + sha256.update(chunk) + return sha256.hexdigest() + + +def build_trace_index(trace_path: str) -> TraceIndex: + """ + Build trace index from JSONL file in single streaming pass. + + Args: + trace_path: Path to trace JSONL file + + Returns: + Complete TraceIndex object + """ + trace_path_obj = Path(trace_path) + if not trace_path_obj.exists(): + raise FileNotFoundError(f"Trace file not found: {trace_path}") + + # Extract run_id from filename + run_id = trace_path_obj.stem + + # Initialize summary + first_ts = "" + last_ts = "" + event_count = 0 + error_count = 0 + final_url = None + + steps_by_id: Dict[str, StepIndex] = {} + step_order: List[str] = [] # Track order of first appearance + + # Stream through file, tracking byte offsets + with open(trace_path, "rb") as f: + byte_offset = 0 + + for line_bytes in f: + line_len = len(line_bytes) + + try: + event = json.loads(line_bytes.decode("utf-8")) + except json.JSONDecodeError: + # Skip malformed lines + byte_offset += line_len + continue + + # Extract event metadata + event_type = event.get("type", "") + ts = event.get("ts") or event.get("timestamp", "") + step_id = event.get("step_id", "step-0") # Default synthetic step + data = event.get("data", {}) + + # Update summary + event_count += 1 + if not first_ts: + first_ts = ts + last_ts = ts + + if event_type == "error": + error_count += 1 + + # Initialize step if first time seeing this step_id + if step_id not in steps_by_id: + step_order.append(step_id) + steps_by_id[step_id] = StepIndex( + step_index=len(step_order), + step_id=step_id, + goal=None, + status="partial", + ts_start=ts, + ts_end=ts, + offset_start=byte_offset, + offset_end=byte_offset + line_len, + url_before=None, + url_after=None, + snapshot_before=SnapshotInfo(), + snapshot_after=SnapshotInfo(), + action=ActionInfo(), + counters=StepCounters(), + ) + + step = steps_by_id[step_id] + + # Update step metadata + step.ts_end = ts + step.offset_end = byte_offset + line_len + step.counters.events += 1 + + # Handle specific event types + if event_type == "step_start": + step.goal = data.get("goal") + step.url_before = data.get("pre_url") + + elif event_type == "snapshot": + snapshot_id = data.get("snapshot_id") + url = data.get("url") + digest = _compute_snapshot_digest(data) + + # First snapshot = before, last snapshot = after + if step.snapshot_before.snapshot_id is None: + step.snapshot_before = SnapshotInfo( + snapshot_id=snapshot_id, digest=digest, url=url + ) + step.url_before = step.url_before or url + + step.snapshot_after = SnapshotInfo( + snapshot_id=snapshot_id, digest=digest, url=url + ) + step.url_after = url + step.counters.snapshots += 1 + final_url = url + + elif event_type == "action": + step.action = ActionInfo( + type=data.get("type"), + target_element_id=data.get("target_element_id"), + args_digest=_compute_action_digest(data), + success=data.get("success", True), + ) + step.counters.actions += 1 + + elif event_type == "llm_response": + step.counters.llm_calls += 1 + + elif event_type == "error": + step.status = "error" + + elif event_type == "step_end": + if step.status != "error": + step.status = "ok" + + byte_offset += line_len + + # Build summary + summary = TraceSummary( + first_ts=first_ts, + last_ts=last_ts, + event_count=event_count, + step_count=len(steps_by_id), + error_count=error_count, + final_url=final_url, + ) + + # Build steps list in order + steps_list = [steps_by_id[sid] for sid in step_order] + + # Build trace file info + trace_file = TraceFileInfo( + path=str(trace_path), + size_bytes=os.path.getsize(trace_path), + sha256=_compute_file_sha256(str(trace_path)), + ) + + # Build final index + index = TraceIndex( + version=1, + run_id=run_id, + created_at=datetime.now(timezone.utc).isoformat(), + trace_file=trace_file, + summary=summary, + steps=steps_list, + ) + + return index + + +def write_trace_index(trace_path: str, index_path: str | None = None) -> str: + """ + Build index and write to file. + + Args: + trace_path: Path to trace JSONL file + index_path: Optional custom path for index file (default: trace_path with .index.json) + + Returns: + Path to written index file + """ + if index_path is None: + index_path = str(Path(trace_path).with_suffix("")) + ".index.json" + + index = build_trace_index(trace_path) + + with open(index_path, "w") as f: + json.dump(index.to_dict(), f, indent=2) + + return index_path + + +def read_step_events( + trace_path: str, offset_start: int, offset_end: int +) -> List[Dict[str, Any]]: + """ + Read events for a specific step using byte offsets from index. + + Args: + trace_path: Path to trace JSONL file + offset_start: Byte offset where step starts + offset_end: Byte offset where step ends + + Returns: + List of event dictionaries for the step + """ + events = [] + + with open(trace_path, "rb") as f: + f.seek(offset_start) + bytes_to_read = offset_end - offset_start + chunk = f.read(bytes_to_read) + + # Parse lines + for line_bytes in chunk.split(b"\n"): + if not line_bytes: + continue + try: + event = json.loads(line_bytes.decode("utf-8")) + events.append(event) + except json.JSONDecodeError: + continue + + return events + + +# CLI entrypoint +def main(): + """CLI tool for building trace index.""" + import sys + + if len(sys.argv) < 2: + print("Usage: python -m sentience.tracing.indexer ") + sys.exit(1) + + trace_path = sys.argv[1] + index_path = write_trace_index(trace_path) + print(f"✅ Index written to: {index_path}") + + +if __name__ == "__main__": + main() diff --git a/sentience/tracing.py b/sentience/tracing.py index fbaf0ec..d15b12d 100644 --- a/sentience/tracing.py +++ b/sentience/tracing.py @@ -103,10 +103,23 @@ def emit(self, event: dict[str, Any]) -> None: self._file.write(json_str + "\n") def close(self) -> None: - """Close the file.""" + """Close the file and generate index.""" if hasattr(self, "_file") and not self._file.closed: self._file.close() + # Generate index after closing file + self._generate_index() + + def _generate_index(self) -> None: + """Generate trace index file (automatic on close).""" + try: + from .trace_indexing import write_trace_index + + write_trace_index(str(self.path)) + except Exception as e: + # Non-fatal: log but don't crash + print(f"⚠️ Failed to generate trace index: {e}") + def __enter__(self): """Context manager support.""" return self diff --git a/tests/test_cloud_tracing.py b/tests/test_cloud_tracing.py index 49b899a..38339c9 100644 --- a/tests/test_cloud_tracing.py +++ b/tests/test_cloud_tracing.py @@ -250,7 +250,9 @@ def test_create_tracer_pro_tier_success(self, capsys): # Mock upload response mock_put.return_value = Mock(status_code=200) - tracer = create_tracer(api_key="sk_pro_test123", run_id="test-run", upload_trace=True) + tracer = create_tracer( + api_key="sk_pro_test123", run_id="test-run", upload_trace=True + ) # Verify Pro tier message captured = capsys.readouterr() @@ -294,7 +296,9 @@ def test_create_tracer_api_forbidden_fallback(self, capsys): mock_post.return_value = mock_response with tempfile.TemporaryDirectory(): - tracer = create_tracer(api_key="sk_free_test123", run_id="test-run", upload_trace=True) + tracer = create_tracer( + api_key="sk_free_test123", run_id="test-run", upload_trace=True + ) # Verify warning message captured = capsys.readouterr() @@ -395,7 +399,10 @@ def test_create_tracer_custom_api_url(self): mock_put.return_value = Mock(status_code=200) tracer = create_tracer( - api_key="sk_test123", run_id="test-run", api_url=custom_api_url, upload_trace=True + api_key="sk_test123", + run_id="test-run", + api_url=custom_api_url, + upload_trace=True, ) # Verify custom API URL was used @@ -463,7 +470,9 @@ def test_create_tracer_orphaned_trace_recovery(self, capsys): mock_put.return_value = Mock(status_code=200) # Create tracer - should trigger orphaned trace recovery - tracer = create_tracer(api_key="sk_test123", run_id="new-run-456", upload_trace=True) + tracer = create_tracer( + api_key="sk_test123", run_id="new-run-456", upload_trace=True + ) # Verify recovery messages captured = capsys.readouterr() diff --git a/tests/test_trace_indexing.py b/tests/test_trace_indexing.py new file mode 100644 index 0000000..bcadffb --- /dev/null +++ b/tests/test_trace_indexing.py @@ -0,0 +1,532 @@ +""" +Tests for trace indexing functionality. +""" + +import json +import os +import tempfile +from pathlib import Path + +import pytest + +from sentience.trace_indexing import ( + build_trace_index, + write_trace_index, + read_step_events, + TraceIndex, + StepIndex, +) + + +class TestTraceIndexing: + """Test trace index building and querying.""" + + def test_empty_trace(self): + """Index of empty file should have zero events.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "empty.jsonl" + trace_path.write_text("") + + index = build_trace_index(str(trace_path)) + + assert isinstance(index, TraceIndex) + assert index.version == 1 + assert index.run_id == "empty" + assert index.summary.event_count == 0 + assert index.summary.step_count == 0 + assert index.summary.error_count == 0 + assert len(index.steps) == 0 + + def test_single_step_trace(self): + """Index should have 1 step with correct offsets.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "single-step.jsonl" + + events = [ + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": {"goal": "Test goal"}, + }, + { + "v": 1, + "type": "action", + "ts": "2025-12-29T10:00:01.000Z", + "step_id": "step-1", + "data": {"type": "CLICK", "target_element_id": 42, "success": True}, + }, + { + "v": 1, + "type": "step_end", + "ts": "2025-12-29T10:00:02.000Z", + "step_id": "step-1", + "data": {}, + }, + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index = build_trace_index(str(trace_path)) + + assert index.summary.event_count == 3 + assert index.summary.step_count == 1 + assert len(index.steps) == 1 + + step = index.steps[0] + assert isinstance(step, StepIndex) + assert step.step_id == "step-1" + assert step.step_index == 1 + assert step.goal == "Test goal" + assert step.status == "ok" + assert step.counters.events == 3 + assert step.counters.actions == 1 + assert step.offset_start == 0 + assert step.offset_end > step.offset_start + + def test_multi_step_trace(self): + """Index should have multiple steps in order.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "multi-step.jsonl" + + events = [ + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": {"goal": "First step"}, + }, + { + "v": 1, + "type": "step_end", + "ts": "2025-12-29T10:00:01.000Z", + "step_id": "step-1", + "data": {}, + }, + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:02.000Z", + "step_id": "step-2", + "data": {"goal": "Second step"}, + }, + { + "v": 1, + "type": "step_end", + "ts": "2025-12-29T10:00:03.000Z", + "step_id": "step-2", + "data": {}, + }, + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index = build_trace_index(str(trace_path)) + + assert index.summary.step_count == 2 + assert len(index.steps) == 2 + assert index.steps[0].step_id == "step-1" + assert index.steps[0].step_index == 1 + assert index.steps[1].step_id == "step-2" + assert index.steps[1].step_index == 2 + + def test_byte_offset_accuracy(self): + """Seeking to offset should return exact events.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "offset-test.jsonl" + + events = [ + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": {}, + }, + { + "v": 1, + "type": "action", + "ts": "2025-12-29T10:00:01.000Z", + "step_id": "step-1", + "data": {"type": "CLICK"}, + }, + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:02.000Z", + "step_id": "step-2", + "data": {}, + }, + { + "v": 1, + "type": "action", + "ts": "2025-12-29T10:00:03.000Z", + "step_id": "step-2", + "data": {"type": "TYPE"}, + }, + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index = build_trace_index(str(trace_path)) + + # Read step-1 events using offset + step1 = index.steps[0] + step1_events = read_step_events( + str(trace_path), step1.offset_start, step1.offset_end + ) + + assert len(step1_events) == 2 + assert step1_events[0]["step_id"] == "step-1" + assert step1_events[0]["type"] == "step_start" + assert step1_events[1]["step_id"] == "step-1" + assert step1_events[1]["type"] == "action" + + # Read step-2 events using offset + step2 = index.steps[1] + step2_events = read_step_events( + str(trace_path), step2.offset_start, step2.offset_end + ) + + assert len(step2_events) == 2 + assert step2_events[0]["step_id"] == "step-2" + assert step2_events[1]["type"] == "action" + + def test_snapshot_digest_determinism(self): + """Same snapshot data should produce same digest.""" + snapshot_data = { + "url": "https://example.com", + "viewport": {"width": 1920, "height": 1080}, + "elements": [ + { + "id": 1, + "role": "button", + "text": "Click me", + "bbox": {"x": 10.0, "y": 20.0, "width": 100.0, "height": 50.0}, + "is_primary": True, + "is_clickable": True, + } + ], + } + + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "digest-test.jsonl" + + events = [ + { + "v": 1, + "type": "snapshot", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": snapshot_data, + } + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index1 = build_trace_index(str(trace_path)) + index2 = build_trace_index(str(trace_path)) + + digest1 = index1.steps[0].snapshot_after.digest + digest2 = index2.steps[0].snapshot_after.digest + + assert digest1 == digest2 + assert digest1.startswith("sha256:") + + def test_snapshot_digest_noise_resistance(self): + """Small changes (2px bbox shift) should produce same digest.""" + base_snapshot = { + "url": "https://example.com", + "viewport": {"width": 1920, "height": 1080}, + "elements": [ + { + "id": 1, + "role": "button", + "text": " Click me ", # Extra whitespace + "bbox": {"x": 10.0, "y": 20.0, "width": 100.0, "height": 50.0}, + } + ], + } + + shifted_snapshot = { + "url": "https://example.com", + "viewport": {"width": 1920, "height": 1080}, + "elements": [ + { + "id": 1, + "role": "button", + "text": "Click me", # No extra whitespace + "bbox": { + "x": 10.5, + "y": 20.5, + "width": 100.5, + "height": 50.5, + }, # Sub-2px shift + } + ], + } + + with tempfile.TemporaryDirectory() as tmpdir: + trace1_path = Path(tmpdir) / "base.jsonl" + trace2_path = Path(tmpdir) / "shifted.jsonl" + + with open(trace1_path, "w") as f: + f.write( + json.dumps( + { + "v": 1, + "type": "snapshot", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": base_snapshot, + } + ) + + "\n" + ) + + with open(trace2_path, "w") as f: + f.write( + json.dumps( + { + "v": 1, + "type": "snapshot", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": shifted_snapshot, + } + ) + + "\n" + ) + + index1 = build_trace_index(str(trace1_path)) + index2 = build_trace_index(str(trace2_path)) + + digest1 = index1.steps[0].snapshot_after.digest + digest2 = index2.steps[0].snapshot_after.digest + + assert digest1 == digest2 # Should be identical despite noise + + def test_action_digest_privacy(self): + """Typed text should not appear in action digest.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "privacy-test.jsonl" + + sensitive_text = "my-secret-password" + events = [ + { + "v": 1, + "type": "action", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": { + "type": "TYPE", + "target_element_id": 15, + "text": sensitive_text, + "success": True, + }, + } + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index = build_trace_index(str(trace_path)) + + # Convert index to JSON string + index_json = json.dumps(index.to_dict()) + + # Verify sensitive text is NOT in index + assert sensitive_text not in index_json + + # Verify action digest exists and is a hash + action_digest = index.steps[0].action.args_digest + assert action_digest is not None + assert action_digest.startswith("sha256:") + + def test_synthetic_step(self): + """Events without step_id should create step-0.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "synthetic-step.jsonl" + + events = [ + {"v": 1, "type": "run_start", "ts": "2025-12-29T10:00:00.000Z", "data": {}}, + {"v": 1, "type": "action", "ts": "2025-12-29T10:00:01.000Z", "data": {}}, + {"v": 1, "type": "run_end", "ts": "2025-12-29T10:00:02.000Z", "data": {}}, + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index = build_trace_index(str(trace_path)) + + assert index.summary.step_count == 1 + assert len(index.steps) == 1 + assert index.steps[0].step_id == "step-0" # Synthetic step + + def test_index_idempotency(self): + """Regenerating index should produce identical output.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "idempotent.jsonl" + + events = [ + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": {}, + }, + { + "v": 1, + "type": "action", + "ts": "2025-12-29T10:00:01.000Z", + "step_id": "step-1", + "data": {"type": "CLICK"}, + }, + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index1 = build_trace_index(str(trace_path)) + index2 = build_trace_index(str(trace_path)) + + # Compare all fields except created_at (timestamp will differ) + assert index1.version == index2.version + assert index1.run_id == index2.run_id + assert index1.trace_file.sha256 == index2.trace_file.sha256 + assert index1.summary.event_count == index2.summary.event_count + assert len(index1.steps) == len(index2.steps) + + for step1, step2 in zip(index1.steps, index2.steps): + assert step1.step_id == step2.step_id + assert step1.offset_start == step2.offset_start + assert step1.offset_end == step2.offset_end + + def test_write_trace_index(self): + """write_trace_index should create index file.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "test.jsonl" + + events = [ + {"v": 1, "type": "run_start", "ts": "2025-12-29T10:00:00.000Z", "data": {}} + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index_path = write_trace_index(str(trace_path)) + + assert os.path.exists(index_path) + assert index_path.endswith(".index.json") + + # Verify index content + with open(index_path, "r") as f: + index_data = json.load(f) + + assert index_data["version"] == 1 + assert index_data["run_id"] == "test" + assert "summary" in index_data + assert "steps" in index_data + + def test_error_counting(self): + """Errors should be counted in summary and affect step status.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "errors.jsonl" + + events = [ + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": {}, + }, + { + "v": 1, + "type": "error", + "ts": "2025-12-29T10:00:01.000Z", + "step_id": "step-1", + "data": {"message": "Something failed"}, + }, + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index = build_trace_index(str(trace_path)) + + assert index.summary.error_count == 1 + assert index.steps[0].status == "error" + + def test_llm_call_counting(self): + """LLM calls should be counted per step.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "llm.jsonl" + + events = [ + { + "v": 1, + "type": "step_start", + "ts": "2025-12-29T10:00:00.000Z", + "step_id": "step-1", + "data": {}, + }, + { + "v": 1, + "type": "llm_response", + "ts": "2025-12-29T10:00:01.000Z", + "step_id": "step-1", + "data": {}, + }, + { + "v": 1, + "type": "llm_response", + "ts": "2025-12-29T10:00:02.000Z", + "step_id": "step-1", + "data": {}, + }, + ] + + with open(trace_path, "w") as f: + for event in events: + f.write(json.dumps(event) + "\n") + + index = build_trace_index(str(trace_path)) + + assert index.steps[0].counters.llm_calls == 2 + + def test_malformed_lines_skipped(self): + """Malformed JSON lines should be skipped gracefully.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "malformed.jsonl" + + with open(trace_path, "w") as f: + f.write('{"v": 1, "type": "run_start", "ts": "2025-12-29T10:00:00.000Z"}\n') + f.write("this is not valid json\n") # Malformed line + f.write('{"v": 1, "type": "run_end", "ts": "2025-12-29T10:00:01.000Z"}\n') + + index = build_trace_index(str(trace_path)) + + # Should have 2 valid events (malformed line skipped) + assert index.summary.event_count == 2 + + def test_file_not_found(self): + """Should raise FileNotFoundError for non-existent file.""" + with pytest.raises(FileNotFoundError): + build_trace_index("/nonexistent/trace.jsonl")