From 2e804598de5b0ccd0e7e3630c4254e2eb1cefa42 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 19:52:58 -0800 Subject: [PATCH 1/7] cloud tracing support --- sentience/__init__.py | 8 +- sentience/cloud_tracing.py | 148 ++++++++ sentience/extension/background.js | 6 +- sentience/extension/content.js | 2 +- sentience/extension/injected_api.js | 216 +++++------ sentience/tracer_factory.py | 105 ++++++ sentience_python.egg-info/PKG-INFO | 2 +- sentience_python.egg-info/SOURCES.txt | 3 + tests/test_cloud_tracing.py | 347 ++++++++++++++++++ ...c8d99417-49ca-4195-bc66-face72cc9494.jsonl | 0 traces/test-run.jsonl | 0 11 files changed, 722 insertions(+), 115 deletions(-) create mode 100644 sentience/cloud_tracing.py create mode 100644 sentience/tracer_factory.py create mode 100644 tests/test_cloud_tracing.py create mode 100644 traces/c8d99417-49ca-4195-bc66-face72cc9494.jsonl create mode 100644 traces/test-run.jsonl diff --git a/sentience/__init__.py b/sentience/__init__.py index 1f0cd12..8bb92e5 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -9,6 +9,9 @@ # Agent Layer (Phase 1 & 2) from .base_agent import BaseAgent from .browser import SentienceBrowser + +# Tracing (v0.12.0+) +from .cloud_tracing import CloudTraceSink from .conversational_agent import ConversationalAgent from .expect import expect @@ -43,8 +46,7 @@ from .recorder import Recorder, Trace, TraceStep, record from .screenshot import screenshot from .snapshot import snapshot - -# Tracing (v0.12.0+) +from .tracer_factory import create_tracer from .tracing import JsonlTraceSink, TraceEvent, Tracer, TraceSink # Utilities (v0.12.0+) @@ -107,7 +109,9 @@ "Tracer", "TraceSink", "JsonlTraceSink", + "CloudTraceSink", "TraceEvent", + "create_tracer", # Utilities (v0.12.0+) "canonical_snapshot_strict", "canonical_snapshot_loose", diff --git a/sentience/cloud_tracing.py b/sentience/cloud_tracing.py new file mode 100644 index 0000000..363aeab --- /dev/null +++ b/sentience/cloud_tracing.py @@ -0,0 +1,148 @@ +""" +Cloud trace sink with pre-signed URL upload. + +Implements "Local Write, Batch Upload" pattern for enterprise cloud tracing. +""" + +import gzip +import json +import os +import tempfile +from typing import Any + +import requests + +from sentience.tracing import TraceEvent, TraceSink + + +class CloudTraceSink(TraceSink): + """ + Enterprise Cloud Sink: "Local Write, Batch Upload" pattern. + + Architecture: + 1. **Local Buffer**: Writes to temp file (zero latency, non-blocking) + 2. **Pre-signed URL**: Uses secure pre-signed PUT URL from backend API + 3. **Batch Upload**: Uploads complete file on close() or at intervals + 4. **Zero Credential Exposure**: Never embeds DigitalOcean credentials in SDK + + This design ensures: + - Fast agent performance (microseconds per emit, not milliseconds) + - Security (credentials stay on backend) + - Reliability (network issues don't crash the agent) + + Tiered Access: + - Free Tier: Falls back to JsonlTraceSink (local-only) + - Pro/Enterprise: Uploads to cloud via pre-signed URLs + + Example: + >>> from sentience.cloud_tracing import CloudTraceSink + >>> from sentience.tracing import Tracer + >>> # Get upload URL from API + >>> upload_url = "https://sentience.nyc3.digitaloceanspaces.com/..." + >>> sink = CloudTraceSink(upload_url) + >>> tracer = Tracer(run_id="demo", sink=sink) + >>> tracer.emit_run_start("SentienceAgent") + >>> tracer.close() # Uploads to cloud + """ + + def __init__(self, upload_url: str): + """ + Initialize cloud trace sink. + + Args: + upload_url: Pre-signed PUT URL from Sentience API + (e.g., "https://sentience.nyc3.digitaloceanspaces.com/...") + """ + self.upload_url = upload_url + + # Create temporary file for buffering + # delete=False so we can read it back before uploading + self._temp_file = tempfile.NamedTemporaryFile( + mode="w+", + encoding="utf-8", + suffix=".jsonl", + delete=False, + ) + self._path = self._temp_file.name + self._closed = False + + def emit(self, event: dict[str, Any]) -> None: + """ + Write event to local temp file (Fast, non-blocking). + + Performance: ~10 microseconds per write vs ~50ms for HTTP request + + Args: + event: Event dictionary from TraceEvent.to_dict() + """ + if self._closed: + raise RuntimeError("CloudTraceSink is closed") + + json_str = json.dumps(event, ensure_ascii=False) + self._temp_file.write(json_str + "\n") + self._temp_file.flush() # Ensure written to disk + + def close(self) -> None: + """ + Upload buffered trace to cloud via pre-signed URL. + + This is the only network call - happens once at the end. + """ + if self._closed: + return + + self._closed = True + + try: + # 1. Close temp file + self._temp_file.close() + + # 2. Compress for upload + with open(self._path, "rb") as f: + trace_data = f.read() + + compressed_data = gzip.compress(trace_data) + + # 3. Upload to DigitalOcean Spaces via pre-signed URL + print(f"šŸ“¤ [Sentience] Uploading trace to cloud ({len(compressed_data)} bytes)...") + + response = requests.put( + self.upload_url, + data=compressed_data, + headers={ + "Content-Type": "application/x-gzip", + "Content-Encoding": "gzip", + }, + timeout=60, # 1 minute timeout for large files + ) + + if response.status_code == 200: + print("āœ… [Sentience] Trace uploaded successfully") + else: + print(f"āŒ [Sentience] Upload failed: HTTP {response.status_code}") + print(f" Response: {response.text}") + print(f" Local trace preserved at: {self._path}") + + except Exception as e: + print(f"āŒ [Sentience] Error uploading trace: {e}") + print(f" Local trace preserved at: {self._path}") + # Don't raise - preserve trace locally even if upload fails + + finally: + # 4. Cleanup temp file (only if upload succeeded) + if os.path.exists(self._path): + try: + # Only delete if upload was successful + if hasattr(self, "_upload_successful") and self._upload_successful: + os.remove(self._path) + except Exception: + pass # Ignore cleanup errors + + def __enter__(self): + """Context manager support.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager cleanup.""" + self.close() + return False diff --git a/sentience/extension/background.js b/sentience/extension/background.js index 811303f..f359ba6 100644 --- a/sentience/extension/background.js +++ b/sentience/extension/background.js @@ -144,13 +144,13 @@ async function handleScreenshotCapture(_tabId, options = {}) { async function handleSnapshotProcessing(rawData, options = {}) { const MAX_ELEMENTS = 10000; // Safety limit to prevent hangs const startTime = performance.now(); - + try { // Safety check: limit element count to prevent hangs if (!Array.isArray(rawData)) { throw new Error('rawData must be an array'); } - + if (rawData.length > MAX_ELEMENTS) { console.warn(`[Sentience Background] āš ļø Large dataset: ${rawData.length} elements. Limiting to ${MAX_ELEMENTS} to prevent hangs.`); rawData = rawData.slice(0, MAX_ELEMENTS); @@ -186,7 +186,7 @@ async function handleSnapshotProcessing(rawData, options = {}) { // Add timeout protection (18 seconds - less than content.js timeout) analyzedElements = await Promise.race([ wasmPromise, - new Promise((_, reject) => + new Promise((_, reject) => setTimeout(() => reject(new Error('WASM processing timeout (>18s)')), 18000) ) ]); diff --git a/sentience/extension/content.js b/sentience/extension/content.js index 833ee05..bdffc54 100644 --- a/sentience/extension/content.js +++ b/sentience/extension/content.js @@ -84,7 +84,7 @@ function handleSnapshotRequest(data) { if (responded) return; // Already responded via timeout responded = true; clearTimeout(timeoutId); - + const duration = performance.now() - startTime; // Handle Chrome extension errors (e.g., background script crashed) diff --git a/sentience/extension/injected_api.js b/sentience/extension/injected_api.js index a47ad55..09ab006 100644 --- a/sentience/extension/injected_api.js +++ b/sentience/extension/injected_api.js @@ -66,10 +66,10 @@ // --- HELPER: Safe Class Name Extractor (Handles SVGAnimatedString) --- function getClassName(el) { if (!el || !el.className) return ''; - + // Handle string (HTML elements) if (typeof el.className === 'string') return el.className; - + // Handle SVGAnimatedString (SVG elements) if (typeof el.className === 'object') { if ('baseVal' in el.className && typeof el.className.baseVal === 'string') { @@ -85,17 +85,17 @@ return ''; } } - + return ''; } // --- HELPER: Paranoid String Converter (Handles SVGAnimatedString) --- function toSafeString(value) { if (value === null || value === undefined) return null; - + // 1. If it's already a primitive string, return it if (typeof value === 'string') return value; - + // 2. Handle SVG objects (SVGAnimatedString, SVGAnimatedNumber, etc.) if (typeof value === 'object') { // Try extracting baseVal (standard SVG property) @@ -114,7 +114,7 @@ return null; } } - + // 3. Last resort cast for primitives try { return String(value); @@ -127,9 +127,9 @@ // For SVG elements, get the fill or stroke color (SVGs use fill/stroke, not backgroundColor) function getSVGColor(el) { if (!el || el.tagName !== 'SVG') return null; - + const style = window.getComputedStyle(el); - + // Try fill first (most common for SVG icons) const fill = style.fill; if (fill && fill !== 'none' && fill !== 'transparent' && fill !== 'rgba(0, 0, 0, 0)') { @@ -144,7 +144,7 @@ return fill; } } - + // Fallback to stroke if fill is not available const stroke = style.stroke; if (stroke && stroke !== 'none' && stroke !== 'transparent' && stroke !== 'rgba(0, 0, 0, 0)') { @@ -158,7 +158,7 @@ return stroke; } } - + return null; } @@ -168,28 +168,28 @@ // This handles rgba(0,0,0,0) and transparent values that browsers commonly return function getEffectiveBackgroundColor(el) { if (!el) return null; - + // For SVG elements, use fill/stroke instead of backgroundColor if (el.tagName === 'SVG') { const svgColor = getSVGColor(el); if (svgColor) return svgColor; } - + let current = el; const maxDepth = 10; // Prevent infinite loops let depth = 0; - + while (current && depth < maxDepth) { const style = window.getComputedStyle(current); - + // For SVG elements in the tree, also check fill/stroke if (current.tagName === 'SVG') { const svgColor = getSVGColor(current); if (svgColor) return svgColor; } - + const bgColor = style.backgroundColor; - + if (bgColor && bgColor !== 'transparent' && bgColor !== 'rgba(0, 0, 0, 0)') { // Check if it's rgba with alpha < 1 (semi-transparent) const rgbaMatch = bgColor.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); @@ -209,12 +209,12 @@ return bgColor; } } - + // Move up the DOM tree current = current.parentElement; depth++; } - + // Fallback: return null if nothing found return null; } @@ -235,7 +235,7 @@ // Only check for elements that are likely to be occluded (overlays, modals, tooltips) const zIndex = parseInt(style.zIndex, 10); const position = style.position; - + // Skip occlusion check for normal flow elements (vast majority) // Only check for positioned elements or high z-index (likely overlays) if (position === 'static' && (isNaN(zIndex) || zIndex <= 10)) { @@ -308,7 +308,7 @@ }; window.addEventListener('message', listener); - + try { window.postMessage({ type: 'SENTIENCE_SNAPSHOT_REQUEST', @@ -514,7 +514,7 @@ function extractRawElementData(el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); - + return { tag: el.tagName, rect: { @@ -548,12 +548,12 @@ // --- HELPER: Generate Unique CSS Selector (for Golden Set) --- function getUniqueSelector(el) { if (!el || !el.tagName) return ''; - + // If element has a unique ID, use it if (el.id) { return `#${el.id}`; } - + // Try data attributes or aria-label for uniqueness for (const attr of el.attributes) { if (attr.name.startsWith('data-') || attr.name === 'aria-label') { @@ -561,21 +561,21 @@ return `${el.tagName.toLowerCase()}[${attr.name}="${value}"]`; } } - + // Build path with classes and nth-child for uniqueness const path = []; let current = el; - + while (current && current !== document.body && current !== document.documentElement) { let selector = current.tagName.toLowerCase(); - + // If current element has ID, use it and stop if (current.id) { selector = `#${current.id}`; path.unshift(selector); break; } - + // Add class if available if (current.className && typeof current.className === 'string') { const classes = current.className.trim().split(/\s+/).filter(c => c); @@ -584,7 +584,7 @@ selector += `.${classes[0]}`; } } - + // Add nth-of-type if needed for uniqueness if (current.parentElement) { const siblings = Array.from(current.parentElement.children); @@ -594,11 +594,11 @@ selector += `:nth-of-type(${index + 1})`; } } - + path.unshift(selector); current = current.parentElement; } - + return path.join(' > ') || el.tagName.toLowerCase(); } @@ -613,7 +613,7 @@ } = options; const startTime = Date.now(); - + return new Promise((resolve) => { // Check if DOM already has enough nodes const nodeCount = document.querySelectorAll('*').length; @@ -623,17 +623,17 @@ const observer = new MutationObserver(() => { lastChange = Date.now(); }); - + observer.observe(document.body, { childList: true, subtree: true, attributes: false }); - + const checkStable = () => { const timeSinceLastChange = Date.now() - lastChange; const totalWait = Date.now() - startTime; - + if (timeSinceLastChange >= quietPeriod) { observer.disconnect(); resolve(); @@ -645,14 +645,14 @@ setTimeout(checkStable, 50); } }; - + checkStable(); } else { // DOM doesn't have enough nodes yet, wait for them const observer = new MutationObserver(() => { const currentCount = document.querySelectorAll('*').length; const totalWait = Date.now() - startTime; - + if (currentCount >= minNodeCount) { observer.disconnect(); // Now wait for quiet period @@ -660,17 +660,17 @@ const quietObserver = new MutationObserver(() => { lastChange = Date.now(); }); - + quietObserver.observe(document.body, { childList: true, subtree: true, attributes: false }); - + const checkQuiet = () => { const timeSinceLastChange = Date.now() - lastChange; const totalWait = Date.now() - startTime; - + if (timeSinceLastChange >= quietPeriod) { quietObserver.disconnect(); resolve(); @@ -682,7 +682,7 @@ setTimeout(checkQuiet, 50); } }; - + checkQuiet(); } else if (totalWait >= maxWait) { observer.disconnect(); @@ -690,13 +690,13 @@ resolve(); } }); - + observer.observe(document.body, { childList: true, subtree: true, attributes: false }); - + // Timeout fallback setTimeout(() => { observer.disconnect(); @@ -710,34 +710,34 @@ // --- HELPER: Collect Iframe Snapshots (Frame Stitching) --- // Recursively collects snapshot data from all child iframes // This enables detection of elements inside iframes (e.g., Stripe forms) - // + // // NOTE: Cross-origin iframes cannot be accessed due to browser security (Same-Origin Policy). // Only same-origin iframes will return snapshot data. Cross-origin iframes will be skipped // with a warning. For cross-origin iframes, users must manually switch frames using // Playwright's page.frame() API. async function collectIframeSnapshots(options = {}) { const iframeData = new Map(); // Map of iframe element -> snapshot data - + // Find all iframe elements in current document const iframes = Array.from(document.querySelectorAll('iframe')); - + if (iframes.length === 0) { return iframeData; } - + console.log(`[SentienceAPI] Found ${iframes.length} iframe(s), requesting snapshots...`); - + // Request snapshot from each iframe const iframePromises = iframes.map((iframe, idx) => { return new Promise((resolve) => { const requestId = `iframe-${idx}-${Date.now()}`; - + // 1. EXTENDED TIMEOUT (Handle slow children) const timeout = setTimeout(() => { console.warn(`[SentienceAPI] āš ļø Iframe ${idx} snapshot TIMEOUT (id: ${requestId})`); resolve(null); }, 10000); // Increased to 10s to handle slow processing - + // 2. ROBUST LISTENER with debugging const listener = (event) => { // Debug: Log all SENTIENCE_IFRAME_SNAPSHOT_RESPONSE messages to see what's happening @@ -747,14 +747,14 @@ // console.log(`[SentienceAPI] Received response for different request: ${event.data.requestId} (expected: ${requestId})`); } } - + // Check if this is the response we're waiting for - if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE' && + if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE' && event.data?.requestId === requestId) { - + clearTimeout(timeout); window.removeEventListener('message', listener); - + if (event.data.error) { console.warn(`[SentienceAPI] Iframe ${idx} returned error:`, event.data.error); resolve(null); @@ -769,9 +769,9 @@ } } }; - + window.addEventListener('message', listener); - + // 3. SEND REQUEST with error handling try { if (iframe.contentWindow) { @@ -779,8 +779,8 @@ iframe.contentWindow.postMessage({ type: 'SENTIENCE_IFRAME_SNAPSHOT_REQUEST', requestId: requestId, - options: { - ...options, + options: { + ...options, collectIframes: true // Enable recursion for nested iframes } }, '*'); // Use '*' for cross-origin, but browser will enforce same-origin policy @@ -798,10 +798,10 @@ } }); }); - + // Wait for all iframe responses const results = await Promise.all(iframePromises); - + // Store iframe data results.forEach((result, idx) => { if (result && result.data && !result.error) { @@ -813,7 +813,7 @@ console.warn(`[SentienceAPI] Iframe ${idx} returned no data (timeout or error)`); } }); - + return iframeData; } @@ -826,7 +826,7 @@ // Security: only respond to snapshot requests from parent frames if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_REQUEST') { const { requestId, options } = event.data; - + try { // Generate snapshot for this iframe's content // Allow recursive collection - querySelectorAll('iframe') only finds direct children, @@ -834,7 +834,7 @@ // waitForStability: false makes performance better - i.e. don't wait for children frames const snapshotOptions = { ...options, collectIframes: true, waitForStability: options.waitForStability === false ? false : false }; const snapshot = await window.sentience.snapshot(snapshotOptions); - + // Send response back to parent if (event.source && event.source.postMessage) { event.source.postMessage({ @@ -858,7 +858,7 @@ } }); } - + // Setup iframe handler when script loads (only once) if (!window.sentience_iframe_handler_setup) { setupIframeSnapshotHandler(); @@ -874,7 +874,7 @@ if (options.waitForStability !== false) { await waitForStability(options.waitForStability || {}); } - + // Step 1: Collect raw DOM data (Main World - CSP can't block this!) const rawData = []; window.sentience_registry = []; @@ -890,17 +890,17 @@ const textVal = getText(el); const inView = isInViewport(rect); - + // Get computed style once (needed for both occlusion check and data collection) const style = window.getComputedStyle(el); - + // Only check occlusion for elements likely to be occluded (optimized) // This avoids layout thrashing for the vast majority of elements const occluded = inView ? isOccluded(el, rect, style) : false; - + // Get effective background color (traverses DOM to find non-transparent color) const effectiveBgColor = getEffectiveBackgroundColor(el); - + rawData.push({ id: idx, tag: el.tagName.toLowerCase(), @@ -937,26 +937,26 @@ // This allows WASM to process all elements uniformly (no recursion needed) let allRawElements = [...rawData]; // Start with main frame elements let totalIframeElements = 0; - + if (options.collectIframes !== false) { try { console.log(`[SentienceAPI] Starting iframe collection...`); const iframeSnapshots = await collectIframeSnapshots(options); console.log(`[SentienceAPI] Iframe collection complete. Received ${iframeSnapshots.size} snapshot(s)`); - + if (iframeSnapshots.size > 0) { // FLATTEN IMMEDIATELY: Don't nest them. Just append them with coordinate translation. iframeSnapshots.forEach((iframeSnapshot, iframeEl) => { // Debug: Log structure to verify data is correct // console.log(`[SentienceAPI] Processing iframe snapshot:`, iframeSnapshot); - + if (iframeSnapshot && iframeSnapshot.raw_elements) { const rawElementsCount = iframeSnapshot.raw_elements.length; console.log(`[SentienceAPI] Processing ${rawElementsCount} elements from iframe (src: ${iframeEl.src || 'unknown'})`); // Get iframe's bounding rect (offset for coordinate translation) const iframeRect = iframeEl.getBoundingClientRect(); const offset = { x: iframeRect.x, y: iframeRect.y }; - + // Get iframe context for frame switching (Playwright needs this) const iframeSrc = iframeEl.src || iframeEl.getAttribute('src') || ''; let isSameOrigin = false; @@ -966,11 +966,11 @@ } catch (e) { isSameOrigin = false; } - + // Adjust coordinates and add iframe context to each element const adjustedElements = iframeSnapshot.raw_elements.map(el => { const adjusted = { ...el }; - + // Adjust rect coordinates to parent viewport if (adjusted.rect) { adjusted.rect = { @@ -979,22 +979,22 @@ y: adjusted.rect.y + offset.y }; } - + // Add iframe context so agents can switch frames in Playwright adjusted.iframe_context = { src: iframeSrc, is_same_origin: isSameOrigin }; - + return adjusted; }); - + // Append flattened iframe elements to main array allRawElements.push(...adjustedElements); totalIframeElements += adjustedElements.length; } }); - + // console.log(`[SentienceAPI] Merged ${iframeSnapshots.size} iframe(s). Total elements: ${allRawElements.length} (${rawData.length} main + ${totalIframeElements} iframe)`); } } catch (error) { @@ -1007,7 +1007,7 @@ // No recursion needed - everything is already flat console.log(`[SentienceAPI] Sending ${allRawElements.length} total elements to WASM (${rawData.length} main + ${totalIframeElements} iframe)`); const processed = await processSnapshotInBackground(allRawElements, options); - + if (!processed || !processed.elements) { throw new Error('WASM processing returned invalid result'); } @@ -1023,10 +1023,10 @@ const cleanedRawElements = cleanElement(processed.raw_elements); // FIXED: Removed undefined 'totalIframeRawElements' - // FIXED: Logic updated for "Flatten Early" architecture. + // FIXED: Logic updated for "Flatten Early" architecture. // processed.elements ALREADY contains the merged iframe elements, // so we simply use .length. No addition needed. - + const totalCount = cleanedElements.length; const totalRaw = cleanedRawElements.length; const iframeCount = totalIframeElements || 0; @@ -1092,23 +1092,23 @@ autoDisableTimeout = 30 * 60 * 1000, // 30 minutes default keyboardShortcut = 'Ctrl+Shift+I' } = options; - + console.log("šŸ”“ [Sentience] Recording Mode STARTED. Click an element to copy its Ground Truth JSON."); console.log(` Press ${keyboardShortcut} or call stopRecording() to stop.`); - + // Validate registry is populated if (!window.sentience_registry || window.sentience_registry.length === 0) { console.warn("āš ļø Registry empty. Call `await window.sentience.snapshot()` first to populate registry."); alert("Registry empty. Run `await window.sentience.snapshot()` first!"); return () => {}; // Return no-op cleanup function } - + // Create reverse mapping for O(1) lookup (fixes registry lookup bug) window.sentience_registry_map = new Map(); window.sentience_registry.forEach((el, idx) => { if (el) window.sentience_registry_map.set(el, idx); }); - + // Create highlight box overlay let highlightBox = document.getElementById('sentience-highlight-box'); if (!highlightBox) { @@ -1126,7 +1126,7 @@ `; document.body.appendChild(highlightBox); } - + // Create visual indicator (red border on page when recording) let recordingIndicator = document.getElementById('sentience-recording-indicator'); if (!recordingIndicator) { @@ -1145,12 +1145,12 @@ document.body.appendChild(recordingIndicator); } recordingIndicator.style.display = 'block'; - + // Hover handler (visual feedback) const mouseOverHandler = (e) => { const el = e.target; if (!el || el === highlightBox || el === recordingIndicator) return; - + const rect = el.getBoundingClientRect(); highlightBox.style.display = 'block'; highlightBox.style.top = (rect.top + window.scrollY) + 'px'; @@ -1158,15 +1158,15 @@ highlightBox.style.width = rect.width + 'px'; highlightBox.style.height = rect.height + 'px'; }; - + // Click handler (capture ground truth data) const clickHandler = (e) => { e.preventDefault(); e.stopPropagation(); - + const el = e.target; if (!el || el === highlightBox || el === recordingIndicator) return; - + // Use Map for reliable O(1) lookup const sentienceId = window.sentience_registry_map.get(el); if (sentienceId === undefined) { @@ -1174,13 +1174,13 @@ alert("Element not in registry. Run `await window.sentience.snapshot()` first!"); return; } - + // Extract raw data (ground truth + raw signals, NOT model outputs) const rawData = extractRawElementData(el); const selector = getUniqueSelector(el); const role = el.getAttribute('role') || el.tagName.toLowerCase(); const text = getText(el); - + // Build golden set JSON (ground truth + raw signals only) const snippet = { task: `Interact with ${text.substring(0, 20)}${text.length > 20 ? '...' : ''}`, @@ -1194,12 +1194,12 @@ }, debug_snapshot: rawData }; - + // Copy to clipboard const jsonString = JSON.stringify(snippet, null, 2); navigator.clipboard.writeText(jsonString).then(() => { console.log("āœ… Copied Ground Truth to clipboard:", snippet); - + // Flash green to indicate success highlightBox.style.border = `2px solid ${successColor}`; highlightBox.style.background = 'rgba(0, 255, 0, 0.2)'; @@ -1212,42 +1212,42 @@ alert("Failed to copy to clipboard. Check console for JSON."); }); }; - + // Auto-disable timeout let timeoutId = null; - + // Cleanup function to stop recording (defined before use) const stopRecording = () => { document.removeEventListener('mouseover', mouseOverHandler, true); document.removeEventListener('click', clickHandler, true); document.removeEventListener('keydown', keyboardHandler, true); - + if (timeoutId) { clearTimeout(timeoutId); timeoutId = null; } - + if (highlightBox) { highlightBox.style.display = 'none'; } - + if (recordingIndicator) { recordingIndicator.style.display = 'none'; } - + // Clean up registry map (optional, but good practice) if (window.sentience_registry_map) { window.sentience_registry_map.clear(); } - + // Remove global reference if (window.sentience_stopRecording === stopRecording) { delete window.sentience_stopRecording; } - + console.log("āšŖ [Sentience] Recording Mode STOPPED."); }; - + // Keyboard shortcut handler (defined after stopRecording) const keyboardHandler = (e) => { // Ctrl+Shift+I or Cmd+Shift+I @@ -1256,12 +1256,12 @@ stopRecording(); } }; - + // Attach event listeners (use capture phase to intercept early) document.addEventListener('mouseover', mouseOverHandler, true); document.addEventListener('click', clickHandler, true); document.addEventListener('keydown', keyboardHandler, true); - + // Set up auto-disable timeout if (autoDisableTimeout > 0) { timeoutId = setTimeout(() => { @@ -1269,10 +1269,10 @@ stopRecording(); }, autoDisableTimeout); } - + // Store stop function globally for keyboard shortcut access window.sentience_stopRecording = stopRecording; - + return stopRecording; } }; diff --git a/sentience/tracer_factory.py b/sentience/tracer_factory.py new file mode 100644 index 0000000..06d1324 --- /dev/null +++ b/sentience/tracer_factory.py @@ -0,0 +1,105 @@ +""" +Tracer factory with automatic tier detection. + +Provides convenient factory function for creating tracers with cloud upload support. +""" + +import uuid +from pathlib import Path +from typing import Optional + +import requests + +from sentience.cloud_tracing import CloudTraceSink +from sentience.tracing import JsonlTraceSink, Tracer + + +def create_tracer( + api_key: str | None = None, + run_id: str | None = None, + api_url: str = "https://api.sentienceapi.com", +) -> Tracer: + """ + Create tracer with automatic tier detection. + + Tier Detection: + - If api_key is provided: Try to initialize CloudTraceSink (Pro/Enterprise) + - If cloud init fails or no api_key: Fall back to JsonlTraceSink (Free tier) + + Args: + api_key: Sentience API key (e.g., "sk_pro_xxxxx") + - Free tier: None or empty + - Pro/Enterprise: Valid API key + run_id: Unique identifier for this agent run. If not provided, generates UUID. + api_url: Sentience API base URL (default: https://api.sentienceapi.com) + + Returns: + Tracer configured with appropriate sink + + Example: + >>> # Pro tier user + >>> tracer = create_tracer(api_key="sk_pro_xyz", run_id="demo") + >>> # Returns: Tracer with CloudTraceSink + >>> + >>> # Free tier user + >>> tracer = create_tracer(run_id="demo") + >>> # Returns: Tracer with JsonlTraceSink (local-only) + >>> + >>> # Use with agent + >>> agent = SentienceAgent(browser, llm, tracer=tracer) + >>> agent.act("Click search") + >>> tracer.close() # Uploads to cloud if Pro tier + """ + if run_id is None: + run_id = str(uuid.uuid4()) + + # 1. Try to initialize Cloud Sink (Pro/Enterprise tier) + if api_key: + try: + # Request pre-signed upload URL from backend + response = requests.post( + f"{api_url}/v1/traces/init", + headers={"Authorization": f"Bearer {api_key}"}, + json={"run_id": run_id}, + timeout=10, + ) + + if response.status_code == 200: + data = response.json() + upload_url = data.get("upload_url") + + if upload_url: + print("ā˜ļø [Sentience] Cloud tracing enabled (Pro tier)") + return Tracer( + run_id=run_id, + sink=CloudTraceSink(upload_url=upload_url), + ) + else: + print("āš ļø [Sentience] Cloud init response missing upload_url") + print(" Falling back to local-only tracing") + + elif response.status_code == 403: + print("āš ļø [Sentience] Cloud tracing requires Pro tier") + print(" Falling back to local-only tracing") + else: + print(f"āš ļø [Sentience] Cloud init failed: HTTP {response.status_code}") + print(" Falling back to local-only tracing") + + except requests.exceptions.Timeout: + print("āš ļø [Sentience] Cloud init timeout") + print(" Falling back to local-only tracing") + except requests.exceptions.ConnectionError: + print("āš ļø [Sentience] Cloud init connection error") + print(" Falling back to local-only tracing") + except Exception as e: + print(f"āš ļø [Sentience] Cloud init error: {e}") + print(" Falling back to local-only tracing") + + # 2. Fallback to Local Sink (Free tier / Offline mode) + traces_dir = Path("traces") + traces_dir.mkdir(exist_ok=True) + + local_path = traces_dir / f"{run_id}.jsonl" + print(f"šŸ’¾ [Sentience] Local tracing: {local_path}") + + return Tracer(run_id=run_id, sink=JsonlTraceSink(str(local_path))) diff --git a/sentience_python.egg-info/PKG-INFO b/sentience_python.egg-info/PKG-INFO index 78aeea8..61bd6e0 100644 --- a/sentience_python.egg-info/PKG-INFO +++ b/sentience_python.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: sentience-python -Version: 0.12.0 +Version: 0.12.1 Summary: Python SDK for Sentience AI Agent Browser Automation Author: Sentience Team License: MIT diff --git a/sentience_python.egg-info/SOURCES.txt b/sentience_python.egg-info/SOURCES.txt index 5b5b22e..7997f40 100644 --- a/sentience_python.egg-info/SOURCES.txt +++ b/sentience_python.egg-info/SOURCES.txt @@ -9,6 +9,7 @@ sentience/agent_config.py sentience/base_agent.py sentience/browser.py sentience/cli.py +sentience/cloud_tracing.py sentience/conversational_agent.py sentience/expect.py sentience/formatting.py @@ -21,6 +22,7 @@ sentience/read.py sentience/recorder.py sentience/screenshot.py sentience/snapshot.py +sentience/tracer_factory.py sentience/tracing.py sentience/utils.py sentience/wait.py @@ -39,6 +41,7 @@ tests/test_actions.py tests/test_agent.py tests/test_agent_config.py tests/test_bot.py +tests/test_cloud_tracing.py tests/test_conversational_agent.py tests/test_formatting.py tests/test_generator.py diff --git a/tests/test_cloud_tracing.py b/tests/test_cloud_tracing.py new file mode 100644 index 0000000..f7ef40b --- /dev/null +++ b/tests/test_cloud_tracing.py @@ -0,0 +1,347 @@ +"""Tests for sentience.cloud_tracing module""" + +import gzip +import json +import tempfile +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch + +import pytest + +from sentience.cloud_tracing import CloudTraceSink +from sentience.tracer_factory import create_tracer +from sentience.tracing import JsonlTraceSink, Tracer + + +class TestCloudTraceSink: + """Test CloudTraceSink functionality.""" + + def test_cloud_trace_sink_upload_success(self): + """Test CloudTraceSink successfully uploads trace to cloud.""" + upload_url = "https://sentience.nyc3.digitaloceanspaces.com/user123/run456/trace.jsonl.gz" + + with patch("sentience.cloud_tracing.requests.put") as mock_put: + # Mock successful response + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = "Success" + mock_put.return_value = mock_response + + # Create sink and emit events + sink = CloudTraceSink(upload_url) + sink.emit({"v": 1, "type": "run_start", "seq": 1, "data": {"agent": "TestAgent"}}) + sink.emit({"v": 1, "type": "run_end", "seq": 2, "data": {"steps": 1}}) + + # Close triggers upload + sink.close() + + # Verify request was made + assert mock_put.called + assert mock_put.call_count == 1 + + # Verify URL and headers + call_args = mock_put.call_args + assert call_args[0][0] == upload_url + assert call_args[1]["headers"]["Content-Type"] == "application/x-gzip" + assert call_args[1]["headers"]["Content-Encoding"] == "gzip" + + # Verify body is gzip compressed + uploaded_data = call_args[1]["data"] + decompressed = gzip.decompress(uploaded_data) + lines = decompressed.decode("utf-8").strip().split("\n") + + assert len(lines) == 2 + event1 = json.loads(lines[0]) + event2 = json.loads(lines[1]) + + assert event1["type"] == "run_start" + assert event2["type"] == "run_end" + + def test_cloud_trace_sink_upload_failure_preserves_trace(self, capsys): + """Test CloudTraceSink preserves trace locally on upload failure.""" + upload_url = "https://sentience.nyc3.digitaloceanspaces.com/user123/run456/trace.jsonl.gz" + + with patch("sentience.cloud_tracing.requests.put") as mock_put: + # Mock failed response + mock_response = Mock() + mock_response.status_code = 500 + mock_response.text = "Internal Server Error" + mock_put.return_value = mock_response + + # Create sink and emit events + sink = CloudTraceSink(upload_url) + sink.emit({"v": 1, "type": "run_start", "seq": 1}) + + # Close triggers upload (which will fail) + sink.close() + + # Verify error message printed + captured = capsys.readouterr() + assert "āŒ" in captured.out + assert "Upload failed: HTTP 500" in captured.out + assert "Local trace preserved" in captured.out + + def test_cloud_trace_sink_emit_after_close_raises(self): + """Test CloudTraceSink raises error when emitting after close.""" + upload_url = "https://test.com/upload" + sink = CloudTraceSink(upload_url) + sink.close() + + with pytest.raises(RuntimeError, match="CloudTraceSink is closed"): + sink.emit({"v": 1, "type": "test", "seq": 1}) + + def test_cloud_trace_sink_context_manager(self): + """Test CloudTraceSink works as context manager.""" + with patch("sentience.cloud_tracing.requests.put") as mock_put: + mock_put.return_value = Mock(status_code=200) + + upload_url = "https://test.com/upload" + with CloudTraceSink(upload_url) as sink: + sink.emit({"v": 1, "type": "test", "seq": 1}) + + # Verify upload was called + assert mock_put.called + + def test_cloud_trace_sink_network_error_graceful_degradation(self, capsys): + """Test CloudTraceSink handles network errors gracefully.""" + upload_url = "https://sentience.nyc3.digitaloceanspaces.com/user123/run456/trace.jsonl.gz" + + with patch("sentience.cloud_tracing.requests.put") as mock_put: + # Simulate network error + mock_put.side_effect = Exception("Network error") + + sink = CloudTraceSink(upload_url) + sink.emit({"v": 1, "type": "test", "seq": 1}) + + # Should not raise, just print warning + sink.close() + + captured = capsys.readouterr() + assert "āŒ" in captured.out + assert "Error uploading trace" in captured.out + + def test_cloud_trace_sink_multiple_close_safe(self): + """Test CloudTraceSink.close() is idempotent.""" + with patch("sentience.cloud_tracing.requests.put") as mock_put: + mock_put.return_value = Mock(status_code=200) + + upload_url = "https://test.com/upload" + sink = CloudTraceSink(upload_url) + sink.emit({"v": 1, "type": "test", "seq": 1}) + + # Close multiple times + sink.close() + sink.close() + sink.close() + + # Upload should only be called once + assert mock_put.call_count == 1 + + +class TestTracerFactory: + """Test create_tracer factory function.""" + + def test_create_tracer_pro_tier_success(self, capsys): + """Test create_tracer returns CloudTraceSink for Pro tier.""" + with patch("sentience.tracer_factory.requests.post") as mock_post: + with patch("sentience.cloud_tracing.requests.put") as mock_put: + # Mock API response + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "upload_url": "https://sentience.nyc3.digitaloceanspaces.com/upload" + } + mock_post.return_value = mock_response + + # Mock upload response + mock_put.return_value = Mock(status_code=200) + + tracer = create_tracer(api_key="sk_pro_test123", run_id="test-run") + + # Verify Pro tier message + captured = capsys.readouterr() + assert "ā˜ļø [Sentience] Cloud tracing enabled (Pro tier)" in captured.out + + # Verify tracer works + assert tracer.run_id == "test-run" + assert isinstance(tracer.sink, CloudTraceSink) + + # Cleanup + tracer.close() + + def test_create_tracer_free_tier_fallback(self, capsys): + """Test create_tracer falls back to local for free tier.""" + with tempfile.TemporaryDirectory(): + tracer = create_tracer(run_id="test-run") + + # Verify local tracing message + captured = capsys.readouterr() + assert "šŸ’¾ [Sentience] Local tracing:" in captured.out + assert "traces/test-run.jsonl" in captured.out + + # Verify tracer works + assert tracer.run_id == "test-run" + assert isinstance(tracer.sink, JsonlTraceSink) + + # Cleanup + tracer.close() + + def test_create_tracer_api_forbidden_fallback(self, capsys): + """Test create_tracer falls back when API returns 403 Forbidden.""" + with patch("sentience.tracer_factory.requests.post") as mock_post: + # Mock API response with 403 + mock_response = Mock() + mock_response.status_code = 403 + mock_post.return_value = mock_response + + with tempfile.TemporaryDirectory(): + tracer = create_tracer(api_key="sk_free_test123", run_id="test-run") + + # Verify warning message + captured = capsys.readouterr() + assert "āš ļø [Sentience] Cloud tracing requires Pro tier" in captured.out + assert "Falling back to local-only tracing" in captured.out + + # Verify fallback to local + assert isinstance(tracer.sink, JsonlTraceSink) + + tracer.close() + + def test_create_tracer_api_timeout_fallback(self, capsys): + """Test create_tracer falls back on timeout.""" + import requests + + with patch("sentience.tracer_factory.requests.post") as mock_post: + # Mock timeout + mock_post.side_effect = requests.exceptions.Timeout("Connection timeout") + + with tempfile.TemporaryDirectory(): + tracer = create_tracer(api_key="sk_test123", run_id="test-run") + + # Verify warning message + captured = capsys.readouterr() + assert "āš ļø [Sentience] Cloud init timeout" in captured.out + assert "Falling back to local-only tracing" in captured.out + + # Verify fallback to local + assert isinstance(tracer.sink, JsonlTraceSink) + + tracer.close() + + def test_create_tracer_api_connection_error_fallback(self, capsys): + """Test create_tracer falls back on connection error.""" + import requests + + with patch("sentience.tracer_factory.requests.post") as mock_post: + # Mock connection error + mock_post.side_effect = requests.exceptions.ConnectionError("Connection refused") + + with tempfile.TemporaryDirectory(): + tracer = create_tracer(api_key="sk_test123", run_id="test-run") + + # Verify warning message + captured = capsys.readouterr() + assert "āš ļø [Sentience] Cloud init connection error" in captured.out + + # Verify fallback to local + assert isinstance(tracer.sink, JsonlTraceSink) + + tracer.close() + + def test_create_tracer_generates_run_id_if_not_provided(self): + """Test create_tracer generates UUID if run_id not provided.""" + with tempfile.TemporaryDirectory(): + tracer = create_tracer() + + # Verify run_id was generated + assert tracer.run_id is not None + assert len(tracer.run_id) == 36 # UUID format + + tracer.close() + + def test_create_tracer_custom_api_url(self): + """Test create_tracer with custom API URL.""" + custom_url = "https://custom.api.com" + + with patch("sentience.tracer_factory.requests.post") as mock_post: + with patch("sentience.cloud_tracing.requests.put") as mock_put: + # Mock API response + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"upload_url": "https://storage.com/upload"} + mock_post.return_value = mock_response + mock_put.return_value = Mock(status_code=200) + + tracer = create_tracer(api_key="sk_test123", run_id="test-run", api_url=custom_url) + + # Verify correct API URL was used + assert mock_post.called + call_args = mock_post.call_args + assert call_args[0][0] == f"{custom_url}/v1/traces/init" + + tracer.close() + + def test_create_tracer_missing_upload_url_in_response(self, capsys): + """Test create_tracer handles missing upload_url gracefully.""" + with patch("sentience.tracer_factory.requests.post") as mock_post: + # Mock API response without upload_url + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"message": "Success"} # Missing upload_url + mock_post.return_value = mock_response + + with tempfile.TemporaryDirectory(): + tracer = create_tracer(api_key="sk_test123", run_id="test-run") + + # Verify warning message + captured = capsys.readouterr() + assert "āš ļø [Sentience] Cloud init response missing upload_url" in captured.out + + # Verify fallback to local + assert isinstance(tracer.sink, JsonlTraceSink) + + tracer.close() + + +class TestRegressionTests: + """Regression tests to ensure cloud tracing doesn't break existing functionality.""" + + def test_local_tracing_still_works(self): + """Test existing JsonlTraceSink functionality unchanged.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "trace.jsonl" + + with JsonlTraceSink(trace_path) as sink: + tracer = Tracer(run_id="test-run", sink=sink) + tracer.emit_run_start("TestAgent", "gpt-4") + tracer.emit_run_end(1) + + # Verify trace file created + assert trace_path.exists() + + lines = trace_path.read_text().strip().split("\n") + assert len(lines) == 2 + + event1 = json.loads(lines[0]) + assert event1["type"] == "run_start" + + def test_tracer_api_unchanged(self): + """Test Tracer API hasn't changed.""" + with tempfile.TemporaryDirectory() as tmpdir: + trace_path = Path(tmpdir) / "trace.jsonl" + sink = JsonlTraceSink(trace_path) + + # All existing methods should still work + tracer = Tracer(run_id="test-run", sink=sink) + + tracer.emit("custom_event", {"data": "value"}) + tracer.emit_run_start("TestAgent") + tracer.emit_step_start("step-1", 1, "Test goal") + tracer.emit_error("step-1", "Test error") + tracer.emit_run_end(1) + + tracer.close() + + # Verify all events written + lines = trace_path.read_text().strip().split("\n") + assert len(lines) == 5 diff --git a/traces/c8d99417-49ca-4195-bc66-face72cc9494.jsonl b/traces/c8d99417-49ca-4195-bc66-face72cc9494.jsonl new file mode 100644 index 0000000..e69de29 diff --git a/traces/test-run.jsonl b/traces/test-run.jsonl new file mode 100644 index 0000000..e69de29 From 700647449e9e121b9766a1cde2ce35cabf25714d Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 20:19:28 -0800 Subject: [PATCH 2/7] fix tests --- tests/test_cloud_tracing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_cloud_tracing.py b/tests/test_cloud_tracing.py index f7ef40b..1b41d57 100644 --- a/tests/test_cloud_tracing.py +++ b/tests/test_cloud_tracing.py @@ -177,7 +177,11 @@ def test_create_tracer_free_tier_fallback(self, capsys): # Verify local tracing message captured = capsys.readouterr() assert "šŸ’¾ [Sentience] Local tracing:" in captured.out - assert "traces/test-run.jsonl" in captured.out + # Use os.path.join for platform-independent path checking + import os + + expected_path = os.path.join("traces", "test-run.jsonl") + assert expected_path in captured.out # Verify tracer works assert tracer.run_id == "test-run" From 7e6479f1e272f5b233897798d7c8554f8b20cc84 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 22:05:36 -0800 Subject: [PATCH 3/7] address risks identified in plan doc --- examples/click_rect_demo.py | 7 +- examples/test_local_llm_agent.py | 5 +- sentience/__init__.py | 3 +- sentience/actions.py | 18 ++- sentience/agent.py | 23 ++-- sentience/cli.py | 4 +- sentience/cloud_tracing.py | 110 ++++++++++++------ sentience/conversational_agent.py | 32 +++-- sentience/recorder.py | 17 ++- sentience/snapshot.py | 3 +- sentience/tracer_factory.py | 99 +++++++++++++++- tests/conftest.py | 3 +- tests/test_actions.py | 7 +- tests/test_agent.py | 3 - tests/test_cloud_tracing.py | 180 +++++++++++++++++++++++++++-- tests/test_conversational_agent.py | 8 +- tests/test_stealth.py | 6 +- 17 files changed, 438 insertions(+), 90 deletions(-) diff --git a/examples/click_rect_demo.py b/examples/click_rect_demo.py index 0c2148c..d472c20 100644 --- a/examples/click_rect_demo.py +++ b/examples/click_rect_demo.py @@ -37,7 +37,12 @@ def main(): print(" Clicking at center of element's bbox...") result = click_rect( browser, - {"x": link.bbox.x, "y": link.bbox.y, "w": link.bbox.width, "h": link.bbox.height}, + { + "x": link.bbox.x, + "y": link.bbox.y, + "w": link.bbox.width, + "h": link.bbox.height, + }, ) print(f" Result: success={result.success}, outcome={result.outcome}") print(f" URL changed: {result.url_changed}\n") diff --git a/examples/test_local_llm_agent.py b/examples/test_local_llm_agent.py index 8dcf1ca..39132a0 100644 --- a/examples/test_local_llm_agent.py +++ b/examples/test_local_llm_agent.py @@ -65,7 +65,10 @@ def test_local_llm_basic(): user_prompt = "What is the next step to achieve the goal?" response = llm.generate( - system_prompt=system_prompt, user_prompt=user_prompt, max_new_tokens=20, temperature=0.0 + system_prompt=system_prompt, + user_prompt=user_prompt, + max_new_tokens=20, + temperature=0.0, ) print(f"Agent Response: {response.content}") diff --git a/sentience/__init__.py b/sentience/__init__.py index 8bb92e5..c4b7d03 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -46,7 +46,7 @@ from .recorder import Recorder, Trace, TraceStep, record from .screenshot import screenshot from .snapshot import snapshot -from .tracer_factory import create_tracer +from .tracer_factory import SENTIENCE_API_URL, create_tracer from .tracing import JsonlTraceSink, TraceEvent, Tracer, TraceSink # Utilities (v0.12.0+) @@ -112,6 +112,7 @@ "CloudTraceSink", "TraceEvent", "create_tracer", + "SENTIENCE_API_URL", # Utilities (v0.12.0+) "canonical_snapshot_strict", "canonical_snapshot_loose", diff --git a/sentience/actions.py b/sentience/actions.py index 2ce8a3f..5df42fc 100644 --- a/sentience/actions.py +++ b/sentience/actions.py @@ -3,14 +3,13 @@ """ import time -from typing import Any, Dict, Optional from .browser import SentienceBrowser from .models import ActionResult, BBox, Snapshot from .snapshot import snapshot -def click( +def click( # noqa: C901 browser: SentienceBrowser, element_id: int, use_mouse: bool = True, @@ -141,7 +140,10 @@ def click( error=( None if success - else {"code": "click_failed", "reason": "Element not found or not clickable"} + else { + "code": "click_failed", + "reason": "Element not found or not clickable", + } ), ) @@ -371,7 +373,10 @@ def click_rect( success=False, duration_ms=0, outcome="error", - error={"code": "invalid_rect", "reason": "Rectangle width and height must be positive"}, + error={ + "code": "invalid_rect", + "reason": "Rectangle width and height must be positive", + }, ) start_time = time.time() @@ -426,6 +431,9 @@ def click_rect( error=( None if success - else {"code": "click_failed", "reason": error_msg if not success else "Click failed"} + else { + "code": "click_failed", + "reason": error_msg if not success else "Click failed", + } ), ) diff --git a/sentience/agent.py b/sentience/agent.py index 8a67900..a7dd305 100644 --- a/sentience/agent.py +++ b/sentience/agent.py @@ -5,7 +5,7 @@ import re import time -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Optional from .actions import click, press, type_text from .base_agent import BaseAgent @@ -93,8 +93,11 @@ def __init__( # Step counter for tracing self._step_count = 0 - def act( - self, goal: str, max_retries: int = 2, snapshot_options: SnapshotOptions | None = None + def act( # noqa: C901 + self, + goal: str, + max_retries: int = 2, + snapshot_options: SnapshotOptions | None = None, ) -> AgentActionResult: """ Execute a high-level goal using observe ā†’ think ā†’ act loop @@ -116,9 +119,9 @@ def act( 42 """ if self.verbose: - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print(f"šŸ¤– Agent Goal: {goal}") - print(f"{'='*70}") + print(f"{'=' * 70}") # Generate step ID for tracing self._step_count += 1 @@ -460,7 +463,9 @@ def _execute_action(self, action_str: str, snap: Snapshot) -> dict[str, Any]: # Parse TYPE(42, "hello world") elif match := re.match( - r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)', action_str, re.IGNORECASE + r'TYPE\s*\(\s*(\d+)\s*,\s*["\']([^"\']*)["\']\s*\)', + action_str, + re.IGNORECASE, ): element_id = int(match.group(1)) text = match.group(2) @@ -486,7 +491,11 @@ def _execute_action(self, action_str: str, snap: Snapshot) -> dict[str, Any]: # Parse FINISH() elif re.match(r"FINISH\s*\(\s*\)", action_str, re.IGNORECASE): - return {"success": True, "action": "finish", "message": "Task marked as complete"} + return { + "success": True, + "action": "finish", + "message": "Task marked as complete", + } else: raise ValueError( diff --git a/sentience/cli.py b/sentience/cli.py index 64112e9..c5f669c 100644 --- a/sentience/cli.py +++ b/sentience/cli.py @@ -104,7 +104,9 @@ def main(): "--snapshots", action="store_true", help="Capture snapshots at each step" ) record_parser.add_argument( - "--mask", action="append", help="Pattern to mask in recorded text (e.g., password)" + "--mask", + action="append", + help="Pattern to mask in recorded text (e.g., password)", ) record_parser.set_defaults(func=cmd_record) diff --git a/sentience/cloud_tracing.py b/sentience/cloud_tracing.py index 363aeab..7418d07 100644 --- a/sentience/cloud_tracing.py +++ b/sentience/cloud_tracing.py @@ -7,12 +7,14 @@ import gzip import json import os -import tempfile +import threading +from collections.abc import Callable +from pathlib import Path from typing import Any import requests -from sentience.tracing import TraceEvent, TraceSink +from sentience.tracing import TraceSink class CloudTraceSink(TraceSink): @@ -20,15 +22,17 @@ class CloudTraceSink(TraceSink): Enterprise Cloud Sink: "Local Write, Batch Upload" pattern. Architecture: - 1. **Local Buffer**: Writes to temp file (zero latency, non-blocking) + 1. **Local Buffer**: Writes to persistent cache directory (zero latency, non-blocking) 2. **Pre-signed URL**: Uses secure pre-signed PUT URL from backend API 3. **Batch Upload**: Uploads complete file on close() or at intervals 4. **Zero Credential Exposure**: Never embeds DigitalOcean credentials in SDK + 5. **Crash Recovery**: Traces survive process crashes (stored in ~/.sentience/traces/pending/) This design ensures: - Fast agent performance (microseconds per emit, not milliseconds) - Security (credentials stay on backend) - Reliability (network issues don't crash the agent) + - Data durability (traces survive crashes and can be recovered) Tiered Access: - Free Tier: Falls back to JsonlTraceSink (local-only) @@ -39,36 +43,40 @@ class CloudTraceSink(TraceSink): >>> from sentience.tracing import Tracer >>> # Get upload URL from API >>> upload_url = "https://sentience.nyc3.digitaloceanspaces.com/..." - >>> sink = CloudTraceSink(upload_url) + >>> sink = CloudTraceSink(upload_url, run_id="demo") >>> tracer = Tracer(run_id="demo", sink=sink) >>> tracer.emit_run_start("SentienceAgent") >>> tracer.close() # Uploads to cloud + >>> # Or non-blocking: + >>> tracer.close(blocking=False) # Returns immediately """ - def __init__(self, upload_url: str): + def __init__(self, upload_url: str, run_id: str): """ Initialize cloud trace sink. Args: upload_url: Pre-signed PUT URL from Sentience API (e.g., "https://sentience.nyc3.digitaloceanspaces.com/...") + run_id: Unique identifier for this agent run (used for persistent cache) """ self.upload_url = upload_url + self.run_id = run_id - # Create temporary file for buffering - # delete=False so we can read it back before uploading - self._temp_file = tempfile.NamedTemporaryFile( - mode="w+", - encoding="utf-8", - suffix=".jsonl", - delete=False, - ) - self._path = self._temp_file.name + # Use persistent cache directory instead of temp file + # This ensures traces survive process crashes + cache_dir = Path.home() / ".sentience" / "traces" / "pending" + cache_dir.mkdir(parents=True, exist_ok=True) + + # Persistent file (survives process crash) + self._path = cache_dir / f"{run_id}.jsonl" + self._trace_file = open(self._path, "w", encoding="utf-8") self._closed = False + self._upload_successful = False def emit(self, event: dict[str, Any]) -> None: """ - Write event to local temp file (Fast, non-blocking). + Write event to local persistent file (Fast, non-blocking). Performance: ~10 microseconds per write vs ~50ms for HTTP request @@ -79,13 +87,21 @@ def emit(self, event: dict[str, Any]) -> None: raise RuntimeError("CloudTraceSink is closed") json_str = json.dumps(event, ensure_ascii=False) - self._temp_file.write(json_str + "\n") - self._temp_file.flush() # Ensure written to disk - - def close(self) -> None: + self._trace_file.write(json_str + "\n") + self._trace_file.flush() # Ensure written to disk + + def close( + self, + blocking: bool = True, + on_progress: Callable[[int, int], None] | None = None, + ) -> None: """ Upload buffered trace to cloud via pre-signed URL. + Args: + blocking: If False, returns immediately and uploads in background thread + on_progress: Optional callback(uploaded_bytes, total_bytes) for progress updates + This is the only network call - happens once at the end. """ if self._closed: @@ -93,17 +109,46 @@ def close(self) -> None: self._closed = True + # Close file first + self._trace_file.close() + + if not blocking: + # Fire-and-forget background upload + thread = threading.Thread( + target=self._do_upload, + args=(on_progress,), + daemon=True, + ) + thread.start() + return # Return immediately + + # Blocking mode + self._do_upload(on_progress) + + def _do_upload(self, on_progress: Callable[[int, int], None] | None = None) -> None: + """ + Internal upload method with progress tracking. + + Args: + on_progress: Optional callback(uploaded_bytes, total_bytes) for progress updates + """ try: - # 1. Close temp file - self._temp_file.close() + # Read file size for progress + file_size = os.path.getsize(self._path) + + if on_progress: + on_progress(0, file_size) - # 2. Compress for upload + # Read and compress with open(self._path, "rb") as f: trace_data = f.read() compressed_data = gzip.compress(trace_data) - # 3. Upload to DigitalOcean Spaces via pre-signed URL + if on_progress: + on_progress(len(compressed_data), file_size) + + # Upload to DigitalOcean Spaces via pre-signed URL print(f"šŸ“¤ [Sentience] Uploading trace to cloud ({len(compressed_data)} bytes)...") response = requests.put( @@ -117,27 +162,26 @@ def close(self) -> None: ) if response.status_code == 200: + self._upload_successful = True print("āœ… [Sentience] Trace uploaded successfully") + # Delete file only on successful upload + if os.path.exists(self._path): + try: + os.remove(self._path) + except Exception: + pass # Ignore cleanup errors else: + self._upload_successful = False print(f"āŒ [Sentience] Upload failed: HTTP {response.status_code}") print(f" Response: {response.text}") print(f" Local trace preserved at: {self._path}") except Exception as e: + self._upload_successful = False print(f"āŒ [Sentience] Error uploading trace: {e}") print(f" Local trace preserved at: {self._path}") # Don't raise - preserve trace locally even if upload fails - finally: - # 4. Cleanup temp file (only if upload succeeded) - if os.path.exists(self._path): - try: - # Only delete if upload was successful - if hasattr(self, "_upload_successful") and self._upload_successful: - os.remove(self._path) - except Exception: - pass # Ignore cleanup errors - def __enter__(self): """Context manager support.""" return self diff --git a/sentience/conversational_agent.py b/sentience/conversational_agent.py index 27a6943..29fc58d 100644 --- a/sentience/conversational_agent.py +++ b/sentience/conversational_agent.py @@ -5,11 +5,11 @@ import json import time -from typing import Any, Dict, List, Optional +from typing import Any from .agent import SentienceAgent from .browser import SentienceBrowser -from .llm_provider import LLMProvider, LLMResponse +from .llm_provider import LLMProvider from .models import Snapshot from .snapshot import snapshot @@ -70,9 +70,9 @@ def execute(self, user_input: str) -> str: The top result is from amazon.com selling Magic Mouse 2 for $79." """ if self.verbose: - print(f"\n{'='*70}") + print(f"\n{'=' * 70}") print(f"šŸ‘¤ User: {user_input}") - print(f"{'='*70}") + print(f"{'=' * 70}") start_time = time.time() @@ -80,7 +80,7 @@ def execute(self, user_input: str) -> str: plan = self._create_plan(user_input) if self.verbose: - print(f"\nšŸ“‹ Execution Plan:") + print("\nšŸ“‹ Execution Plan:") for i, step in enumerate(plan["steps"], 1): print(f" {i}. {step['description']}") @@ -176,7 +176,10 @@ def _create_plan(self, user_input: str) -> dict[str, Any]: try: response = self.llm.generate( - system_prompt, user_prompt, json_mode=self.llm.supports_json_mode(), temperature=0.0 + system_prompt, + user_prompt, + json_mode=self.llm.supports_json_mode(), + temperature=0.0, ) # Parse JSON response @@ -262,7 +265,11 @@ def _execute_step(self, step: dict[str, Any]) -> dict[str, Any]: elif action == "WAIT": duration = params.get("duration", 2.0) time.sleep(duration) - return {"success": True, "action": action, "data": {"duration": duration}} + return { + "success": True, + "action": action, + "data": {"duration": duration}, + } elif action == "EXTRACT_INFO": info_type = params["info_type"] @@ -337,7 +344,11 @@ def _extract_information(self, snap: Snapshot, info_type: str) -> dict[str, Any] ) return json.loads(response.content) except: - return {"found": False, "data": {}, "summary": "Failed to extract information"} + return { + "found": False, + "data": {}, + "summary": "Failed to extract information", + } def _verify_condition(self, condition: str) -> bool: """ @@ -375,7 +386,10 @@ def _verify_condition(self, condition: str) -> bool: return False def _synthesize_response( - self, user_input: str, plan: dict[str, Any], execution_results: list[dict[str, Any]] + self, + user_input: str, + plan: dict[str, Any], + execution_results: list[dict[str, Any]], ) -> str: """ Synthesize a natural language response from execution results diff --git a/sentience/recorder.py b/sentience/recorder.py index 7f0a84f..44ef70f 100644 --- a/sentience/recorder.py +++ b/sentience/recorder.py @@ -4,11 +4,10 @@ import json from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any from .browser import SentienceBrowser from .models import Element, Snapshot -from .query import find from .snapshot import snapshot @@ -83,14 +82,22 @@ def add_click(self, element_id: int, selector: str | None = None) -> None: self.add_step(step) def add_type( - self, element_id: int, text: str, selector: str | None = None, mask: bool = False + self, + element_id: int, + text: str, + selector: str | None = None, + mask: bool = False, ) -> None: """Add type step""" ts = int((datetime.now() - self._start_time).total_seconds() * 1000) # Mask sensitive data if requested masked_text = "***" if mask else text step = TraceStep( - ts=ts, type="type", element_id=element_id, text=masked_text, selector=selector + ts=ts, + type="type", + element_id=element_id, + text=masked_text, + selector=selector, ) self.add_step(step) @@ -188,7 +195,7 @@ def _cleanup_listeners(self) -> None: """Clean up event listeners""" pass - def _infer_selector(self, element_id: int) -> str | None: + def _infer_selector(self, element_id: int) -> str | None: # noqa: C901 """ Infer a semantic selector for an element diff --git a/sentience/snapshot.py b/sentience/snapshot.py index cb71baa..c13d252 100644 --- a/sentience/snapshot.py +++ b/sentience/snapshot.py @@ -97,7 +97,8 @@ def _snapshot_via_extension( # may not be immediately available after page load try: browser.page.wait_for_function( - "typeof window.sentience !== 'undefined'", timeout=5000 # 5 second timeout + "typeof window.sentience !== 'undefined'", + timeout=5000, # 5 second timeout ) except Exception as e: # Gather diagnostics if wait fails diff --git a/sentience/tracer_factory.py b/sentience/tracer_factory.py index 06d1324..b1e8e31 100644 --- a/sentience/tracer_factory.py +++ b/sentience/tracer_factory.py @@ -4,20 +4,23 @@ Provides convenient factory function for creating tracers with cloud upload support. """ +import gzip +import os import uuid from pathlib import Path -from typing import Optional import requests from sentience.cloud_tracing import CloudTraceSink from sentience.tracing import JsonlTraceSink, Tracer +# Sentience API base URL (constant) +SENTIENCE_API_URL = "https://api.sentienceapi.com" + def create_tracer( api_key: str | None = None, run_id: str | None = None, - api_url: str = "https://api.sentienceapi.com", ) -> Tracer: """ Create tracer with automatic tier detection. @@ -31,7 +34,6 @@ def create_tracer( - Free tier: None or empty - Pro/Enterprise: Valid API key run_id: Unique identifier for this agent run. If not provided, generates UUID. - api_url: Sentience API base URL (default: https://api.sentienceapi.com) Returns: Tracer configured with appropriate sink @@ -53,12 +55,16 @@ def create_tracer( if run_id is None: run_id = str(uuid.uuid4()) + # 0. Check for orphaned traces from previous crashes (if api_key provided) + if api_key: + _recover_orphaned_traces(api_key, SENTIENCE_API_URL) + # 1. Try to initialize Cloud Sink (Pro/Enterprise tier) if api_key: try: # Request pre-signed upload URL from backend response = requests.post( - f"{api_url}/v1/traces/init", + f"{SENTIENCE_API_URL}/v1/traces/init", headers={"Authorization": f"Bearer {api_key}"}, json={"run_id": run_id}, timeout=10, @@ -72,7 +78,7 @@ def create_tracer( print("ā˜ļø [Sentience] Cloud tracing enabled (Pro tier)") return Tracer( run_id=run_id, - sink=CloudTraceSink(upload_url=upload_url), + sink=CloudTraceSink(upload_url=upload_url, run_id=run_id), ) else: print("āš ļø [Sentience] Cloud init response missing upload_url") @@ -103,3 +109,86 @@ def create_tracer( print(f"šŸ’¾ [Sentience] Local tracing: {local_path}") return Tracer(run_id=run_id, sink=JsonlTraceSink(str(local_path))) + + +def _recover_orphaned_traces(api_key: str, api_url: str = SENTIENCE_API_URL) -> None: + """ + Attempt to upload orphaned traces from previous crashed runs. + + Scans ~/.sentience/traces/pending/ for un-uploaded trace files and + attempts to upload them using the provided API key. + + Args: + api_key: Sentience API key for authentication + api_url: Sentience API base URL (defaults to SENTIENCE_API_URL) + """ + pending_dir = Path.home() / ".sentience" / "traces" / "pending" + + if not pending_dir.exists(): + return + + orphaned = list(pending_dir.glob("*.jsonl")) + + if not orphaned: + return + + print(f"āš ļø [Sentience] Found {len(orphaned)} un-uploaded trace(s) from previous runs") + print(" Attempting to upload now...") + + for trace_file in orphaned: + try: + # Extract run_id from filename (format: {run_id}.jsonl) + run_id = trace_file.stem + + # Request new upload URL for this run_id + response = requests.post( + f"{api_url}/v1/traces/init", + headers={"Authorization": f"Bearer {api_key}"}, + json={"run_id": run_id}, + timeout=10, + ) + + if response.status_code != 200: + print(f"āŒ Failed to get upload URL for {run_id}: HTTP {response.status_code}") + continue + + data = response.json() + upload_url = data.get("upload_url") + + if not upload_url: + print(f"āŒ Upload URL missing for {run_id}") + continue + + # Read and compress trace file + with open(trace_file, "rb") as f: + trace_data = f.read() + + compressed_data = gzip.compress(trace_data) + + # Upload to cloud + upload_response = requests.put( + upload_url, + data=compressed_data, + headers={ + "Content-Type": "application/x-gzip", + "Content-Encoding": "gzip", + }, + timeout=60, + ) + + if upload_response.status_code == 200: + print(f"āœ… Uploaded orphaned trace: {run_id}") + # Delete file on successful upload + try: + os.remove(trace_file) + except Exception: + pass # Ignore cleanup errors + else: + print(f"āŒ Failed to upload {run_id}: HTTP {upload_response.status_code}") + + except requests.exceptions.Timeout: + print(f"āŒ Timeout uploading {trace_file.name}") + except requests.exceptions.ConnectionError: + print(f"āŒ Connection error uploading {trace_file.name}") + except Exception as e: + print(f"āŒ Error uploading {trace_file.name}: {e}") diff --git a/tests/conftest.py b/tests/conftest.py index 07e179a..fd79df5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,8 @@ def pytest_configure(config): """Register custom markers""" config.addinivalue_line( - "markers", "requires_extension: mark test as requiring the sentience-chrome extension" + "markers", + "requires_extension: mark test as requiring the sentience-chrome extension", ) diff --git a/tests/test_actions.py b/tests/test_actions.py index 2b731cf..104a368 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -77,7 +77,12 @@ def test_click_rect_with_bbox(): if link: result = click_rect( browser, - {"x": link.bbox.x, "y": link.bbox.y, "w": link.bbox.width, "h": link.bbox.height}, + { + "x": link.bbox.x, + "y": link.bbox.y, + "w": link.bbox.width, + "h": link.bbox.height, + }, ) assert result.success is True assert result.duration_ms > 0 diff --git a/tests/test_agent.py b/tests/test_agent.py index cb68823..52182c7 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -293,7 +293,6 @@ def test_agent_act_full_cycle(): patch("sentience.agent.snapshot") as mock_snapshot, patch("sentience.agent.click") as mock_click, ): - from sentience.models import ActionResult mock_snapshot.return_value = create_mock_snapshot() @@ -392,7 +391,6 @@ def test_agent_retry_on_failure(): patch("sentience.agent.snapshot") as mock_snapshot, patch("sentience.agent.click") as mock_click, ): - mock_snapshot.return_value = create_mock_snapshot() # Simulate click failure mock_click.side_effect = RuntimeError("Element not found") @@ -417,7 +415,6 @@ def test_agent_action_parsing_variations(): patch("sentience.agent.type_text") as mock_type, patch("sentience.agent.press") as mock_press, ): - from sentience.models import ActionResult mock_result = ActionResult(success=True, duration_ms=100, outcome="dom_updated") diff --git a/tests/test_cloud_tracing.py b/tests/test_cloud_tracing.py index 1b41d57..9462115 100644 --- a/tests/test_cloud_tracing.py +++ b/tests/test_cloud_tracing.py @@ -2,7 +2,9 @@ import gzip import json +import os import tempfile +import time from pathlib import Path from unittest.mock import MagicMock, Mock, patch @@ -19,6 +21,7 @@ class TestCloudTraceSink: def test_cloud_trace_sink_upload_success(self): """Test CloudTraceSink successfully uploads trace to cloud.""" upload_url = "https://sentience.nyc3.digitaloceanspaces.com/user123/run456/trace.jsonl.gz" + run_id = "test-run-123" with patch("sentience.cloud_tracing.requests.put") as mock_put: # Mock successful response @@ -28,7 +31,7 @@ def test_cloud_trace_sink_upload_success(self): mock_put.return_value = mock_response # Create sink and emit events - sink = CloudTraceSink(upload_url) + sink = CloudTraceSink(upload_url, run_id=run_id) sink.emit({"v": 1, "type": "run_start", "seq": 1, "data": {"agent": "TestAgent"}}) sink.emit({"v": 1, "type": "run_end", "seq": 2, "data": {"steps": 1}}) @@ -57,9 +60,15 @@ def test_cloud_trace_sink_upload_success(self): assert event1["type"] == "run_start" assert event2["type"] == "run_end" + # Verify file was deleted on successful upload + cache_dir = Path.home() / ".sentience" / "traces" / "pending" + trace_path = cache_dir / f"{run_id}.jsonl" + assert not trace_path.exists(), "Trace file should be deleted after successful upload" + def test_cloud_trace_sink_upload_failure_preserves_trace(self, capsys): """Test CloudTraceSink preserves trace locally on upload failure.""" upload_url = "https://sentience.nyc3.digitaloceanspaces.com/user123/run456/trace.jsonl.gz" + run_id = "test-run-456" with patch("sentience.cloud_tracing.requests.put") as mock_put: # Mock failed response @@ -69,7 +78,7 @@ def test_cloud_trace_sink_upload_failure_preserves_trace(self, capsys): mock_put.return_value = mock_response # Create sink and emit events - sink = CloudTraceSink(upload_url) + sink = CloudTraceSink(upload_url, run_id=run_id) sink.emit({"v": 1, "type": "run_start", "seq": 1}) # Close triggers upload (which will fail) @@ -81,10 +90,19 @@ def test_cloud_trace_sink_upload_failure_preserves_trace(self, capsys): assert "Upload failed: HTTP 500" in captured.out assert "Local trace preserved" in captured.out + # Verify file was preserved on failure + cache_dir = Path.home() / ".sentience" / "traces" / "pending" + trace_path = cache_dir / f"{run_id}.jsonl" + assert trace_path.exists(), "Trace file should be preserved on upload failure" + + # Cleanup + if trace_path.exists(): + os.remove(trace_path) + def test_cloud_trace_sink_emit_after_close_raises(self): """Test CloudTraceSink raises error when emitting after close.""" upload_url = "https://test.com/upload" - sink = CloudTraceSink(upload_url) + sink = CloudTraceSink(upload_url, run_id="test-run-789") sink.close() with pytest.raises(RuntimeError, match="CloudTraceSink is closed"): @@ -96,7 +114,7 @@ def test_cloud_trace_sink_context_manager(self): mock_put.return_value = Mock(status_code=200) upload_url = "https://test.com/upload" - with CloudTraceSink(upload_url) as sink: + with CloudTraceSink(upload_url, run_id="test-run-context") as sink: sink.emit({"v": 1, "type": "test", "seq": 1}) # Verify upload was called @@ -105,12 +123,13 @@ def test_cloud_trace_sink_context_manager(self): def test_cloud_trace_sink_network_error_graceful_degradation(self, capsys): """Test CloudTraceSink handles network errors gracefully.""" upload_url = "https://sentience.nyc3.digitaloceanspaces.com/user123/run456/trace.jsonl.gz" + run_id = "test-run-network-error" with patch("sentience.cloud_tracing.requests.put") as mock_put: # Simulate network error mock_put.side_effect = Exception("Network error") - sink = CloudTraceSink(upload_url) + sink = CloudTraceSink(upload_url, run_id=run_id) sink.emit({"v": 1, "type": "test", "seq": 1}) # Should not raise, just print warning @@ -120,13 +139,22 @@ def test_cloud_trace_sink_network_error_graceful_degradation(self, capsys): assert "āŒ" in captured.out assert "Error uploading trace" in captured.out + # Verify file was preserved + cache_dir = Path.home() / ".sentience" / "traces" / "pending" + trace_path = cache_dir / f"{run_id}.jsonl" + assert trace_path.exists(), "Trace file should be preserved on network error" + + # Cleanup + if trace_path.exists(): + os.remove(trace_path) + def test_cloud_trace_sink_multiple_close_safe(self): """Test CloudTraceSink.close() is idempotent.""" with patch("sentience.cloud_tracing.requests.put") as mock_put: mock_put.return_value = Mock(status_code=200) upload_url = "https://test.com/upload" - sink = CloudTraceSink(upload_url) + sink = CloudTraceSink(upload_url, run_id="test-run-multiple-close") sink.emit({"v": 1, "type": "test", "seq": 1}) # Close multiple times @@ -137,6 +165,72 @@ def test_cloud_trace_sink_multiple_close_safe(self): # Upload should only be called once assert mock_put.call_count == 1 + def test_cloud_trace_sink_persistent_cache_directory(self): + """Test CloudTraceSink uses persistent cache directory instead of temp file.""" + upload_url = "https://test.com/upload" + run_id = "test-run-persistent" + + sink = CloudTraceSink(upload_url, run_id=run_id) + sink.emit({"v": 1, "type": "test", "seq": 1}) + + # Verify file is in persistent cache directory + cache_dir = Path.home() / ".sentience" / "traces" / "pending" + trace_path = cache_dir / f"{run_id}.jsonl" + assert trace_path.exists(), "Trace file should be in persistent cache directory" + assert cache_dir.exists(), "Cache directory should exist" + + # Cleanup + sink.close() + if trace_path.exists(): + os.remove(trace_path) + + def test_cloud_trace_sink_non_blocking_close(self): + """Test CloudTraceSink.close(blocking=False) returns immediately.""" + upload_url = "https://test.com/upload" + run_id = "test-run-nonblocking" + + with patch("sentience.cloud_tracing.requests.put") as mock_put: + mock_put.return_value = Mock(status_code=200) + + sink = CloudTraceSink(upload_url, run_id=run_id) + sink.emit({"v": 1, "type": "test", "seq": 1}) + + # Non-blocking close should return immediately + start_time = time.time() + sink.close(blocking=False) + elapsed = time.time() - start_time + + # Should return in < 0.1 seconds (much faster than upload) + assert elapsed < 0.1, "Non-blocking close should return immediately" + + # Wait a bit for background thread to complete + time.sleep(0.5) + + # Verify upload was called + assert mock_put.called + + def test_cloud_trace_sink_progress_callback(self): + """Test CloudTraceSink.close() with progress callback.""" + upload_url = "https://test.com/upload" + run_id = "test-run-progress" + progress_calls = [] + + def progress_callback(uploaded: int, total: int): + progress_calls.append((uploaded, total)) + + with patch("sentience.cloud_tracing.requests.put") as mock_put: + mock_put.return_value = Mock(status_code=200) + + sink = CloudTraceSink(upload_url, run_id=run_id) + sink.emit({"v": 1, "type": "test", "seq": 1}) + + sink.close(blocking=True, on_progress=progress_callback) + + # Verify progress callback was called + assert len(progress_calls) > 0, "Progress callback should be called" + # Last call should have uploaded == total + assert progress_calls[-1][0] == progress_calls[-1][1], "Final progress should be 100%" + class TestTracerFactory: """Test create_tracer factory function.""" @@ -165,6 +259,7 @@ def test_create_tracer_pro_tier_success(self, capsys): # Verify tracer works assert tracer.run_id == "test-run" assert isinstance(tracer.sink, CloudTraceSink) + assert tracer.sink.run_id == "test-run" # Verify run_id is passed # Cleanup tracer.close() @@ -263,9 +358,9 @@ def test_create_tracer_generates_run_id_if_not_provided(self): tracer.close() - def test_create_tracer_custom_api_url(self): - """Test create_tracer with custom API URL.""" - custom_url = "https://custom.api.com" + def test_create_tracer_uses_constant_api_url(self): + """Test create_tracer uses constant SENTIENCE_API_URL.""" + from sentience.tracer_factory import SENTIENCE_API_URL with patch("sentience.tracer_factory.requests.post") as mock_post: with patch("sentience.cloud_tracing.requests.put") as mock_put: @@ -276,12 +371,13 @@ def test_create_tracer_custom_api_url(self): mock_post.return_value = mock_response mock_put.return_value = Mock(status_code=200) - tracer = create_tracer(api_key="sk_test123", run_id="test-run", api_url=custom_url) + tracer = create_tracer(api_key="sk_test123", run_id="test-run") - # Verify correct API URL was used + # Verify correct API URL was used (constant) assert mock_post.called call_args = mock_post.call_args - assert call_args[0][0] == f"{custom_url}/v1/traces/init" + assert call_args[0][0] == f"{SENTIENCE_API_URL}/v1/traces/init" + assert SENTIENCE_API_URL == "https://api.sentienceapi.com" tracer.close() @@ -306,6 +402,66 @@ def test_create_tracer_missing_upload_url_in_response(self, capsys): tracer.close() + def test_create_tracer_orphaned_trace_recovery(self, capsys): + """Test create_tracer recovers and uploads orphaned traces from previous crashes.""" + import gzip + from pathlib import Path + + # Create orphaned trace file + cache_dir = Path.home() / ".sentience" / "traces" / "pending" + cache_dir.mkdir(parents=True, exist_ok=True) + orphaned_run_id = "orphaned-run-123" + orphaned_path = cache_dir / f"{orphaned_run_id}.jsonl" + + # Write test trace data + with open(orphaned_path, "w") as f: + f.write('{"v": 1, "type": "run_start", "seq": 1}\n') + + try: + with patch("sentience.tracer_factory.requests.post") as mock_post: + with patch("sentience.tracer_factory.requests.put") as mock_put: + # Mock API response for orphaned trace recovery + mock_recovery_response = Mock() + mock_recovery_response.status_code = 200 + mock_recovery_response.json.return_value = { + "upload_url": "https://storage.com/orphaned-upload" + } + + # Mock API response for new tracer creation + mock_new_response = Mock() + mock_new_response.status_code = 200 + mock_new_response.json.return_value = { + "upload_url": "https://storage.com/new-upload" + } + + # First call for orphaned recovery, second for new tracer + mock_post.side_effect = [mock_recovery_response, mock_new_response] + mock_put.return_value = Mock(status_code=200) + + # Create tracer - should trigger orphaned trace recovery + tracer = create_tracer(api_key="sk_test123", run_id="new-run-456") + + # Verify recovery messages + captured = capsys.readouterr() + assert "Found" in captured.out and "un-uploaded trace" in captured.out + assert "Uploaded orphaned trace" in captured.out or "Failed" in captured.out + + # Verify orphaned file was processed (either uploaded and deleted, or failed) + # If successful, file should be deleted + # If failed, file should still exist + # We check that recovery was attempted + assert mock_post.call_count >= 1, "Orphaned trace recovery should be attempted" + + # Verify new tracer was created + assert tracer.run_id == "new-run-456" + + tracer.close() + + finally: + # Cleanup orphaned file if it still exists + if orphaned_path.exists(): + os.remove(orphaned_path) + class TestRegressionTests: """Regression tests to ensure cloud tracing doesn't break existing functionality.""" diff --git a/tests/test_conversational_agent.py b/tests/test_conversational_agent.py index 3f1f585..29e8d20 100644 --- a/tests/test_conversational_agent.py +++ b/tests/test_conversational_agent.py @@ -217,7 +217,6 @@ def test_execute_find_and_click_step(): patch("sentience.agent.snapshot") as mock_snapshot, patch("sentience.agent.click") as mock_click, ): - from sentience.models import ActionResult mock_snapshot.return_value = create_mock_snapshot() @@ -247,7 +246,6 @@ def test_execute_find_and_type_step(): patch("sentience.agent.snapshot") as mock_snapshot, patch("sentience.agent.type_text") as mock_type, ): - from sentience.models import ActionResult mock_snapshot.return_value = create_mock_snapshot() @@ -342,7 +340,11 @@ def test_synthesize_response(): agent = ConversationalAgent(browser, llm, verbose=False) - plan = {"intent": "Search for magic mouse", "steps": [], "expected_outcome": "Success"} + plan = { + "intent": "Search for magic mouse", + "steps": [], + "expected_outcome": "Success", + } execution_results = [{"success": True, "action": "NAVIGATE"}] diff --git a/tests/test_stealth.py b/tests/test_stealth.py index 5675513..c1415d3 100644 --- a/tests/test_stealth.py +++ b/tests/test_stealth.py @@ -91,7 +91,11 @@ def test_stealth_features(): # noqa: C901 print("\n7. Testing against bot detection site...") try: # Navigate to a bot detection test site - page.goto("https://bot.sannysoft.com/", wait_until="domcontentloaded", timeout=10000) + page.goto( + "https://bot.sannysoft.com/", + wait_until="domcontentloaded", + timeout=10000, + ) page.wait_for_timeout(2000) # Wait for page to load # Check if we're detected From 74850d4b315c698e6c1b0ce423d6908ec73ad03f Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 22:15:54 -0800 Subject: [PATCH 4/7] bug fix --- sentience/cloud_tracing.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/sentience/cloud_tracing.py b/sentience/cloud_tracing.py index 7418d07..9d778af 100644 --- a/sentience/cloud_tracing.py +++ b/sentience/cloud_tracing.py @@ -133,23 +133,19 @@ def _do_upload(self, on_progress: Callable[[int, int], None] | None = None) -> N on_progress: Optional callback(uploaded_bytes, total_bytes) for progress updates """ try: - # Read file size for progress - file_size = os.path.getsize(self._path) - - if on_progress: - on_progress(0, file_size) - # Read and compress with open(self._path, "rb") as f: trace_data = f.read() compressed_data = gzip.compress(trace_data) + compressed_size = len(compressed_data) + # Report progress: start if on_progress: - on_progress(len(compressed_data), file_size) + on_progress(0, compressed_size) # Upload to DigitalOcean Spaces via pre-signed URL - print(f"šŸ“¤ [Sentience] Uploading trace to cloud ({len(compressed_data)} bytes)...") + print(f"šŸ“¤ [Sentience] Uploading trace to cloud ({compressed_size} bytes)...") response = requests.put( self.upload_url, @@ -164,6 +160,11 @@ def _do_upload(self, on_progress: Callable[[int, int], None] | None = None) -> N if response.status_code == 200: self._upload_successful = True print("āœ… [Sentience] Trace uploaded successfully") + + # Report progress: complete + if on_progress: + on_progress(compressed_size, compressed_size) + # Delete file only on successful upload if os.path.exists(self._path): try: From a13bd141053aeec21e62e4b6fc965dc10622becd Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 22:27:51 -0800 Subject: [PATCH 5/7] cloud trading example --- examples/cloud_tracing_agent.py | 111 ++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 examples/cloud_tracing_agent.py diff --git a/examples/cloud_tracing_agent.py b/examples/cloud_tracing_agent.py new file mode 100644 index 0000000..888ef45 --- /dev/null +++ b/examples/cloud_tracing_agent.py @@ -0,0 +1,111 @@ +""" +Example: Agent with Cloud Tracing + +Demonstrates how to use cloud tracing with SentienceAgent to upload traces +and screenshots to cloud storage for remote viewing and analysis. + +Requirements: +- Pro or Enterprise tier API key (SENTIENCE_API_KEY) +- OpenAI API key (OPENAI_API_KEY) for LLM + +Usage: + python examples/cloud_tracing_agent.py +""" + +import os + +from sentience import SentienceAgent, SentienceBrowser +from sentience.agent_config import AgentConfig +from sentience.llm_provider import OpenAIProvider +from sentience.tracer_factory import create_tracer + + +def main(): + # Get API keys from environment + sentience_key = os.environ.get("SENTIENCE_API_KEY") + openai_key = os.environ.get("OPENAI_API_KEY") + + if not sentience_key: + print("āŒ Error: SENTIENCE_API_KEY not set") + print(" Cloud tracing requires Pro or Enterprise tier") + print(" Get your API key at: https://sentience.studio") + return + + if not openai_key: + print("āŒ Error: OPENAI_API_KEY not set") + return + + print("šŸš€ Starting Agent with Cloud Tracing Demo\n") + + # 1. Create tracer with automatic tier detection + # If api_key is Pro/Enterprise, uses CloudTraceSink + # If api_key is missing/invalid, falls back to local JsonlTraceSink + run_id = "cloud-tracing-demo" + tracer = create_tracer(api_key=sentience_key, run_id=run_id) + + print(f"šŸ†” Run ID: {run_id}\n") + + # 2. Configure agent with screenshot capture + config = AgentConfig( + snapshot_limit=50, + capture_screenshots=True, # Enable screenshot capture + screenshot_format="jpeg", # JPEG for smaller file size + screenshot_quality=80, # 80% quality (good balance) + ) + + # 3. Create browser and LLM + browser = SentienceBrowser(api_key=sentience_key, headless=False) + llm = OpenAIProvider(api_key=openai_key, model="gpt-4o-mini") + + # 4. Create agent with tracer + agent = SentienceAgent(browser, llm, tracer=tracer, config=config) + + try: + # 5. Navigate and execute agent actions + print("šŸŒ Navigating to Google...\n") + browser.start() + browser.page.goto("https://www.google.com") + browser.page.wait_for_load_state("networkidle") + + # All actions are automatically traced! + print("šŸ“ Executing agent actions (all automatically traced)...\n") + agent.act("Click the search box") + agent.act("Type 'Sentience AI agent SDK' into the search field") + agent.act("Press Enter key") + + # Wait for results + import time + + time.sleep(2) + + agent.act("Click the first non-ad search result") + + print("\nāœ… Agent execution complete!") + + # 6. Get token usage stats + stats = agent.get_token_stats() + print("\nšŸ“Š Token Usage:") + print(f" Total tokens: {stats['total_tokens']}") + print(f" Prompt tokens: {stats['total_prompt_tokens']}") + print(f" Completion tokens: {stats['total_completion_tokens']}") + + except Exception as e: + print(f"\nāŒ Error during execution: {e}") + raise + + finally: + # 7. Close tracer (uploads to cloud) + print("\nšŸ“¤ Uploading trace to cloud...") + try: + tracer.close(blocking=True) # Wait for upload to complete + print("āœ… Trace uploaded successfully!") + print(f" View at: https://studio.sentienceapi.com (run_id: {run_id})") + except Exception as e: + print(f"āš ļø Upload failed: {e}") + print(f" Trace preserved locally at: ~/.sentience/traces/pending/{run_id}.jsonl") + + browser.close() + + +if __name__ == "__main__": + main() From c080f4b0241cc37df84ddd01a52adc3e5f9e1eb4 Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 23:11:42 -0800 Subject: [PATCH 6/7] restore api_url in tracer_factory --- .gitignore | 3 +++ sentience/tracer_factory.py | 9 +++++++-- tests/test_cloud_tracing.py | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 902a8d0..4aba1fa 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,9 @@ ENV/ htmlcov/ .tox/ +# Traces (runtime and test-generated) +traces/ + # Jupyter .ipynb_checkpoints diff --git a/sentience/tracer_factory.py b/sentience/tracer_factory.py index b1e8e31..04ffb1a 100644 --- a/sentience/tracer_factory.py +++ b/sentience/tracer_factory.py @@ -21,6 +21,7 @@ def create_tracer( api_key: str | None = None, run_id: str | None = None, + api_url: str | None = None, ) -> Tracer: """ Create tracer with automatic tier detection. @@ -34,6 +35,7 @@ def create_tracer( - Free tier: None or empty - Pro/Enterprise: Valid API key run_id: Unique identifier for this agent run. If not provided, generates UUID. + api_url: Sentience API base URL (default: https://api.sentienceapi.com) Returns: Tracer configured with appropriate sink @@ -55,16 +57,19 @@ def create_tracer( if run_id is None: run_id = str(uuid.uuid4()) + if api_url is None: + api_url = SENTIENCE_API_URL + # 0. Check for orphaned traces from previous crashes (if api_key provided) if api_key: - _recover_orphaned_traces(api_key, SENTIENCE_API_URL) + _recover_orphaned_traces(api_key, api_url) # 1. Try to initialize Cloud Sink (Pro/Enterprise tier) if api_key: try: # Request pre-signed upload URL from backend response = requests.post( - f"{SENTIENCE_API_URL}/v1/traces/init", + f"{api_url}/v1/traces/init", headers={"Authorization": f"Bearer {api_key}"}, json={"run_id": run_id}, timeout=10, diff --git a/tests/test_cloud_tracing.py b/tests/test_cloud_tracing.py index 9462115..23ee260 100644 --- a/tests/test_cloud_tracing.py +++ b/tests/test_cloud_tracing.py @@ -381,6 +381,30 @@ def test_create_tracer_uses_constant_api_url(self): tracer.close() + def test_create_tracer_custom_api_url(self): + """Test create_tracer accepts custom api_url parameter.""" + custom_api_url = "https://custom.api.example.com" + + with patch("sentience.tracer_factory.requests.post") as mock_post: + with patch("sentience.cloud_tracing.requests.put") as mock_put: + # Mock API response + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"upload_url": "https://storage.com/upload"} + mock_post.return_value = mock_response + mock_put.return_value = Mock(status_code=200) + + tracer = create_tracer( + api_key="sk_test123", run_id="test-run", api_url=custom_api_url + ) + + # Verify custom API URL was used + assert mock_post.called + call_args = mock_post.call_args + assert call_args[0][0] == f"{custom_api_url}/v1/traces/init" + + tracer.close() + def test_create_tracer_missing_upload_url_in_response(self, capsys): """Test create_tracer handles missing upload_url gracefully.""" with patch("sentience.tracer_factory.requests.post") as mock_post: From 0805ab90581dda040570e31305668e45f77551cc Mon Sep 17 00:00:00 2001 From: rcholic Date: Fri, 26 Dec 2025 23:23:39 -0800 Subject: [PATCH 7/7] fix cloud agent tracing bugs --- examples/cloud_tracing_agent.py | 6 ++-- sentience/agent.py | 57 ++++++++++++++++++++++++++------- sentience/tracing.py | 23 ++++++++++--- tests/test_agent.py | 37 ++++++++++++++++++++- 4 files changed, 104 insertions(+), 19 deletions(-) diff --git a/examples/cloud_tracing_agent.py b/examples/cloud_tracing_agent.py index 888ef45..5c9dc25 100644 --- a/examples/cloud_tracing_agent.py +++ b/examples/cloud_tracing_agent.py @@ -85,9 +85,9 @@ def main(): # 6. Get token usage stats stats = agent.get_token_stats() print("\nšŸ“Š Token Usage:") - print(f" Total tokens: {stats['total_tokens']}") - print(f" Prompt tokens: {stats['total_prompt_tokens']}") - print(f" Completion tokens: {stats['total_completion_tokens']}") + print(f" Total tokens: {stats.total_tokens}") + print(f" Prompt tokens: {stats.total_prompt_tokens}") + print(f" Completion tokens: {stats.total_completion_tokens}") except Exception as e: print(f"\nāŒ Error during execution: {e}") diff --git a/sentience/agent.py b/sentience/agent.py index a7dd305..17686b8 100644 --- a/sentience/agent.py +++ b/sentience/agent.py @@ -237,7 +237,7 @@ def act( # noqa: C901 self._track_tokens(goal, llm_response) # Parse action from LLM response - action_str = llm_response.content.strip() + action_str = self._extract_action_from_response(llm_response.content) # 4. EXECUTE: Parse and run action result_dict = self._execute_action(action_str, filtered_snap) @@ -395,6 +395,34 @@ def _build_context(self, snap: Snapshot, goal: str) -> str: return "\n".join(lines) + def _extract_action_from_response(self, response: str) -> str: + """ + Extract action command from LLM response, handling cases where + the LLM adds extra explanation despite instructions. + + Args: + response: Raw LLM response text + + Returns: + Cleaned action command string + """ + import re + + # Remove markdown code blocks if present + response = re.sub(r"```[\w]*\n?", "", response) + response = response.strip() + + # Try to find action patterns in the response + # Pattern matches: CLICK(123), TYPE(123, "text"), PRESS("key"), FINISH() + action_pattern = r'(CLICK\s*\(\s*\d+\s*\)|TYPE\s*\(\s*\d+\s*,\s*["\'].*?["\']\s*\)|PRESS\s*\(\s*["\'].*?["\']\s*\)|FINISH\s*\(\s*\))' + + match = re.search(action_pattern, response, re.IGNORECASE) + if match: + return match.group(1) + + # If no pattern match, return the original response (will likely fail parsing) + return response + def _query_llm(self, dom_context: str, goal: str) -> LLMResponse: """ Query LLM with standardized prompt template @@ -418,23 +446,30 @@ def _query_llm(self, dom_context: str, goal: str) -> LLMResponse: - {{CLICKABLE}}: Element is clickable - {{color:X}}: Background color name -RESPONSE FORMAT: -Return ONLY the function call, no explanation or markdown. - -Available actions: +CRITICAL RESPONSE FORMAT: +You MUST respond with ONLY ONE of these exact action formats: - CLICK(id) - Click element by ID - TYPE(id, "text") - Type text into element - PRESS("key") - Press keyboard key (Enter, Escape, Tab, ArrowDown, etc) - FINISH() - Task complete -Examples: -- CLICK(42) -- TYPE(15, "magic mouse") -- PRESS("Enter") -- FINISH() +DO NOT include any explanation, reasoning, or natural language. +DO NOT use markdown formatting or code blocks. +DO NOT say "The next step is..." or anything similar. + +CORRECT Examples: +CLICK(42) +TYPE(15, "magic mouse") +PRESS("Enter") +FINISH() + +INCORRECT Examples (DO NOT DO THIS): +"The next step is to click..." +"I will type..." +```CLICK(42)``` """ - user_prompt = "What is the next step to achieve the goal?" + user_prompt = "Return the single action command:" return self.llm.generate(system_prompt, user_prompt, temperature=0.0) diff --git a/sentience/tracing.py b/sentience/tracing.py index 64b37de..fbaf0ec 100644 --- a/sentience/tracing.py +++ b/sentience/tracing.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, Optional, Union +from typing import Any @dataclass @@ -243,9 +243,24 @@ def emit_error( } self.emit("error", data, step_id=step_id) - def close(self) -> None: - """Close the underlying sink.""" - self.sink.close() + def close(self, **kwargs) -> None: + """ + Close the underlying sink. + + Args: + **kwargs: Passed through to sink.close() (e.g., blocking=True for CloudTraceSink) + """ + # Check if sink.close() accepts kwargs (CloudTraceSink does, JsonlTraceSink doesn't) + import inspect + + sig = inspect.signature(self.sink.close) + if any( + p.kind in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.KEYWORD_ONLY) + for p in sig.parameters.values() + ): + self.sink.close(**kwargs) + else: + self.sink.close() def __enter__(self): """Context manager support.""" diff --git a/tests/test_agent.py b/tests/test_agent.py index 52182c7..259042a 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -3,7 +3,7 @@ Tests LLM providers and SentienceAgent without requiring browser """ -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import Mock, patch import pytest @@ -432,3 +432,38 @@ def test_agent_action_parsing_variations(): assert mock_click.call_count == 2 assert mock_type.call_count == 1 assert mock_press.call_count == 1 + + +def test_agent_extract_action_from_llm_response(): + """Test extraction of action commands from LLM responses with extra text""" + browser = create_mock_browser() + llm = MockLLMProvider() + agent = SentienceAgent(browser, llm, verbose=False) + + # Test clean action (should pass through) + assert agent._extract_action_from_response("CLICK(42)") == "CLICK(42)" + assert agent._extract_action_from_response('TYPE(15, "test")') == 'TYPE(15, "test")' + assert agent._extract_action_from_response('PRESS("Enter")') == 'PRESS("Enter")' + assert agent._extract_action_from_response("FINISH()") == "FINISH()" + + # Test with natural language prefix (the bug case) + assert ( + agent._extract_action_from_response("The next step is to click the button. CLICK(42)") + == "CLICK(42)" + ) + assert ( + agent._extract_action_from_response( + 'The next step is to type "Sentience AI agent SDK" into the search field. TYPE(15, "Sentience AI agent SDK")' + ) + == 'TYPE(15, "Sentience AI agent SDK")' + ) + + # Test with markdown code blocks + assert agent._extract_action_from_response("```\nCLICK(42)\n```") == "CLICK(42)" + assert ( + agent._extract_action_from_response('```python\nTYPE(15, "test")\n```') + == 'TYPE(15, "test")' + ) + + # Test with explanation after action + assert agent._extract_action_from_response("CLICK(42) to submit the form") == "CLICK(42)"