diff --git a/sentience/__init__.py b/sentience/__init__.py index 91ebe36..d3663df 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -77,6 +77,9 @@ ViewportRect, WaitResult, ) + +# Ordinal support (Phase 3) +from .ordinal import OrdinalIntent, boost_ordinal_elements, detect_ordinal_intent, select_by_ordinal from .overlay import clear_overlay, show_overlay from .query import find, query from .read import read @@ -242,4 +245,9 @@ "all_of", "any_of", "custom", + # Ordinal support (Phase 3) + "OrdinalIntent", + "detect_ordinal_intent", + "select_by_ordinal", + "boost_ordinal_elements", ] diff --git a/sentience/extension/background.js b/sentience/extension/background.js index 2923f55..10bff46 100644 --- a/sentience/extension/background.js +++ b/sentience/extension/background.js @@ -1,104 +1,240 @@ import init, { analyze_page_with_options, analyze_page, prune_for_api } from "../pkg/sentience_core.js"; -let wasmReady = !1, wasmInitPromise = null; +// background.js - Service Worker with WASM (CSP-Immune!) +// This runs in an isolated environment, completely immune to page CSP policies + +console.log('[Sentience Background] Initializing...'); + +// Global WASM initialization state +let wasmReady = false; +let wasmInitPromise = null; + +/** + * Initialize WASM module - called once on service worker startup + * Uses static imports (not dynamic import()) which is required for Service Workers + */ async function initWASM() { - if (!wasmReady) return wasmInitPromise || (wasmInitPromise = (async () => { - try { - globalThis.js_click_element = () => {}, await init(), wasmReady = !0; - } catch (error) { - throw error; - } - })(), wasmInitPromise); -} + if (wasmReady) return; + if (wasmInitPromise) return wasmInitPromise; -async function handleScreenshotCapture(_tabId, options = {}) { + wasmInitPromise = (async () => { try { - const {format: format = "png", quality: quality = 90} = options; - return await chrome.tabs.captureVisibleTab(null, { - format: format, - quality: quality - }); + console.log('[Sentience Background] Loading WASM module...'); + + // Define the js_click_element function that WASM expects + // In Service Workers, use 'globalThis' instead of 'window' + // In background context, we can't actually click, so we log a warning + globalThis.js_click_element = () => { + console.warn('[Sentience Background] js_click_element called in background (ignored)'); + }; + + // Initialize WASM - this calls the init() function from the static import + // The init() function handles fetching and instantiating the .wasm file + await init(); + + wasmReady = true; + console.log('[Sentience Background] ✓ WASM ready!'); + console.log( + '[Sentience Background] Available functions: analyze_page, analyze_page_with_options, prune_for_api' + ); } catch (error) { - throw new Error(`Failed to capture screenshot: ${error.message}`); + console.error('[Sentience Background] WASM initialization failed:', error); + throw error; } + })(); + + return wasmInitPromise; } +// Initialize WASM on service worker startup +initWASM().catch((err) => { + console.error('[Sentience Background] Failed to initialize WASM:', err); +}); + +/** + * Message handler for all extension communication + * Includes global error handling to prevent extension crashes + */ +chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + // Global error handler to prevent extension crashes + try { + // Handle screenshot requests (existing functionality) + if (request.action === 'captureScreenshot') { + handleScreenshotCapture(sender.tab.id, request.options) + .then((screenshot) => { + sendResponse({ success: true, screenshot }); + }) + .catch((error) => { + console.error('[Sentience Background] Screenshot capture failed:', error); + sendResponse({ + success: false, + error: error.message || 'Screenshot capture failed', + }); + }); + return true; // Async response + } + + // Handle WASM processing requests (NEW!) + if (request.action === 'processSnapshot') { + handleSnapshotProcessing(request.rawData, request.options) + .then((result) => { + sendResponse({ success: true, result }); + }) + .catch((error) => { + console.error('[Sentience Background] Snapshot processing failed:', error); + sendResponse({ + success: false, + error: error.message || 'Snapshot processing failed', + }); + }); + return true; // Async response + } + + // Unknown action + console.warn('[Sentience Background] Unknown action:', request.action); + sendResponse({ success: false, error: 'Unknown action' }); + return false; + } catch (error) { + // Catch any synchronous errors that might crash the extension + console.error('[Sentience Background] Fatal error in message handler:', error); + try { + sendResponse({ + success: false, + error: `Fatal error: ${error.message || 'Unknown error'}`, + }); + } catch (e) { + // If sendResponse already called, ignore + } + return false; + } +}); + +/** + * Handle screenshot capture (existing functionality) + */ +async function handleScreenshotCapture(_tabId, options = {}) { + try { + const { format = 'png', quality = 90 } = options; + + const dataUrl = await chrome.tabs.captureVisibleTab(null, { + format, + quality, + }); + + console.log( + `[Sentience Background] Screenshot captured: ${format}, size: ${dataUrl.length} bytes` + ); + return dataUrl; + } catch (error) { + console.error('[Sentience Background] Screenshot error:', error); + throw new Error(`Failed to capture screenshot: ${error.message}`); + } +} + +/** + * Handle snapshot processing with WASM (NEW!) + * This is where the magic happens - completely CSP-immune! + * Includes safeguards to prevent crashes and hangs. + * + * @param {Array} rawData - Raw element data from injected_api.js + * @param {Object} options - Snapshot options (limit, filter, etc.) + * @returns {Promise} Processed snapshot result + */ async function handleSnapshotProcessing(rawData, options = {}) { - const startTime = performance.now(); + const MAX_ELEMENTS = 10000; // Safety limit to prevent hangs + const startTime = performance.now(); + + try { + // Safety check: limit element count to prevent hangs + if (!Array.isArray(rawData)) { + throw new Error('rawData must be an array'); + } + + if (rawData.length > MAX_ELEMENTS) { + console.warn( + `[Sentience Background] ⚠️ Large dataset: ${rawData.length} elements. Limiting to ${MAX_ELEMENTS} to prevent hangs.` + ); + rawData = rawData.slice(0, MAX_ELEMENTS); + } + + // Ensure WASM is initialized + await initWASM(); + if (!wasmReady) { + throw new Error('WASM module not initialized'); + } + + console.log( + `[Sentience Background] Processing ${rawData.length} elements with options:`, + options + ); + + // Run WASM processing using the imported functions directly + // Wrap in try-catch with timeout protection + let analyzedElements; try { - if (!Array.isArray(rawData)) throw new Error("rawData must be an array"); - if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(), - !wasmReady) throw new Error("WASM module not initialized"); - let analyzedElements, prunedRawData; - try { - const wasmPromise = new Promise((resolve, reject) => { - try { - let result; - result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData), - resolve(result); - } catch (e) { - reject(e); - } - }); - analyzedElements = await Promise.race([ wasmPromise, new Promise((_, reject) => setTimeout(() => reject(new Error("WASM processing timeout (>18s)")), 18e3)) ]); - } catch (e) { - const errorMsg = e.message || "Unknown WASM error"; - throw new Error(`WASM analyze_page failed: ${errorMsg}`); - } + // Use a timeout wrapper to prevent infinite hangs + const wasmPromise = new Promise((resolve, reject) => { try { - prunedRawData = prune_for_api(rawData); + let result; + if (options.limit || options.filter) { + result = analyze_page_with_options(rawData, options); + } else { + result = analyze_page(rawData); + } + resolve(result); } catch (e) { - prunedRawData = rawData; + reject(e); } - performance.now(); - return { - elements: analyzedElements, - raw_elements: prunedRawData - }; - } catch (error) { - performance.now(); - throw error; + }); + + // Add timeout protection (18 seconds - less than content.js timeout) + analyzedElements = await Promise.race([ + wasmPromise, + new Promise((_, reject) => + setTimeout(() => reject(new Error('WASM processing timeout (>18s)')), 18000) + ), + ]); + } catch (e) { + const errorMsg = e.message || 'Unknown WASM error'; + console.error(`[Sentience Background] WASM analyze_page failed: ${errorMsg}`, e); + throw new Error(`WASM analyze_page failed: ${errorMsg}`); } -} -initWASM().catch(err => {}), chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { + // Prune elements for API (prevents 413 errors on large sites) + let prunedRawData; try { - return "captureScreenshot" === request.action ? (handleScreenshotCapture(sender.tab.id, request.options).then(screenshot => { - sendResponse({ - success: !0, - screenshot: screenshot - }); - }).catch(error => { - sendResponse({ - success: !1, - error: error.message || "Screenshot capture failed" - }); - }), !0) : "processSnapshot" === request.action ? (handleSnapshotProcessing(request.rawData, request.options).then(result => { - sendResponse({ - success: !0, - result: result - }); - }).catch(error => { - sendResponse({ - success: !1, - error: error.message || "Snapshot processing failed" - }); - }), !0) : (sendResponse({ - success: !1, - error: "Unknown action" - }), !1); - } catch (error) { - try { - sendResponse({ - success: !1, - error: `Fatal error: ${error.message || "Unknown error"}` - }); - } catch (e) {} - return !1; + prunedRawData = prune_for_api(rawData); + } catch (e) { + console.warn('[Sentience Background] prune_for_api failed, using original data:', e); + prunedRawData = rawData; } -}), self.addEventListener("error", event => { - event.preventDefault(); -}), self.addEventListener("unhandledrejection", event => { - event.preventDefault(); -}); \ No newline at end of file + + const duration = performance.now() - startTime; + console.log( + `[Sentience Background] ✓ Processed: ${analyzedElements.length} analyzed, ${prunedRawData.length} pruned (${duration.toFixed(1)}ms)` + ); + + return { + elements: analyzedElements, + raw_elements: prunedRawData, + }; + } catch (error) { + const duration = performance.now() - startTime; + console.error(`[Sentience Background] Processing error after ${duration.toFixed(1)}ms:`, error); + throw error; + } +} + +console.log('[Sentience Background] Service worker ready'); + +// Global error handlers to prevent extension crashes +self.addEventListener('error', (event) => { + console.error('[Sentience Background] Global error caught:', event.error); + event.preventDefault(); // Prevent extension crash +}); + +self.addEventListener('unhandledrejection', (event) => { + console.error('[Sentience Background] Unhandled promise rejection:', event.reason); + event.preventDefault(); // Prevent extension crash +}); diff --git a/sentience/extension/content.js b/sentience/extension/content.js index e94cde1..931ef5a 100644 --- a/sentience/extension/content.js +++ b/sentience/extension/content.js @@ -1,126 +1,329 @@ -!function() { - "use strict"; - window, window.top; - document.documentElement.dataset.sentienceExtensionId = chrome.runtime.id, window.addEventListener("message", event => { - var data; - if (event.source === window) switch (event.data.type) { - case "SENTIENCE_SCREENSHOT_REQUEST": - data = event.data, chrome.runtime.sendMessage({ - action: "captureScreenshot", - options: data.options - }, response => { - window.postMessage({ - type: "SENTIENCE_SCREENSHOT_RESULT", - requestId: data.requestId, - screenshot: response?.success ? response.screenshot : null, - error: response?.error - }, "*"); - }); - break; - - case "SENTIENCE_SNAPSHOT_REQUEST": - !function(data) { - const startTime = performance.now(); - let responded = !1; - const timeoutId = setTimeout(() => { - if (!responded) { - responded = !0; - const duration = performance.now() - startTime; - window.postMessage({ - type: "SENTIENCE_SNAPSHOT_RESULT", - requestId: data.requestId, - error: "WASM processing timeout - background script may be unresponsive", - duration: duration - }, "*"); - } - }, 2e4); - try { - chrome.runtime.sendMessage({ - action: "processSnapshot", - rawData: data.rawData, - options: data.options - }, response => { - if (responded) return; - responded = !0, clearTimeout(timeoutId); - const duration = performance.now() - startTime; - chrome.runtime.lastError ? window.postMessage({ - type: "SENTIENCE_SNAPSHOT_RESULT", - requestId: data.requestId, - error: `Chrome runtime error: ${chrome.runtime.lastError.message}`, - duration: duration - }, "*") : response?.success ? window.postMessage({ - type: "SENTIENCE_SNAPSHOT_RESULT", - requestId: data.requestId, - elements: response.result.elements, - raw_elements: response.result.raw_elements, - duration: duration - }, "*") : window.postMessage({ - type: "SENTIENCE_SNAPSHOT_RESULT", - requestId: data.requestId, - error: response?.error || "Processing failed", - duration: duration - }, "*"); - }); - } catch (error) { - if (!responded) { - responded = !0, clearTimeout(timeoutId); - const duration = performance.now() - startTime; - window.postMessage({ - type: "SENTIENCE_SNAPSHOT_RESULT", - requestId: data.requestId, - error: `Failed to send message: ${error.message}`, - duration: duration - }, "*"); - } - } - }(event.data); - break; - - case "SENTIENCE_SHOW_OVERLAY": - !function(data) { - const {elements: elements, targetElementId: targetElementId} = data; - if (!elements || !Array.isArray(elements)) return; - removeOverlay(); - const host = document.createElement("div"); - host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ", - document.body.appendChild(host); - const shadow = host.attachShadow({ - mode: "closed" - }), maxImportance = Math.max(...elements.map(e => e.importance || 0), 1); - elements.forEach(element => { - const bbox = element.bbox; - if (!bbox) return; - const isTarget = element.id === targetElementId, isPrimary = element.visual_cues?.is_primary || !1, importance = element.importance || 0; - let color; - color = isTarget ? "#FF0000" : isPrimary ? "#0066FF" : "#00FF00"; - const importanceRatio = maxImportance > 0 ? importance / maxImportance : .5, borderOpacity = isTarget ? 1 : isPrimary ? .9 : Math.max(.4, .5 + .5 * importanceRatio), fillOpacity = .2 * borderOpacity, borderWidth = isTarget ? 2 : isPrimary ? 1.5 : Math.max(.5, Math.round(2 * importanceRatio)), hexOpacity = Math.round(255 * fillOpacity).toString(16).padStart(2, "0"), box = document.createElement("div"); - if (box.style.cssText = `\n position: absolute;\n left: ${bbox.x}px;\n top: ${bbox.y}px;\n width: ${bbox.width}px;\n height: ${bbox.height}px;\n border: ${borderWidth}px solid ${color};\n background-color: ${color}${hexOpacity};\n box-sizing: border-box;\n opacity: ${borderOpacity};\n pointer-events: none;\n `, - importance > 0 || isPrimary) { - const badge = document.createElement("span"); - badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `, - box.appendChild(badge); - } - if (isTarget) { - const targetIndicator = document.createElement("span"); - targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ", - box.appendChild(targetIndicator); - } - shadow.appendChild(box); - }), overlayTimeout = setTimeout(() => { - removeOverlay(); - }, 5e3); - }(event.data); - break; - - case "SENTIENCE_CLEAR_OVERLAY": - removeOverlay(); +// Sentience Chrome Extension - Content Script +// Auto-generated from modular source +(function () { + 'use strict'; + + // content.js - ISOLATED WORLD (Bridge between Main World and Background) + console.log('[Sentience Bridge] Loaded.'); + + // Detect if we're in a child frame (for iframe support) + const isChildFrame = window !== window.top; + if (isChildFrame) { + console.log('[Sentience Bridge] Running in child frame:', window.location.href); + } + + // 1. Pass Extension ID to Main World (So API knows where to find resources) + document.documentElement.dataset.sentienceExtensionId = chrome.runtime.id; + + // 2. Message Router - Handles all communication between page and background + window.addEventListener('message', (event) => { + // Security check: only accept messages from same window + if (event.source !== window) return; + + // Route different message types + switch (event.data.type) { + case 'SENTIENCE_SCREENSHOT_REQUEST': + handleScreenshotRequest(event.data); + break; + + case 'SENTIENCE_SNAPSHOT_REQUEST': + handleSnapshotRequest(event.data); + break; + + case 'SENTIENCE_SHOW_OVERLAY': + handleShowOverlay(event.data); + break; + + case 'SENTIENCE_CLEAR_OVERLAY': + handleClearOverlay(); + break; + } + }); + + /** + * Handle screenshot requests (existing functionality) + */ + function handleScreenshotRequest(data) { + chrome.runtime.sendMessage({ action: 'captureScreenshot', options: data.options }, (response) => { + window.postMessage( + { + type: 'SENTIENCE_SCREENSHOT_RESULT', + requestId: data.requestId, + screenshot: response?.success ? response.screenshot : null, + error: response?.error, + }, + '*' + ); + }); + } + + /** + * Handle snapshot processing requests (NEW!) + * Sends raw DOM data to background worker for WASM processing + * Includes timeout protection to prevent extension crashes + */ + function handleSnapshotRequest(data) { + const startTime = performance.now(); + const TIMEOUT_MS = 20000; // 20 seconds (longer than injected_api timeout) + let responded = false; + + // Timeout protection: if background doesn't respond, send error + const timeoutId = setTimeout(() => { + if (!responded) { + responded = true; + const duration = performance.now() - startTime; + console.error(`[Sentience Bridge] ⚠️ WASM processing timeout after ${duration.toFixed(1)}ms`); + window.postMessage( + { + type: 'SENTIENCE_SNAPSHOT_RESULT', + requestId: data.requestId, + error: 'WASM processing timeout - background script may be unresponsive', + duration, + }, + '*' + ); + } + }, TIMEOUT_MS); + + try { + chrome.runtime.sendMessage( + { + action: 'processSnapshot', + rawData: data.rawData, + options: data.options, + }, + (response) => { + if (responded) return; // Already responded via timeout + responded = true; + clearTimeout(timeoutId); + + const duration = performance.now() - startTime; + + // Handle Chrome extension errors (e.g., background script crashed) + if (chrome.runtime.lastError) { + console.error( + '[Sentience Bridge] Chrome runtime error:', + chrome.runtime.lastError.message + ); + window.postMessage( + { + type: 'SENTIENCE_SNAPSHOT_RESULT', + requestId: data.requestId, + error: `Chrome runtime error: ${chrome.runtime.lastError.message}`, + duration, + }, + '*' + ); + return; + } + + if (response?.success) { + console.log(`[Sentience Bridge] ✓ WASM processing complete in ${duration.toFixed(1)}ms`); + window.postMessage( + { + type: 'SENTIENCE_SNAPSHOT_RESULT', + requestId: data.requestId, + elements: response.result.elements, + raw_elements: response.result.raw_elements, + duration, + }, + '*' + ); + } else { + console.error('[Sentience Bridge] WASM processing failed:', response?.error); + window.postMessage( + { + type: 'SENTIENCE_SNAPSHOT_RESULT', + requestId: data.requestId, + error: response?.error || 'Processing failed', + duration, + }, + '*' + ); + } } + ); + } catch (error) { + if (!responded) { + responded = true; + clearTimeout(timeoutId); + const duration = performance.now() - startTime; + console.error('[Sentience Bridge] Exception sending message:', error); + window.postMessage( + { + type: 'SENTIENCE_SNAPSHOT_RESULT', + requestId: data.requestId, + error: `Failed to send message: ${error.message}`, + duration, + }, + '*' + ); + } + } + } + + // ============================================================================ + // Visual Overlay - Shadow DOM Implementation + // ============================================================================ + + const OVERLAY_HOST_ID = 'sentience-overlay-host'; + let overlayTimeout = null; + + /** + * Show visual overlay highlighting elements using Shadow DOM + * @param {Object} data - Message data with elements and targetElementId + */ + function handleShowOverlay(data) { + const { elements, targetElementId } = data; + + if (!elements || !Array.isArray(elements)) { + console.warn('[Sentience Bridge] showOverlay: elements must be an array'); + return; + } + + removeOverlay(); + + // Create host with Shadow DOM for CSS isolation + const host = document.createElement('div'); + host.id = OVERLAY_HOST_ID; + host.style.cssText = ` + position: fixed !important; + top: 0 !important; + left: 0 !important; + width: 100vw !important; + height: 100vh !important; + pointer-events: none !important; + z-index: 2147483647 !important; + margin: 0 !important; + padding: 0 !important; + `; + document.body.appendChild(host); + + // Attach shadow root (closed mode for security and CSS isolation) + const shadow = host.attachShadow({ mode: 'closed' }); + + // Calculate max importance for scaling + const maxImportance = Math.max(...elements.map((e) => e.importance || 0), 1); + + elements.forEach((element) => { + const bbox = element.bbox; + if (!bbox) return; + + const isTarget = element.id === targetElementId; + const isPrimary = element.visual_cues?.is_primary || false; + const importance = element.importance || 0; + + // Color: Red (target), Blue (primary), Green (regular) + let color; + if (isTarget) color = '#FF0000'; + else if (isPrimary) color = '#0066FF'; + else color = '#00FF00'; + + // Scale opacity and border width based on importance + const importanceRatio = maxImportance > 0 ? importance / maxImportance : 0.5; + const borderOpacity = isTarget + ? 1.0 + : isPrimary + ? 0.9 + : Math.max(0.4, 0.5 + importanceRatio * 0.5); + const fillOpacity = borderOpacity * 0.2; + const borderWidth = isTarget + ? 2 + : isPrimary + ? 1.5 + : Math.max(0.5, Math.round(importanceRatio * 2)); + + // Convert fill opacity to hex for background-color + const hexOpacity = Math.round(fillOpacity * 255) + .toString(16) + .padStart(2, '0'); + + // Create box with semi-transparent fill + const box = document.createElement('div'); + box.style.cssText = ` + position: absolute; + left: ${bbox.x}px; + top: ${bbox.y}px; + width: ${bbox.width}px; + height: ${bbox.height}px; + border: ${borderWidth}px solid ${color}; + background-color: ${color}${hexOpacity}; + box-sizing: border-box; + opacity: ${borderOpacity}; + pointer-events: none; + `; + + // Add badge showing importance score + if (importance > 0 || isPrimary) { + const badge = document.createElement('span'); + badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`; + badge.style.cssText = ` + position: absolute; + top: -18px; + left: 0; + background: ${color}; + color: white; + font-size: 11px; + font-weight: bold; + padding: 2px 6px; + font-family: Arial, sans-serif; + border-radius: 3px; + opacity: 0.95; + white-space: nowrap; + pointer-events: none; + `; + box.appendChild(badge); + } + + // Add target emoji for target element + if (isTarget) { + const targetIndicator = document.createElement('span'); + targetIndicator.textContent = '🎯'; + targetIndicator.style.cssText = ` + position: absolute; + top: -18px; + right: 0; + font-size: 16px; + pointer-events: none; + `; + box.appendChild(targetIndicator); + } + + shadow.appendChild(box); }); - const OVERLAY_HOST_ID = "sentience-overlay-host"; - let overlayTimeout = null; - function removeOverlay() { - const existing = document.getElementById(OVERLAY_HOST_ID); - existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout), - overlayTimeout = null); + + console.log(`[Sentience Bridge] Overlay shown for ${elements.length} elements`); + + // Auto-remove after 5 seconds + overlayTimeout = setTimeout(() => { + removeOverlay(); + console.log('[Sentience Bridge] Overlay auto-cleared after 5 seconds'); + }, 5000); + } + + /** + * Clear overlay manually + */ + function handleClearOverlay() { + removeOverlay(); + console.log('[Sentience Bridge] Overlay cleared manually'); + } + + /** + * Remove overlay from DOM + */ + function removeOverlay() { + const existing = document.getElementById(OVERLAY_HOST_ID); + if (existing) { + existing.remove(); + } + + if (overlayTimeout) { + clearTimeout(overlayTimeout); + overlayTimeout = null; } -}(); \ No newline at end of file + } + + // console.log('[Sentience Bridge] Ready - Extension ID:', chrome.runtime.id); + +})(); diff --git a/sentience/extension/injected_api.js b/sentience/extension/injected_api.js index c62bcab..f334e0c 100644 --- a/sentience/extension/injected_api.js +++ b/sentience/extension/injected_api.js @@ -1,898 +1,2142 @@ -!function() { - "use strict"; - function getAllElements(root = document) { - const elements = [], filter = { - acceptNode: node => [ "SCRIPT", "STYLE", "NOSCRIPT", "META", "LINK", "HEAD" ].includes(node.tagName) || node.parentNode && "SVG" === node.parentNode.tagName && "SVG" !== node.tagName ? NodeFilter.FILTER_REJECT : NodeFilter.FILTER_ACCEPT - }, walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, filter); - for (;walker.nextNode(); ) { - const node = walker.currentNode; - node.isConnected && (elements.push(node), node.shadowRoot && elements.push(...getAllElements(node.shadowRoot))); +// Sentience Chrome Extension - Injected API +// Auto-generated from modular source +(function () { + 'use strict'; + + // utils.js - Helper Functions (CSP-Resistant) + // All utility functions needed for DOM data collection + + // --- HELPER: Deep Walker with Native Filter --- + function getAllElements(root = document) { + const elements = []; + const filter = { + acceptNode(node) { + // Skip metadata and script/style tags + if (['SCRIPT', 'STYLE', 'NOSCRIPT', 'META', 'LINK', 'HEAD'].includes(node.tagName)) { + return NodeFilter.FILTER_REJECT; } - return elements; - } - const DEFAULT_INFERENCE_CONFIG = { - allowedTags: [ "label", "span", "div" ], - allowedRoles: [], - allowedClassPatterns: [], - maxParentDepth: 2, - maxSiblingDistance: 1, - requireSameContainer: !0, - containerTags: [ "form", "fieldset", "div" ], - methods: { - explicitLabel: !0, - ariaLabelledby: !0, - parentTraversal: !0, - siblingProximity: !0 + // Skip deep SVG children + if (node.parentNode && node.parentNode.tagName === 'SVG' && node.tagName !== 'SVG') { + return NodeFilter.FILTER_REJECT; } + return NodeFilter.FILTER_ACCEPT; + }, }; - function isInferenceSource(el, config) { - if (!el || !el.tagName) return !1; - const tag = el.tagName.toLowerCase(), role = el.getAttribute ? el.getAttribute("role") : "", className = ((el.className || "") + "").toLowerCase(); - if (config.allowedTags.includes(tag)) return !0; - if (config.allowedRoles.length > 0 && role && config.allowedRoles.includes(role)) return !0; - if (config.allowedClassPatterns.length > 0) for (const pattern of config.allowedClassPatterns) if (className.includes(pattern.toLowerCase())) return !0; - return !1; - } - function isInSameValidContainer(element, candidate, limits) { - if (!element || !candidate) return !1; - if (limits.requireSameContainer) { - const commonParent = function(el1, el2) { - if (!el1 || !el2) return null; - const doc = "undefined" != typeof global && global.document || "undefined" != typeof window && window.document || "undefined" != typeof document && document || null, parents1 = []; - let current = el1; - for (;current && (parents1.push(current), current.parentElement) && (!doc || current !== doc.body && current !== doc.documentElement); ) current = current.parentElement; - for (current = el2; current; ) { - if (-1 !== parents1.indexOf(current)) return current; - if (!current.parentElement) break; - if (doc && (current === doc.body || current === doc.documentElement)) break; - current = current.parentElement; + + const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, filter); + while (walker.nextNode()) { + const node = walker.currentNode; + if (node.isConnected) { + elements.push(node); + if (node.shadowRoot) elements.push(...getAllElements(node.shadowRoot)); + } + } + return elements; + } + + // ============================================================================ + // LABEL INFERENCE SYSTEM + // ============================================================================ + + // Default inference configuration (conservative - Stage 1 equivalent) + const DEFAULT_INFERENCE_CONFIG = { + // Allowed tag names that can be inference sources + allowedTags: ['label', 'span', 'div'], + + // Allowed ARIA roles that can be inference sources + allowedRoles: [], + + // Class name patterns (substring match, case-insensitive) + allowedClassPatterns: [], + + // DOM tree traversal limits + maxParentDepth: 2, // Max 2 levels up DOM tree + maxSiblingDistance: 1, // Only immediate previous/next sibling + + // Container requirements (no distance-based checks) + requireSameContainer: true, // Must share common parent + containerTags: ['form', 'fieldset', 'div'], + + // Enable/disable specific inference methods + methods: { + explicitLabel: true, // el.labels API + ariaLabelledby: true, // aria-labelledby attribute + parentTraversal: true, // Check parent/grandparent + siblingProximity: true, // Check preceding sibling (same container only) + }, + }; + + // Merge user config with defaults + function mergeInferenceConfig(userConfig = {}) { + return { + ...DEFAULT_INFERENCE_CONFIG, + ...userConfig, + methods: { + ...DEFAULT_INFERENCE_CONFIG.methods, + ...(userConfig.methods || {}), + }, + }; + } + + // Check if element matches inference source criteria + function isInferenceSource(el, config) { + if (!el || !el.tagName) return false; + + const tag = el.tagName.toLowerCase(); + const role = el.getAttribute ? el.getAttribute('role') : ''; + const className = ((el.className || '') + '').toLowerCase(); + + // Check tag name + if (config.allowedTags.includes(tag)) { + return true; + } + + // Check role + if (config.allowedRoles.length > 0 && role && config.allowedRoles.includes(role)) { + return true; + } + + // Check class patterns + if (config.allowedClassPatterns.length > 0) { + for (const pattern of config.allowedClassPatterns) { + if (className.includes(pattern.toLowerCase())) { + return true; + } + } + } + + return false; + } + + // Helper: Find common parent element + function findCommonParent(el1, el2) { + if (!el1 || !el2) return null; + + // Get document reference safely for stopping conditions + // eslint-disable-next-line no-undef + const doc = + (typeof global !== 'undefined' && global.document) || + (typeof window !== 'undefined' && window.document) || + (typeof document !== 'undefined' && document) || + null; + + const parents1 = []; + let current = el1; + // Collect all parents (including el1 itself) + while (current) { + parents1.push(current); + // Stop if no parent + if (!current.parentElement) { + break; + } + // Stop at body or documentElement if they exist + if (doc && (current === doc.body || current === doc.documentElement)) { + break; + } + current = current.parentElement; + } + + // Check if el2 or any of its parents are in parents1 + current = el2; + while (current) { + // Use indexOf for more reliable comparison (handles object identity) + if (parents1.indexOf(current) !== -1) { + return current; + } + // Stop if no parent + if (!current.parentElement) { + break; + } + // Stop at body or documentElement if they exist + if (doc && (current === doc.body || current === doc.documentElement)) { + break; + } + current = current.parentElement; + } + + return null; + } + + // Helper: Check if element is a valid container + function isValidContainer(el, validTags) { + if (!el || !el.tagName) return false; + const tag = el.tagName.toLowerCase(); + // Handle both string and object className + let className = ''; + try { + className = (el.className || '') + ''; + } catch (e) { + className = ''; + } + return ( + validTags.includes(tag) || + className.toLowerCase().includes('form') || + className.toLowerCase().includes('field') + ); + } + + // Helper: Check container requirements (no distance-based checks) + function isInSameValidContainer(element, candidate, limits) { + if (!element || !candidate) return false; + + // Check same container requirement + if (limits.requireSameContainer) { + const commonParent = findCommonParent(element, candidate); + if (!commonParent) { + return false; + } + // Check if common parent is a valid container + if (!isValidContainer(commonParent, limits.containerTags)) { + return false; + } + } + + return true; + } + + // Main inference function + function getInferredLabel(el, options = {}) { + if (!el) return null; + + const { + enableInference = true, + inferenceConfig = {}, // User-provided config, merged with defaults + } = options; + + if (!enableInference) return null; + + // OPTIMIZATION: If element already has text or aria-label, skip inference entirely + // Check this BEFORE checking labels, so we don't infer if element already has text + // Note: For INPUT elements, we check value/placeholder, not innerText + // For IMG elements, we check alt, not innerText + // For other elements, innerText is considered explicit text + const ariaLabel = el.getAttribute ? el.getAttribute('aria-label') : null; + const hasAriaLabel = ariaLabel && ariaLabel.trim(); + const hasInputValue = el.tagName === 'INPUT' && (el.value || el.placeholder); + const hasImgAlt = el.tagName === 'IMG' && el.alt; + // For non-input/img elements, check innerText - but only if it's not empty + // Access innerText safely - it might be a getter or property + let innerTextValue = ''; + try { + innerTextValue = el.innerText || ''; + } catch (e) { + // If innerText access fails, treat as empty + innerTextValue = ''; + } + const hasInnerText = + el.tagName !== 'INPUT' && el.tagName !== 'IMG' && innerTextValue && innerTextValue.trim(); + + if (hasAriaLabel || hasInputValue || hasImgAlt || hasInnerText) { + return null; + } + + // Merge config with defaults + const config = mergeInferenceConfig(inferenceConfig); + + // Method 1: Explicit label association (el.labels API) + if (config.methods.explicitLabel && el.labels && el.labels.length > 0) { + const label = el.labels[0]; + if (isInferenceSource(label, config)) { + const text = (label.innerText || '').trim(); + if (text) { + return { + text, + source: 'explicit_label', + }; + } + } + } + + // Method 2: aria-labelledby (supports space-separated list of IDs) + // NOTE: aria-labelledby is an EXPLICIT reference, so it should work with ANY element + // regardless of inference source criteria. The config only controls whether this method runs. + if (config.methods.ariaLabelledby && el.hasAttribute && el.hasAttribute('aria-labelledby')) { + const labelIdsAttr = el.getAttribute('aria-labelledby'); + if (labelIdsAttr) { + // Split by whitespace to support multiple IDs (space-separated list) + const labelIds = labelIdsAttr.split(/\s+/).filter((id) => id.trim()); + const labelTexts = []; + + // Helper function to get document.getElementById from available contexts + const getDocument = () => { + // eslint-disable-next-line no-undef + if (typeof global !== 'undefined' && global.document) { + // eslint-disable-next-line no-undef + return global.document; + } + if (typeof window !== 'undefined' && window.document) { + return window.document; + } + if (typeof document !== 'undefined') { + return document; + } + return null; + }; + + const doc = getDocument(); + if (!doc || !doc.getElementById) ; else { + // Process each ID in the space-separated list + for (const labelId of labelIds) { + if (!labelId.trim()) continue; + + let labelEl = null; + try { + labelEl = doc.getElementById(labelId); + } catch (e) { + // If getElementById fails, skip this ID + continue; + } + + // aria-labelledby is an explicit reference - use ANY element, not just those matching inference criteria + // This follows ARIA spec: aria-labelledby can reference any element in the document + if (labelEl) { + // Extract text from the referenced element + let text = ''; + try { + // Try innerText first (preferred for visible text) + text = (labelEl.innerText || '').trim(); + // Fallback to textContent if innerText is empty + if (!text && labelEl.textContent) { + text = labelEl.textContent.trim(); } - return null; - }(element, candidate); - if (!commonParent) return !1; - if (!function(el, validTags) { - if (!el || !el.tagName) return !1; - const tag = el.tagName.toLowerCase(); - let className = ""; - try { - className = (el.className || "") + ""; - } catch (e) { - className = ""; + // Fallback to aria-label if available + if (!text && labelEl.getAttribute) { + const ariaLabel = labelEl.getAttribute('aria-label'); + if (ariaLabel) { + text = ariaLabel.trim(); + } } - return validTags.includes(tag) || className.toLowerCase().includes("form") || className.toLowerCase().includes("field"); - }(commonParent, limits.containerTags)) return !1; + } catch (e) { + // If text extraction fails, skip this element + continue; + } + + if (text) { + labelTexts.push(text); + } + } + } } - return !0; - } - function getInferredLabel(el, options = {}) { - if (!el) return null; - const {enableInference: enableInference = !0, inferenceConfig: inferenceConfig = {}} = options; - if (!enableInference) return null; - const ariaLabel = el.getAttribute ? el.getAttribute("aria-label") : null, hasAriaLabel = ariaLabel && ariaLabel.trim(), hasInputValue = "INPUT" === el.tagName && (el.value || el.placeholder), hasImgAlt = "IMG" === el.tagName && el.alt; - let innerTextValue = ""; - try { - innerTextValue = el.innerText || ""; - } catch (e) { - innerTextValue = ""; + + // Combine multiple label texts (space-separated) + if (labelTexts.length > 0) { + return { + text: labelTexts.join(' '), + source: 'aria_labelledby', + }; } - const hasInnerText = "INPUT" !== el.tagName && "IMG" !== el.tagName && innerTextValue && innerTextValue.trim(); - if (hasAriaLabel || hasInputValue || hasImgAlt || hasInnerText) return null; - const config = function(userConfig = {}) { + } + } + + // Method 3: Parent/grandparent traversal + if (config.methods.parentTraversal) { + let parent = el.parentElement; + let depth = 0; + while (parent && depth < config.maxParentDepth) { + if (isInferenceSource(parent, config)) { + const text = (parent.innerText || '').trim(); + if (text) { return { - ...DEFAULT_INFERENCE_CONFIG, - ...userConfig, - methods: { - ...DEFAULT_INFERENCE_CONFIG.methods, - ...userConfig.methods || {} - } + text, + source: 'parent_label', }; - }(inferenceConfig); - if (config.methods.explicitLabel && el.labels && el.labels.length > 0) { - const label = el.labels[0]; - if (isInferenceSource(label, config)) { - const text = (label.innerText || "").trim(); - if (text) return { - text: text, - source: "explicit_label" - }; - } + } } - if (config.methods.ariaLabelledby && el.hasAttribute && el.hasAttribute("aria-labelledby")) { - const labelIdsAttr = el.getAttribute("aria-labelledby"); - if (labelIdsAttr) { - const labelIds = labelIdsAttr.split(/\s+/).filter(id => id.trim()), labelTexts = [], doc = (() => "undefined" != typeof global && global.document ? global.document : "undefined" != typeof window && window.document ? window.document : "undefined" != typeof document ? document : null)(); - if (doc && doc.getElementById) for (const labelId of labelIds) { - if (!labelId.trim()) continue; - let labelEl = null; - try { - labelEl = doc.getElementById(labelId); - } catch (e) { - continue; - } - if (labelEl) { - let text = ""; - try { - if (text = (labelEl.innerText || "").trim(), !text && labelEl.textContent && (text = labelEl.textContent.trim()), - !text && labelEl.getAttribute) { - const ariaLabel = labelEl.getAttribute("aria-label"); - ariaLabel && (text = ariaLabel.trim()); - } - } catch (e) { - continue; - } - text && labelTexts.push(text); - } - } else ; - if (labelTexts.length > 0) return { - text: labelTexts.join(" "), - source: "aria_labelledby" - }; - } + parent = parent.parentElement; + depth++; + } + } + + // Method 4: Preceding sibling (no distance-based checks, only DOM structure) + if (config.methods.siblingProximity) { + const prev = el.previousElementSibling; + if (prev && isInferenceSource(prev, config)) { + // Only check if they're in the same valid container (no pixel distance) + if ( + isInSameValidContainer(el, prev, { + requireSameContainer: config.requireSameContainer, + containerTags: config.containerTags, + }) + ) { + const text = (prev.innerText || '').trim(); + if (text) { + return { + text, + source: 'sibling_label', + }; + } } - if (config.methods.parentTraversal) { - let parent = el.parentElement, depth = 0; - for (;parent && depth < config.maxParentDepth; ) { - if (isInferenceSource(parent, config)) { - const text = (parent.innerText || "").trim(); - if (text) return { - text: text, - source: "parent_label" - }; - } - parent = parent.parentElement, depth++; - } + } + } + + return null; + } + + // Helper: Check if element is interactable (should have role inferred) + function isInteractableElement(el) { + if (!el || !el.tagName) return false; + + const tag = el.tagName.toLowerCase(); + const role = el.getAttribute ? el.getAttribute('role') : null; + const hasTabIndex = el.hasAttribute ? el.hasAttribute('tabindex') : false; + const hasHref = el.tagName === 'A' && (el.hasAttribute ? el.hasAttribute('href') : false); + + // Native interactive elements + const interactiveTags = [ + 'button', + 'input', + 'textarea', + 'select', + 'option', + 'details', + 'summary', + 'a', + ]; + if (interactiveTags.includes(tag)) { + // For , only if it has href + if (tag === 'a' && !hasHref) return false; + return true; + } + + // Elements with explicit interactive roles + const interactiveRoles = [ + 'button', + 'link', + 'tab', + 'menuitem', + 'checkbox', + 'radio', + 'switch', + 'slider', + 'combobox', + 'textbox', + 'searchbox', + 'spinbutton', + ]; + if (role && interactiveRoles.includes(role.toLowerCase())) { + return true; + } + + // Focusable elements (tabindex makes them interactive) + if (hasTabIndex) { + return true; + } + + // Elements with event handlers (custom interactive elements) + if (el.onclick || el.onkeydown || el.onkeypress || el.onkeyup) { + return true; + } + + // Check for inline event handlers via attributes + if (el.getAttribute) { + const hasInlineHandler = + el.getAttribute('onclick') || + el.getAttribute('onkeydown') || + el.getAttribute('onkeypress') || + el.getAttribute('onkeyup'); + if (hasInlineHandler) { + return true; + } + } + + return false; + } + + // Helper: Infer ARIA role for interactable elements + function getInferredRole(el, options = {}) { + const { + enableInference = true, + // inferenceConfig reserved for future extensibility + } = options; + + if (!enableInference) return null; + + // Only infer roles for interactable elements + if (!isInteractableElement(el)) { + return null; + } + + // CRITICAL: Only infer if element has NO aria-label AND NO explicit role + const hasAriaLabel = el.getAttribute ? el.getAttribute('aria-label') : null; + const hasExplicitRole = el.getAttribute ? el.getAttribute('role') : null; + + if (hasAriaLabel || hasExplicitRole) { + return null; // Skip inference if element already has aria-label or role + } + + // If element is native semantic HTML, it already has a role + const tag = el.tagName.toLowerCase(); + const semanticTags = ['button', 'a', 'input', 'textarea', 'select', 'option']; + if (semanticTags.includes(tag)) { + return null; // Native HTML already has role + } + + // Infer role based on element behavior or context + // Check for click handlers first (most common) + if (el.onclick || (el.getAttribute && el.getAttribute('onclick'))) { + return 'button'; + } + + // Check for keyboard handlers + if ( + el.onkeydown || + el.onkeypress || + el.onkeyup || + (el.getAttribute && + (el.getAttribute('onkeydown') || el.getAttribute('onkeypress') || el.getAttribute('onkeyup'))) + ) { + return 'button'; // Default to button for keyboard-interactive elements + } + + // Focusable div/span likely needs a role + if (el.hasAttribute && el.hasAttribute('tabindex') && (tag === 'div' || tag === 'span')) { + return 'button'; // Default assumption for focusable elements + } + + return null; + } + + // --- HELPER: Smart Text Extractor --- + function getText(el) { + if (el.getAttribute('aria-label')) return el.getAttribute('aria-label'); + if (el.tagName === 'INPUT') return el.value || el.placeholder || ''; + if (el.tagName === 'IMG') return el.alt || ''; + return (el.innerText || '').replace(/\s+/g, ' ').trim().substring(0, 100); + } + + // Enhanced semantic text extractor with inference support + function getSemanticText(el, options = {}) { + if (!el) { + return { + text: '', + source: null, + }; + } + + // First check explicit aria-label (highest priority) + const explicitAriaLabel = el.getAttribute ? el.getAttribute('aria-label') : null; + if (explicitAriaLabel && explicitAriaLabel.trim()) { + return { + text: explicitAriaLabel.trim(), + source: 'explicit_aria_label', + }; + } + + // Check for existing text (visible text, input value, etc.) + // This matches the existing getText() logic + if (el.tagName === 'INPUT') { + const value = (el.value || el.placeholder || '').trim(); + if (value) { + return { + text: value, + source: 'input_value', + }; + } + } + + if (el.tagName === 'IMG') { + const alt = (el.alt || '').trim(); + if (alt) { + return { + text: alt, + source: 'img_alt', + }; + } + } + + const innerText = (el.innerText || '').trim(); + if (innerText) { + return { + text: innerText.substring(0, 100), // Match existing getText() limit + source: 'inner_text', + }; + } + + // Only try inference if we have NO explicit text/label + // Pass inferenceConfig from options to getInferredLabel + const inferred = getInferredLabel(el, { + enableInference: options.enableInference !== false, + inferenceConfig: options.inferenceConfig, // Pass config through + }); + if (inferred) { + return inferred; + } + + // Fallback: return empty with no source + return { + text: '', + source: null, + }; + } + + // --- HELPER: Safe Class Name Extractor (Handles SVGAnimatedString) --- + function getClassName(el) { + if (!el || !el.className) return ''; + + // Handle string (HTML elements) + if (typeof el.className === 'string') return el.className; + + // Handle SVGAnimatedString (SVG elements) + if (typeof el.className === 'object') { + if ('baseVal' in el.className && typeof el.className.baseVal === 'string') { + return el.className.baseVal; + } + if ('animVal' in el.className && typeof el.className.animVal === 'string') { + return el.className.animVal; + } + // Fallback: convert to string + try { + return String(el.className); + } catch (e) { + return ''; + } + } + + return ''; + } + + // --- HELPER: Paranoid String Converter (Handles SVGAnimatedString) --- + function toSafeString(value) { + if (value === null || value === undefined) return null; + + // 1. If it's already a primitive string, return it + if (typeof value === 'string') return value; + + // 2. Handle SVG objects (SVGAnimatedString, SVGAnimatedNumber, etc.) + if (typeof value === 'object') { + // Try extracting baseVal (standard SVG property) + if ('baseVal' in value && typeof value.baseVal === 'string') { + return value.baseVal; + } + // Try animVal as fallback + if ('animVal' in value && typeof value.animVal === 'string') { + return value.animVal; + } + // Fallback: Force to string (prevents WASM crash even if data is less useful) + // This prevents the "Invalid Type" crash, even if the data is "[object SVGAnimatedString]" + try { + return String(value); + } catch (e) { + return null; + } + } + + // 3. Last resort cast for primitives + try { + return String(value); + } catch (e) { + return null; + } + } + + // --- HELPER: Get SVG Fill/Stroke Color --- + // For SVG elements, get the fill or stroke color (SVGs use fill/stroke, not backgroundColor) + function getSVGColor(el) { + if (!el || el.tagName !== 'SVG') return null; + + const style = window.getComputedStyle(el); + + // Try fill first (most common for SVG icons) + const fill = style.fill; + if (fill && fill !== 'none' && fill !== 'transparent' && fill !== 'rgba(0, 0, 0, 0)') { + // Convert fill to rgb() format if needed + const rgbaMatch = fill.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); + if (rgbaMatch) { + const alpha = rgbaMatch[4] ? parseFloat(rgbaMatch[4]) : 1.0; + if (alpha >= 0.9) { + return `rgb(${rgbaMatch[1]}, ${rgbaMatch[2]}, ${rgbaMatch[3]})`; } - if (config.methods.siblingProximity) { - const prev = el.previousElementSibling; - if (prev && isInferenceSource(prev, config) && isInSameValidContainer(el, prev, { - requireSameContainer: config.requireSameContainer, - containerTags: config.containerTags - })) { - const text = (prev.innerText || "").trim(); - if (text) return { - text: text, - source: "sibling_label" - }; - } + } else if (fill.startsWith('rgb(')) { + return fill; + } + } + + // Fallback to stroke if fill is not available + const stroke = style.stroke; + if (stroke && stroke !== 'none' && stroke !== 'transparent' && stroke !== 'rgba(0, 0, 0, 0)') { + const rgbaMatch = stroke.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); + if (rgbaMatch) { + const alpha = rgbaMatch[4] ? parseFloat(rgbaMatch[4]) : 1.0; + if (alpha >= 0.9) { + return `rgb(${rgbaMatch[1]}, ${rgbaMatch[2]}, ${rgbaMatch[3]})`; } - return null; + } else if (stroke.startsWith('rgb(')) { + return stroke; + } } - function getText(el) { - return el.getAttribute("aria-label") ? el.getAttribute("aria-label") : "INPUT" === el.tagName ? el.value || el.placeholder || "" : "IMG" === el.tagName ? el.alt || "" : (el.innerText || "").replace(/\s+/g, " ").trim().substring(0, 100); + + return null; + } + + // --- HELPER: Get Effective Background Color --- + // Traverses up the DOM tree to find the nearest non-transparent background color + // For SVGs, also checks fill/stroke properties + // This handles rgba(0,0,0,0) and transparent values that browsers commonly return + function getEffectiveBackgroundColor(el) { + if (!el) return null; + + // For SVG elements, use fill/stroke instead of backgroundColor + if (el.tagName === 'SVG') { + const svgColor = getSVGColor(el); + if (svgColor) return svgColor; } - function getClassName(el) { - if (!el || !el.className) return ""; - if ("string" == typeof el.className) return el.className; - if ("object" == typeof el.className) { - if ("baseVal" in el.className && "string" == typeof el.className.baseVal) return el.className.baseVal; - if ("animVal" in el.className && "string" == typeof el.className.animVal) return el.className.animVal; - try { - return String(el.className); - } catch (e) { - return ""; - } + + let current = el; + const maxDepth = 10; // Prevent infinite loops + let depth = 0; + + while (current && depth < maxDepth) { + const style = window.getComputedStyle(current); + + // For SVG elements in the tree, also check fill/stroke + if (current.tagName === 'SVG') { + const svgColor = getSVGColor(current); + if (svgColor) return svgColor; + } + + const bgColor = style.backgroundColor; + + if (bgColor && bgColor !== 'transparent' && bgColor !== 'rgba(0, 0, 0, 0)') { + // Check if it's rgba with alpha < 1 (semi-transparent) + const rgbaMatch = bgColor.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); + if (rgbaMatch) { + const alpha = rgbaMatch[4] ? parseFloat(rgbaMatch[4]) : 1.0; + // If alpha is high enough (>= 0.9), consider it opaque enough + if (alpha >= 0.9) { + // Convert to rgb() format for Gateway compatibility + return `rgb(${rgbaMatch[1]}, ${rgbaMatch[2]}, ${rgbaMatch[3]})`; + } + // If semi-transparent, continue up the tree + } else if (bgColor.startsWith('rgb(')) { + // Already in rgb() format, use it + return bgColor; + } else { + // Named color or other format, return as-is + return bgColor; } - return ""; - } - function toSafeString(value) { - if (null == value) return null; - if ("string" == typeof value) return value; - if ("object" == typeof value) { - if ("baseVal" in value && "string" == typeof value.baseVal) return value.baseVal; - if ("animVal" in value && "string" == typeof value.animVal) return value.animVal; - try { - return String(value); - } catch (e) { - return null; - } + } + + // Move up the DOM tree + current = current.parentElement; + depth++; + } + + // Fallback: return null if nothing found + return null; + } + + // --- HELPER: Viewport Check --- + function isInViewport(rect) { + return ( + rect.top < window.innerHeight && + rect.bottom > 0 && + rect.left < window.innerWidth && + rect.right > 0 + ); + } + + // --- HELPER: Occlusion Check (Optimized to avoid layout thrashing) --- + // Only checks occlusion for elements likely to be occluded (high z-index, positioned) + // This avoids forced reflow for most elements, dramatically improving performance + function isOccluded(el, rect, style) { + // Fast path: Skip occlusion check for most elements + // Only check for elements that are likely to be occluded (overlays, modals, tooltips) + const zIndex = parseInt(style.zIndex, 10); + const position = style.position; + + // Skip occlusion check for normal flow elements (vast majority) + // Only check for positioned elements or high z-index (likely overlays) + if (position === 'static' && (isNaN(zIndex) || zIndex <= 10)) { + return false; // Assume not occluded for performance + } + + // For positioned/high z-index elements, do the expensive check + const cx = rect.x + rect.width / 2; + const cy = rect.y + rect.height / 2; + + if (cx < 0 || cx > window.innerWidth || cy < 0 || cy > window.innerHeight) return false; + + const topEl = document.elementFromPoint(cx, cy); + if (!topEl) return false; + + return !(el === topEl || el.contains(topEl) || topEl.contains(el)); + } + + // --- HELPER: Screenshot Bridge --- + function captureScreenshot(options) { + return new Promise((resolve) => { + const requestId = Math.random().toString(36).substring(7); + const listener = (e) => { + if (e.data.type === 'SENTIENCE_SCREENSHOT_RESULT' && e.data.requestId === requestId) { + window.removeEventListener('message', listener); + resolve(e.data.screenshot); + } + }; + window.addEventListener('message', listener); + window.postMessage({ type: 'SENTIENCE_SCREENSHOT_REQUEST', requestId, options }, '*'); + setTimeout(() => { + window.removeEventListener('message', listener); + resolve(null); + }, 10000); // 10s timeout + }); + } + + // --- HELPER: Snapshot Processing Bridge (NEW!) --- + function processSnapshotInBackground(rawData, options) { + return new Promise((resolve, reject) => { + const requestId = Math.random().toString(36).substring(7); + const TIMEOUT_MS = 25000; // 25 seconds (longer than content.js timeout) + let resolved = false; + + const timeout = setTimeout(() => { + if (!resolved) { + resolved = true; + window.removeEventListener('message', listener); + reject( + new Error( + 'WASM processing timeout - extension may be unresponsive. Try reloading the extension.' + ) + ); } + }, TIMEOUT_MS); + + const listener = (e) => { + if (e.data.type === 'SENTIENCE_SNAPSHOT_RESULT' && e.data.requestId === requestId) { + if (resolved) return; // Already handled + resolved = true; + clearTimeout(timeout); + window.removeEventListener('message', listener); + + if (e.data.error) { + reject(new Error(e.data.error)); + } else { + resolve({ + elements: e.data.elements, + raw_elements: e.data.raw_elements, + duration: e.data.duration, + }); + } + } + }; + + window.addEventListener('message', listener); + + try { + window.postMessage( + { + type: 'SENTIENCE_SNAPSHOT_REQUEST', + requestId, + rawData, + options, + }, + '*' + ); + } catch (error) { + if (!resolved) { + resolved = true; + clearTimeout(timeout); + window.removeEventListener('message', listener); + reject(new Error(`Failed to send snapshot request: ${error.message}`)); + } + } + }); + } + + // --- HELPER: Raw HTML Extractor (unchanged) --- + function getRawHTML(root) { + const sourceRoot = root || document.body; + const clone = sourceRoot.cloneNode(true); + + const unwantedTags = ['nav', 'footer', 'header', 'script', 'style', 'noscript', 'iframe', 'svg']; + unwantedTags.forEach((tag) => { + const elements = clone.querySelectorAll(tag); + elements.forEach((el) => { + if (el.parentNode) el.parentNode.removeChild(el); + }); + }); + + // Remove invisible elements + const invisibleSelectors = []; + const walker = document.createTreeWalker(sourceRoot, NodeFilter.SHOW_ELEMENT, null, false); + let node; + while ((node = walker.nextNode())) { + const tag = node.tagName.toLowerCase(); + if (tag === 'head' || tag === 'title') continue; + + const style = window.getComputedStyle(node); + if ( + style.display === 'none' || + style.visibility === 'hidden' || + (node.offsetWidth === 0 && node.offsetHeight === 0) + ) { + let selector = tag; + if (node.id) { + selector = `#${node.id}`; + } else if (node.className && typeof node.className === 'string') { + const classes = node.className + .trim() + .split(/\s+/) + .filter((c) => c); + if (classes.length > 0) { + selector = `${tag}.${classes.join('.')}`; + } + } + invisibleSelectors.push(selector); + } + } + + invisibleSelectors.forEach((selector) => { + try { + const elements = clone.querySelectorAll(selector); + elements.forEach((el) => { + if (el.parentNode) el.parentNode.removeChild(el); + }); + } catch (e) { + // Invalid selector, skip + } + }); + + // Resolve relative URLs + const links = clone.querySelectorAll('a[href]'); + links.forEach((link) => { + const href = link.getAttribute('href'); + if ( + href && + !href.startsWith('http://') && + !href.startsWith('https://') && + !href.startsWith('#') + ) { try { - return String(value); + link.setAttribute('href', new URL(href, document.baseURI).href); } catch (e) { - return null; + // Ignore invalid URLs } + } + }); + + const images = clone.querySelectorAll('img[src]'); + images.forEach((img) => { + const src = img.getAttribute('src'); + if ( + src && + !src.startsWith('http://') && + !src.startsWith('https://') && + !src.startsWith('data:') + ) { + try { + img.setAttribute('src', new URL(src, document.baseURI).href); + } catch (e) { + // Ignore invalid URLs + } + } + }); + + return clone.innerHTML; + } + + // --- HELPER: Markdown Converter (unchanged) --- + function convertToMarkdown(root) { + const rawHTML = getRawHTML(root); + const tempDiv = document.createElement('div'); + tempDiv.innerHTML = rawHTML; + + let markdown = ''; + let insideLink = false; + + function walk(node) { + if (node.nodeType === Node.TEXT_NODE) { + const text = node.textContent.replace(/[\r\n]+/g, ' ').replace(/\s+/g, ' '); + if (text.trim()) markdown += text; + return; + } + + if (node.nodeType !== Node.ELEMENT_NODE) return; + + const tag = node.tagName.toLowerCase(); + + // Prefix + if (tag === 'h1') markdown += '\n# '; + if (tag === 'h2') markdown += '\n## '; + if (tag === 'h3') markdown += '\n### '; + if (tag === 'li') markdown += '\n- '; + if (!insideLink && (tag === 'p' || tag === 'div' || tag === 'br')) markdown += '\n'; + if (tag === 'strong' || tag === 'b') markdown += '**'; + if (tag === 'em' || tag === 'i') markdown += '_'; + if (tag === 'a') { + markdown += '['; + insideLink = true; + } + + // Children + if (node.shadowRoot) { + Array.from(node.shadowRoot.childNodes).forEach(walk); + } else { + node.childNodes.forEach(walk); + } + + // Suffix + if (tag === 'a') { + const href = node.getAttribute('href'); + if (href) markdown += `](${href})`; + else markdown += ']'; + insideLink = false; + } + if (tag === 'strong' || tag === 'b') markdown += '**'; + if (tag === 'em' || tag === 'i') markdown += '_'; + if ( + !insideLink && + (tag === 'h1' || tag === 'h2' || tag === 'h3' || tag === 'p' || tag === 'div') + ) + markdown += '\n'; } - function getSVGColor(el) { - if (!el || "SVG" !== el.tagName) return null; - const style = window.getComputedStyle(el), fill = style.fill; - if (fill && "none" !== fill && "transparent" !== fill && "rgba(0, 0, 0, 0)" !== fill) { - const rgbaMatch = fill.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); - if (rgbaMatch) { - if ((rgbaMatch[4] ? parseFloat(rgbaMatch[4]) : 1) >= .9) return `rgb(${rgbaMatch[1]}, ${rgbaMatch[2]}, ${rgbaMatch[3]})`; - } else if (fill.startsWith("rgb(")) return fill; + + walk(tempDiv); + return markdown.replace(/\n{3,}/g, '\n\n').trim(); + } + + // --- HELPER: Text Extractor (unchanged) --- + function convertToText(root) { + let text = ''; + function walk(node) { + if (node.nodeType === Node.TEXT_NODE) { + text += node.textContent; + return; + } + if (node.nodeType === Node.ELEMENT_NODE) { + const tag = node.tagName.toLowerCase(); + if (['nav', 'footer', 'header', 'script', 'style', 'noscript', 'iframe', 'svg'].includes(tag)) + return; + + const style = window.getComputedStyle(node); + if (style.display === 'none' || style.visibility === 'hidden') return; + + const isBlock = + style.display === 'block' || + style.display === 'flex' || + node.tagName === 'P' || + node.tagName === 'DIV'; + if (isBlock) text += ' '; + + if (node.shadowRoot) { + Array.from(node.shadowRoot.childNodes).forEach(walk); + } else { + node.childNodes.forEach(walk); } - const stroke = style.stroke; - if (stroke && "none" !== stroke && "transparent" !== stroke && "rgba(0, 0, 0, 0)" !== stroke) { - const rgbaMatch = stroke.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); - if (rgbaMatch) { - if ((rgbaMatch[4] ? parseFloat(rgbaMatch[4]) : 1) >= .9) return `rgb(${rgbaMatch[1]}, ${rgbaMatch[2]}, ${rgbaMatch[3]})`; - } else if (stroke.startsWith("rgb(")) return stroke; + + if (isBlock) text += '\n'; + } + } + walk(root || document.body); + return text.replace(/\n{3,}/g, '\n\n').trim(); + } + + // --- HELPER: Clean null/undefined fields --- + function cleanElement(obj) { + if (Array.isArray(obj)) { + return obj.map(cleanElement); + } + if (obj !== null && typeof obj === 'object') { + const cleaned = {}; + for (const [key, value] of Object.entries(obj)) { + if (value !== null && value !== undefined) { + if (typeof value === 'object') { + const deepClean = cleanElement(value); + if (Object.keys(deepClean).length > 0) { + cleaned[key] = deepClean; + } + } else { + cleaned[key] = value; + } } - return null; + } + return cleaned; + } + return obj; + } + + // --- HELPER: Extract Raw Element Data (for Golden Set) --- + function extractRawElementData(el) { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + + return { + tag: el.tagName, + rect: { + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height), + }, + styles: { + cursor: style.cursor || null, + backgroundColor: style.backgroundColor || null, + color: style.color || null, + fontWeight: style.fontWeight || null, + fontSize: style.fontSize || null, + display: style.display || null, + position: style.position || null, + zIndex: style.zIndex || null, + opacity: style.opacity || null, + visibility: style.visibility || null, + }, + attributes: { + role: el.getAttribute('role') || null, + type: el.getAttribute('type') || null, + ariaLabel: el.getAttribute('aria-label') || null, + id: el.id || null, + className: el.className || null, + }, + }; + } + + // --- HELPER: Generate Unique CSS Selector (for Golden Set) --- + function getUniqueSelector(el) { + if (!el || !el.tagName) return ''; + + // If element has a unique ID, use it + if (el.id) { + return `#${el.id}`; } - function getRawHTML(root) { - const sourceRoot = root || document.body, clone = sourceRoot.cloneNode(!0); - [ "nav", "footer", "header", "script", "style", "noscript", "iframe", "svg" ].forEach(tag => { - clone.querySelectorAll(tag).forEach(el => { - el.parentNode && el.parentNode.removeChild(el); + + // Try data attributes or aria-label for uniqueness + for (const attr of el.attributes) { + if (attr.name.startsWith('data-') || attr.name === 'aria-label') { + const value = attr.value ? attr.value.replace(/"/g, '\\"') : ''; + return `${el.tagName.toLowerCase()}[${attr.name}="${value}"]`; + } + } + + // Build path with classes and nth-child for uniqueness + const path = []; + let current = el; + + while (current && current !== document.body && current !== document.documentElement) { + let selector = current.tagName.toLowerCase(); + + // If current element has ID, use it and stop + if (current.id) { + selector = `#${current.id}`; + path.unshift(selector); + break; + } + + // Add class if available + if (current.className && typeof current.className === 'string') { + const classes = current.className + .trim() + .split(/\s+/) + .filter((c) => c); + if (classes.length > 0) { + // Use first class for simplicity + selector += `.${classes[0]}`; + } + } + + // Add nth-of-type if needed for uniqueness + if (current.parentElement) { + const siblings = Array.from(current.parentElement.children); + const sameTagSiblings = siblings.filter((s) => s.tagName === current.tagName); + const index = sameTagSiblings.indexOf(current); + if (index > 0 || sameTagSiblings.length > 1) { + selector += `:nth-of-type(${index + 1})`; + } + } + + path.unshift(selector); + current = current.parentElement; + } + + return path.join(' > ') || el.tagName.toLowerCase(); + } + + // --- HELPER: Wait for DOM Stability (SPA Hydration) --- + // Waits for the DOM to stabilize before taking a snapshot + // Useful for React/Vue apps that render empty skeletons before hydration + async function waitForStability(options = {}) { + const { + minNodeCount = 500, + quietPeriod = 200, // milliseconds + maxWait = 5000, // maximum wait time + } = options; + + const startTime = Date.now(); + + return new Promise((resolve) => { + // Check if DOM already has enough nodes + const nodeCount = document.querySelectorAll('*').length; + if (nodeCount >= minNodeCount) { + // DOM seems ready, but wait for quiet period to ensure stability + let lastChange = Date.now(); + const observer = new MutationObserver(() => { + lastChange = Date.now(); + }); + + observer.observe(document.body, { + childList: true, + subtree: true, + attributes: false, + }); + + const checkStable = () => { + const timeSinceLastChange = Date.now() - lastChange; + const totalWait = Date.now() - startTime; + + if (timeSinceLastChange >= quietPeriod) { + observer.disconnect(); + resolve(); + } else if (totalWait >= maxWait) { + observer.disconnect(); + console.warn('[SentienceAPI] DOM stability timeout - proceeding anyway'); + resolve(); + } else { + setTimeout(checkStable, 50); + } + }; + + checkStable(); + } else { + // DOM doesn't have enough nodes yet, wait for them + const observer = new MutationObserver(() => { + const currentCount = document.querySelectorAll('*').length; + const totalWait = Date.now() - startTime; + + if (currentCount >= minNodeCount) { + observer.disconnect(); + // Now wait for quiet period + let lastChange = Date.now(); + const quietObserver = new MutationObserver(() => { + lastChange = Date.now(); + }); + + quietObserver.observe(document.body, { + childList: true, + subtree: true, + attributes: false, }); + + const checkQuiet = () => { + const timeSinceLastChange = Date.now() - lastChange; + const totalWait = Date.now() - startTime; + + if (timeSinceLastChange >= quietPeriod) { + quietObserver.disconnect(); + resolve(); + } else if (totalWait >= maxWait) { + quietObserver.disconnect(); + console.warn('[SentienceAPI] DOM stability timeout - proceeding anyway'); + resolve(); + } else { + setTimeout(checkQuiet, 50); + } + }; + + checkQuiet(); + } else if (totalWait >= maxWait) { + observer.disconnect(); + console.warn('[SentienceAPI] DOM node count timeout - proceeding anyway'); + resolve(); + } }); - const invisibleSelectors = [], walker = document.createTreeWalker(sourceRoot, NodeFilter.SHOW_ELEMENT, null, !1); - let node; - for (;node = walker.nextNode(); ) { - const tag = node.tagName.toLowerCase(); - if ("head" === tag || "title" === tag) continue; - const style = window.getComputedStyle(node); - if ("none" === style.display || "hidden" === style.visibility || 0 === node.offsetWidth && 0 === node.offsetHeight) { - let selector = tag; - if (node.id) selector = `#${node.id}`; else if (node.className && "string" == typeof node.className) { - const classes = node.className.trim().split(/\s+/).filter(c => c); - classes.length > 0 && (selector = `${tag}.${classes.join(".")}`); - } - invisibleSelectors.push(selector); + + observer.observe(document.body, { + childList: true, + subtree: true, + attributes: false, + }); + + // Timeout fallback + setTimeout(() => { + observer.disconnect(); + console.warn('[SentienceAPI] DOM stability max wait reached - proceeding'); + resolve(); + }, maxWait); + } + }); + } + + // --- HELPER: Collect Iframe Snapshots (Frame Stitching) --- + // Recursively collects snapshot data from all child iframes + // This enables detection of elements inside iframes (e.g., Stripe forms) + // + // NOTE: Cross-origin iframes cannot be accessed due to browser security (Same-Origin Policy). + // Only same-origin iframes will return snapshot data. Cross-origin iframes will be skipped + // with a warning. For cross-origin iframes, users must manually switch frames using + // Playwright's page.frame() API. + async function collectIframeSnapshots(options = {}) { + const iframeData = new Map(); // Map of iframe element -> snapshot data + + // Find all iframe elements in current document + const iframes = Array.from(document.querySelectorAll('iframe')); + + if (iframes.length === 0) { + return iframeData; + } + + console.log(`[SentienceAPI] Found ${iframes.length} iframe(s), requesting snapshots...`); + // Request snapshot from each iframe + const iframePromises = iframes.map((iframe, idx) => { + // OPTIMIZATION: Skip common ad domains to save time + const src = iframe.src || ''; + if ( + src.includes('doubleclick') || + src.includes('googleadservices') || + src.includes('ads system') + ) { + console.log(`[SentienceAPI] Skipping ad iframe: ${src.substring(0, 30)}...`); + return Promise.resolve(null); + } + + return new Promise((resolve) => { + const requestId = `iframe-${idx}-${Date.now()}`; + + // 1. EXTENDED TIMEOUT (Handle slow children) + const timeout = setTimeout(() => { + console.warn(`[SentienceAPI] ⚠️ Iframe ${idx} snapshot TIMEOUT (id: ${requestId})`); + resolve(null); + }, 5000); // Increased to 5s to handle slow processing + + // 2. ROBUST LISTENER with debugging + const listener = (event) => { + // Debug: Log all SENTIENCE_IFRAME_SNAPSHOT_RESPONSE messages to see what's happening + if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE') { + // Only log if it's not our request (for debugging) + if (event.data?.requestId !== requestId) ; + } + + // Check if this is the response we're waiting for + if ( + event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE' && + event.data?.requestId === requestId + ) { + clearTimeout(timeout); + window.removeEventListener('message', listener); + + if (event.data.error) { + console.warn(`[SentienceAPI] Iframe ${idx} returned error:`, event.data.error); + resolve(null); + } else { + const elementCount = event.data.snapshot?.raw_elements?.length || 0; + console.log( + `[SentienceAPI] ✓ Received ${elementCount} elements from Iframe ${idx} (id: ${requestId})` + ); + resolve({ + iframe, + data: event.data.snapshot, + error: null, + }); } + } + }; + + window.addEventListener('message', listener); + + // 3. SEND REQUEST with error handling + try { + if (iframe.contentWindow) { + // console.log(`[SentienceAPI] Sending request to Iframe ${idx} (id: ${requestId})`); + iframe.contentWindow.postMessage( + { + type: 'SENTIENCE_IFRAME_SNAPSHOT_REQUEST', + requestId, + options: { + ...options, + collectIframes: true, // Enable recursion for nested iframes + }, + }, + '*' + ); // Use '*' for cross-origin, but browser will enforce same-origin policy + } else { + console.warn( + `[SentienceAPI] Iframe ${idx} contentWindow is inaccessible (Cross-Origin?)` + ); + clearTimeout(timeout); + window.removeEventListener('message', listener); + resolve(null); + } + } catch (error) { + console.error(`[SentienceAPI] Failed to postMessage to Iframe ${idx}:`, error); + clearTimeout(timeout); + window.removeEventListener('message', listener); + resolve(null); } - invisibleSelectors.forEach(selector => { - try { - clone.querySelectorAll(selector).forEach(el => { - el.parentNode && el.parentNode.removeChild(el); - }); - } catch (e) {} + }); + }); + + // Wait for all iframe responses + const results = await Promise.all(iframePromises); + + // Store iframe data + results.forEach((result, idx) => { + if (result && result.data && !result.error) { + iframeData.set(iframes[idx], result.data); + console.log(`[SentienceAPI] ✓ Collected snapshot from iframe ${idx}`); + } else if (result && result.error) { + console.warn(`[SentienceAPI] Iframe ${idx} snapshot error:`, result.error); + } else if (!result) { + console.warn(`[SentienceAPI] Iframe ${idx} returned no data (timeout or error)`); + } + }); + + return iframeData; + } + + // --- HELPER: Handle Iframe Snapshot Request (for child frames) --- + // When a parent frame requests snapshot, this handler responds with local snapshot + // NOTE: Recursion is safe because querySelectorAll('iframe') only finds direct children. + // Iframe A can ask Iframe B, but won't go back up to parent (no circular dependency risk). + function setupIframeSnapshotHandler() { + window.addEventListener('message', async (event) => { + // Security: only respond to snapshot requests from parent frames + if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_REQUEST') { + const { requestId, options } = event.data; + + try { + // Generate snapshot for this iframe's content + // Allow recursive collection - querySelectorAll('iframe') only finds direct children, + // so Iframe A will ask Iframe B, but won't go back up to parent (safe recursion) + // waitForStability: false makes performance better - i.e. don't wait for children frames + const snapshotOptions = { + ...options, + collectIframes: true, + waitForStability: options.waitForStability === false ? false : false, + }; + const snapshot = await window.sentience.snapshot(snapshotOptions); + + // Send response back to parent + if (event.source && event.source.postMessage) { + event.source.postMessage( + { + type: 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE', + requestId, + snapshot, + error: null, + }, + '*' + ); + } + } catch (error) { + // Send error response + if (event.source && event.source.postMessage) { + event.source.postMessage( + { + type: 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE', + requestId, + snapshot: null, + error: error.message, + }, + '*' + ); + } + } + } + }); + } + + // snapshot.js - Snapshot Method (Main DOM Collection Logic) + + // 1. Geometry snapshot (NEW ARCHITECTURE - No WASM in Main World!) + async function snapshot(options = {}) { + try { + // Step 0: Wait for DOM stability if requested (for SPA hydration) + if (options.waitForStability !== false) { + await waitForStability(options.waitForStability || {}); + } + + // Step 1: Collect raw DOM data (Main World - CSP can't block this!) + const rawData = []; + window.sentience_registry = []; + + const nodes = getAllElements(); + + nodes.forEach((el, idx) => { + if (!el.getBoundingClientRect) return; + const rect = el.getBoundingClientRect(); + if (rect.width < 5 || rect.height < 5) return; + + window.sentience_registry[idx] = el; + + // Use getSemanticText for inference support (falls back to getText if no inference) + const semanticText = getSemanticText(el, { + enableInference: options.enableInference !== false, // Default: true + inferenceConfig: options.inferenceConfig, // Pass configurable inference settings }); - clone.querySelectorAll("a[href]").forEach(link => { - const href = link.getAttribute("href"); - if (href && !href.startsWith("http://") && !href.startsWith("https://") && !href.startsWith("#")) try { - link.setAttribute("href", new URL(href, document.baseURI).href); - } catch (e) {} + const textVal = semanticText.text || getText(el); // Fallback to getText for backward compat + + // Infer role for interactable elements (only if no aria-label and no explicit role) + const inferredRole = getInferredRole(el, { + enableInference: options.enableInference !== false, + inferenceConfig: options.inferenceConfig, }); - return clone.querySelectorAll("img[src]").forEach(img => { - const src = img.getAttribute("src"); - if (src && !src.startsWith("http://") && !src.startsWith("https://") && !src.startsWith("data:")) try { - img.setAttribute("src", new URL(src, document.baseURI).href); - } catch (e) {} - }), clone.innerHTML; - } - function cleanElement(obj) { - if (Array.isArray(obj)) return obj.map(cleanElement); - if (null !== obj && "object" == typeof obj) { - const cleaned = {}; - for (const [key, value] of Object.entries(obj)) if (null != value) if ("object" == typeof value) { - const deepClean = cleanElement(value); - Object.keys(deepClean).length > 0 && (cleaned[key] = deepClean); - } else cleaned[key] = value; - return cleaned; - } - return obj; - } - async function snapshot(options = {}) { + const inView = isInViewport(rect); + + // Get computed style once (needed for both occlusion check and data collection) + const style = window.getComputedStyle(el); + + // Only check occlusion for elements likely to be occluded (optimized) + // This avoids layout thrashing for the vast majority of elements + const occluded = inView ? isOccluded(el, rect, style) : false; + + // Get effective background color (traverses DOM to find non-transparent color) + const effectiveBgColor = getEffectiveBackgroundColor(el); + + rawData.push({ + id: idx, + tag: el.tagName.toLowerCase(), + rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height }, + styles: { + display: toSafeString(style.display), + visibility: toSafeString(style.visibility), + opacity: toSafeString(style.opacity), + z_index: toSafeString(style.zIndex || 'auto'), + position: toSafeString(style.position), + bg_color: toSafeString(effectiveBgColor || style.backgroundColor), + color: toSafeString(style.color), + cursor: toSafeString(style.cursor), + font_weight: toSafeString(style.fontWeight), + font_size: toSafeString(style.fontSize), + }, + attributes: { + role: toSafeString(el.getAttribute('role')), + type_: toSafeString(el.getAttribute('type')), + aria_label: + semanticText?.source === 'explicit_aria_label' + ? semanticText.text + : toSafeString(el.getAttribute('aria-label')), // Keep original for backward compat + inferred_label: + semanticText?.source && + !['explicit_aria_label', 'input_value', 'img_alt', 'inner_text'].includes( + semanticText.source + ) + ? toSafeString(semanticText.text) + : null, + label_source: semanticText?.source || null, // Track source for gateway + inferred_role: inferredRole ? toSafeString(inferredRole) : null, // Inferred role for interactable elements + href: toSafeString(el.href || el.getAttribute('href') || null), + class: toSafeString(getClassName(el)), + // Capture dynamic input state (not just initial attributes) + value: + el.value !== undefined + ? toSafeString(el.value) + : toSafeString(el.getAttribute('value')), + checked: el.checked !== undefined ? String(el.checked) : null, + }, + text: toSafeString(textVal), + in_viewport: inView, + is_occluded: occluded, + // Phase 1: Pass scroll position for doc_y computation in WASM + scroll_y: window.scrollY, + }); + }); + + console.log(`[SentienceAPI] Collected ${rawData.length} elements from main frame`); + + // Step 1.5: Collect iframe snapshots and FLATTEN immediately + // "Flatten Early" architecture: Merge iframe elements into main array before WASM + // This allows WASM to process all elements uniformly (no recursion needed) + const allRawElements = [...rawData]; // Start with main frame elements + let totalIframeElements = 0; + + if (options.collectIframes !== false) { try { - !1 !== options.waitForStability && await async function(options = {}) { - const {minNodeCount: minNodeCount = 500, quietPeriod: quietPeriod = 200, maxWait: maxWait = 5e3} = options, startTime = Date.now(); - return new Promise(resolve => { - if (document.querySelectorAll("*").length >= minNodeCount) { - let lastChange = Date.now(); - const observer = new MutationObserver(() => { - lastChange = Date.now(); - }); - observer.observe(document.body, { - childList: !0, - subtree: !0, - attributes: !1 - }); - const checkStable = () => { - const timeSinceLastChange = Date.now() - lastChange, totalWait = Date.now() - startTime; - timeSinceLastChange >= quietPeriod || totalWait >= maxWait ? (observer.disconnect(), - resolve()) : setTimeout(checkStable, 50); - }; - checkStable(); - } else { - const observer = new MutationObserver(() => { - const currentCount = document.querySelectorAll("*").length, totalWait = Date.now() - startTime; - if (currentCount >= minNodeCount) { - observer.disconnect(); - let lastChange = Date.now(); - const quietObserver = new MutationObserver(() => { - lastChange = Date.now(); - }); - quietObserver.observe(document.body, { - childList: !0, - subtree: !0, - attributes: !1 - }); - const checkQuiet = () => { - const timeSinceLastChange = Date.now() - lastChange, totalWait = Date.now() - startTime; - timeSinceLastChange >= quietPeriod || totalWait >= maxWait ? (quietObserver.disconnect(), - resolve()) : setTimeout(checkQuiet, 50); - }; - checkQuiet(); - } else totalWait >= maxWait && (observer.disconnect(), resolve()); - }); - observer.observe(document.body, { - childList: !0, - subtree: !0, - attributes: !1 - }), setTimeout(() => { - observer.disconnect(), resolve(); - }, maxWait); - } - }); - }(options.waitForStability || {}); - const rawData = []; - window.sentience_registry = []; - getAllElements().forEach((el, idx) => { - if (!el.getBoundingClientRect) return; - const rect = el.getBoundingClientRect(); - if (rect.width < 5 || rect.height < 5) return; - window.sentience_registry[idx] = el; - const semanticText = function(el, options = {}) { - if (!el) return { - text: "", - source: null - }; - const explicitAriaLabel = el.getAttribute ? el.getAttribute("aria-label") : null; - if (explicitAriaLabel && explicitAriaLabel.trim()) return { - text: explicitAriaLabel.trim(), - source: "explicit_aria_label" - }; - if ("INPUT" === el.tagName) { - const value = (el.value || el.placeholder || "").trim(); - if (value) return { - text: value, - source: "input_value" - }; - } - if ("IMG" === el.tagName) { - const alt = (el.alt || "").trim(); - if (alt) return { - text: alt, - source: "img_alt" - }; - } - const innerText = (el.innerText || "").trim(); - if (innerText) return { - text: innerText.substring(0, 100), - source: "inner_text" - }; - const inferred = getInferredLabel(el, { - enableInference: !1 !== options.enableInference, - inferenceConfig: options.inferenceConfig - }); - return inferred || { - text: "", - source: null + console.log(`[SentienceAPI] Starting iframe collection...`); + const iframeSnapshots = await collectIframeSnapshots(options); + console.log( + `[SentienceAPI] Iframe collection complete. Received ${iframeSnapshots.size} snapshot(s)` + ); + + if (iframeSnapshots.size > 0) { + // FLATTEN IMMEDIATELY: Don't nest them. Just append them with coordinate translation. + iframeSnapshots.forEach((iframeSnapshot, iframeEl) => { + // Debug: Log structure to verify data is correct + // console.log(`[SentienceAPI] Processing iframe snapshot:`, iframeSnapshot); + + if (iframeSnapshot && iframeSnapshot.raw_elements) { + const rawElementsCount = iframeSnapshot.raw_elements.length; + console.log( + `[SentienceAPI] Processing ${rawElementsCount} elements from iframe (src: ${iframeEl.src || 'unknown'})` + ); + // Get iframe's bounding rect (offset for coordinate translation) + const iframeRect = iframeEl.getBoundingClientRect(); + const offset = { x: iframeRect.x, y: iframeRect.y }; + + // Get iframe context for frame switching (Playwright needs this) + const iframeSrc = iframeEl.src || iframeEl.getAttribute('src') || ''; + let isSameOrigin = false; + try { + // Try to access contentWindow to check if same-origin + isSameOrigin = iframeEl.contentWindow !== null; + } catch (e) { + isSameOrigin = false; + } + + // Adjust coordinates and add iframe context to each element + const adjustedElements = iframeSnapshot.raw_elements.map((el) => { + const adjusted = { ...el }; + + // Adjust rect coordinates to parent viewport + if (adjusted.rect) { + adjusted.rect = { + ...adjusted.rect, + x: adjusted.rect.x + offset.x, + y: adjusted.rect.y + offset.y, }; - }(el, { - enableInference: !1 !== options.enableInference, - inferenceConfig: options.inferenceConfig - }), textVal = semanticText.text || getText(el), inferredRole = function(el, options = {}) { - const {enableInference: enableInference = !0} = options; - if (!enableInference) return null; - if (!function(el) { - if (!el || !el.tagName) return !1; - const tag = el.tagName.toLowerCase(), role = el.getAttribute ? el.getAttribute("role") : null, hasTabIndex = !!el.hasAttribute && el.hasAttribute("tabindex"), hasHref = "A" === el.tagName && !!el.hasAttribute && el.hasAttribute("href"); - return [ "button", "input", "textarea", "select", "option", "details", "summary", "a" ].includes(tag) ? !("a" === tag && !hasHref) : !(!role || ![ "button", "link", "tab", "menuitem", "checkbox", "radio", "switch", "slider", "combobox", "textbox", "searchbox", "spinbutton" ].includes(role.toLowerCase())) || (!!hasTabIndex || (!!(el.onclick || el.onkeydown || el.onkeypress || el.onkeyup) || !(!el.getAttribute || !(el.getAttribute("onclick") || el.getAttribute("onkeydown") || el.getAttribute("onkeypress") || el.getAttribute("onkeyup"))))); - }(el)) return null; - const hasAriaLabel = el.getAttribute ? el.getAttribute("aria-label") : null, hasExplicitRole = el.getAttribute ? el.getAttribute("role") : null; - if (hasAriaLabel || hasExplicitRole) return null; - const tag = el.tagName.toLowerCase(); - return [ "button", "a", "input", "textarea", "select", "option" ].includes(tag) ? null : el.onclick || el.getAttribute && el.getAttribute("onclick") || el.onkeydown || el.onkeypress || el.onkeyup || el.getAttribute && (el.getAttribute("onkeydown") || el.getAttribute("onkeypress") || el.getAttribute("onkeyup")) || el.hasAttribute && el.hasAttribute("tabindex") && ("div" === tag || "span" === tag) ? "button" : null; - }(el, { - enableInference: !1 !== options.enableInference, - inferenceConfig: options.inferenceConfig - }), inView = function(rect) { - return rect.top < window.innerHeight && rect.bottom > 0 && rect.left < window.innerWidth && rect.right > 0; - }(rect), style = window.getComputedStyle(el), occluded = !!inView && function(el, rect, style) { - const zIndex = parseInt(style.zIndex, 10); - if ("static" === style.position && (isNaN(zIndex) || zIndex <= 10)) return !1; - const cx = rect.x + rect.width / 2, cy = rect.y + rect.height / 2; - if (cx < 0 || cx > window.innerWidth || cy < 0 || cy > window.innerHeight) return !1; - const topEl = document.elementFromPoint(cx, cy); - return !!topEl && !(el === topEl || el.contains(topEl) || topEl.contains(el)); - }(el, rect, style), effectiveBgColor = function(el) { - if (!el) return null; - if ("SVG" === el.tagName) { - const svgColor = getSVGColor(el); - if (svgColor) return svgColor; - } - let current = el, depth = 0; - for (;current && depth < 10; ) { - const style = window.getComputedStyle(current); - if ("SVG" === current.tagName) { - const svgColor = getSVGColor(current); - if (svgColor) return svgColor; - } - const bgColor = style.backgroundColor; - if (bgColor && "transparent" !== bgColor && "rgba(0, 0, 0, 0)" !== bgColor) { - const rgbaMatch = bgColor.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); - if (!rgbaMatch) return bgColor.startsWith("rgb("), bgColor; - if ((rgbaMatch[4] ? parseFloat(rgbaMatch[4]) : 1) >= .9) return `rgb(${rgbaMatch[1]}, ${rgbaMatch[2]}, ${rgbaMatch[3]})`; - } - current = current.parentElement, depth++; - } - return null; - }(el); - rawData.push({ - id: idx, - tag: el.tagName.toLowerCase(), - rect: { - x: rect.x, - y: rect.y, - width: rect.width, - height: rect.height - }, - styles: { - display: toSafeString(style.display), - visibility: toSafeString(style.visibility), - opacity: toSafeString(style.opacity), - z_index: toSafeString(style.zIndex || "auto"), - position: toSafeString(style.position), - bg_color: toSafeString(effectiveBgColor || style.backgroundColor), - color: toSafeString(style.color), - cursor: toSafeString(style.cursor), - font_weight: toSafeString(style.fontWeight), - font_size: toSafeString(style.fontSize) - }, - attributes: { - role: toSafeString(el.getAttribute("role")), - type_: toSafeString(el.getAttribute("type")), - aria_label: "explicit_aria_label" === semanticText?.source ? semanticText.text : toSafeString(el.getAttribute("aria-label")), - inferred_label: semanticText?.source && ![ "explicit_aria_label", "input_value", "img_alt", "inner_text" ].includes(semanticText.source) ? toSafeString(semanticText.text) : null, - label_source: semanticText?.source || null, - inferred_role: inferredRole ? toSafeString(inferredRole) : null, - href: toSafeString(el.href || el.getAttribute("href") || null), - class: toSafeString(getClassName(el)), - value: void 0 !== el.value ? toSafeString(el.value) : toSafeString(el.getAttribute("value")), - checked: void 0 !== el.checked ? String(el.checked) : null - }, - text: toSafeString(textVal), - in_viewport: inView, - is_occluded: occluded + } + + // Add iframe context so agents can switch frames in Playwright + adjusted.iframe_context = { + src: iframeSrc, + is_same_origin: isSameOrigin, + }; + + return adjusted; }); + + // Append flattened iframe elements to main array + allRawElements.push(...adjustedElements); + totalIframeElements += adjustedElements.length; + } }); - const allRawElements = [ ...rawData ]; - let totalIframeElements = 0; - if (!1 !== options.collectIframes) try { - const iframeSnapshots = await async function(options = {}) { - const iframeData = new Map, iframes = Array.from(document.querySelectorAll("iframe")); - if (0 === iframes.length) return iframeData; - const iframePromises = iframes.map((iframe, idx) => { - const src = iframe.src || ""; - return src.includes("doubleclick") || src.includes("googleadservices") || src.includes("ads system") ? Promise.resolve(null) : new Promise(resolve => { - const requestId = `iframe-${idx}-${Date.now()}`, timeout = setTimeout(() => { - resolve(null); - }, 5e3), listener = event => { - "SENTIENCE_IFRAME_SNAPSHOT_RESPONSE" === event.data?.type && event.data, "SENTIENCE_IFRAME_SNAPSHOT_RESPONSE" === event.data?.type && event.data?.requestId === requestId && (clearTimeout(timeout), - window.removeEventListener("message", listener), event.data.error ? resolve(null) : (event.data.snapshot, - resolve({ - iframe: iframe, - data: event.data.snapshot, - error: null - }))); - }; - window.addEventListener("message", listener); - try { - iframe.contentWindow ? iframe.contentWindow.postMessage({ - type: "SENTIENCE_IFRAME_SNAPSHOT_REQUEST", - requestId: requestId, - options: { - ...options, - collectIframes: !0 - } - }, "*") : (clearTimeout(timeout), window.removeEventListener("message", listener), - resolve(null)); - } catch (error) { - clearTimeout(timeout), window.removeEventListener("message", listener), resolve(null); - } - }); - }); - return (await Promise.all(iframePromises)).forEach((result, idx) => { - result && result.data && !result.error ? iframeData.set(iframes[idx], result.data) : result && result.error; - }), iframeData; - }(options); - iframeSnapshots.size > 0 && iframeSnapshots.forEach((iframeSnapshot, iframeEl) => { - if (iframeSnapshot && iframeSnapshot.raw_elements) { - iframeSnapshot.raw_elements.length; - const iframeRect = iframeEl.getBoundingClientRect(), offset = { - x: iframeRect.x, - y: iframeRect.y - }, iframeSrc = iframeEl.src || iframeEl.getAttribute("src") || ""; - let isSameOrigin = !1; - try { - isSameOrigin = null !== iframeEl.contentWindow; - } catch (e) { - isSameOrigin = !1; - } - const adjustedElements = iframeSnapshot.raw_elements.map(el => { - const adjusted = { - ...el - }; - return adjusted.rect && (adjusted.rect = { - ...adjusted.rect, - x: adjusted.rect.x + offset.x, - y: adjusted.rect.y + offset.y - }), adjusted.iframe_context = { - src: iframeSrc, - is_same_origin: isSameOrigin - }, adjusted; - }); - allRawElements.push(...adjustedElements), totalIframeElements += adjustedElements.length; - } - }); - } catch (error) {} - const processed = await function(rawData, options) { - return new Promise((resolve, reject) => { - const requestId = Math.random().toString(36).substring(7); - let resolved = !1; - const timeout = setTimeout(() => { - resolved || (resolved = !0, window.removeEventListener("message", listener), reject(new Error("WASM processing timeout - extension may be unresponsive. Try reloading the extension."))); - }, 25e3), listener = e => { - if ("SENTIENCE_SNAPSHOT_RESULT" === e.data.type && e.data.requestId === requestId) { - if (resolved) return; - resolved = !0, clearTimeout(timeout), window.removeEventListener("message", listener), - e.data.error ? reject(new Error(e.data.error)) : resolve({ - elements: e.data.elements, - raw_elements: e.data.raw_elements, - duration: e.data.duration - }); - } - }; - window.addEventListener("message", listener); - try { - window.postMessage({ - type: "SENTIENCE_SNAPSHOT_REQUEST", - requestId: requestId, - rawData: rawData, - options: options - }, "*"); - } catch (error) { - resolved || (resolved = !0, clearTimeout(timeout), window.removeEventListener("message", listener), - reject(new Error(`Failed to send snapshot request: ${error.message}`))); - } - }); - }(allRawElements, options); - if (!processed || !processed.elements) throw new Error("WASM processing returned invalid result"); - let screenshot = null; - options.screenshot && (screenshot = await function(options) { - return new Promise(resolve => { - const requestId = Math.random().toString(36).substring(7), listener = e => { - "SENTIENCE_SCREENSHOT_RESULT" === e.data.type && e.data.requestId === requestId && (window.removeEventListener("message", listener), - resolve(e.data.screenshot)); - }; - window.addEventListener("message", listener), window.postMessage({ - type: "SENTIENCE_SCREENSHOT_REQUEST", - requestId: requestId, - options: options - }, "*"), setTimeout(() => { - window.removeEventListener("message", listener), resolve(null); - }, 1e4); - }); - }(options.screenshot)); - const cleanedElements = cleanElement(processed.elements), cleanedRawElements = cleanElement(processed.raw_elements); - cleanedElements.length, cleanedRawElements.length; - return { - status: "success", - url: window.location.href, - viewport: { - width: window.innerWidth, - height: window.innerHeight - }, - elements: cleanedElements, - raw_elements: cleanedRawElements, - screenshot: screenshot - }; + + // console.log(`[SentienceAPI] Merged ${iframeSnapshots.size} iframe(s). Total elements: ${allRawElements.length} (${rawData.length} main + ${totalIframeElements} iframe)`); + } } catch (error) { - return { - status: "error", - error: error.message || "Unknown error", - stack: error.stack - }; + console.warn('[SentienceAPI] Iframe collection failed:', error); } + } + + // Step 2: Send EVERYTHING to WASM (One giant flat list) + // Now WASM prunes iframe elements and main elements in one pass! + // No recursion needed - everything is already flat + console.log( + `[SentienceAPI] Sending ${allRawElements.length} total elements to WASM (${rawData.length} main + ${totalIframeElements} iframe)` + ); + const processed = await processSnapshotInBackground(allRawElements, options); + + if (!processed || !processed.elements) { + throw new Error('WASM processing returned invalid result'); + } + + // Step 3: Capture screenshot if requested + let screenshot = null; + if (options.screenshot) { + screenshot = await captureScreenshot(options.screenshot); + } + + // Step 4: Clean and return + const cleanedElements = cleanElement(processed.elements); + const cleanedRawElements = cleanElement(processed.raw_elements); + + // FIXED: Removed undefined 'totalIframeRawElements' + // FIXED: Logic updated for "Flatten Early" architecture. + // processed.elements ALREADY contains the merged iframe elements, + // so we simply use .length. No addition needed. + + const totalCount = cleanedElements.length; + const totalRaw = cleanedRawElements.length; + const iframeCount = totalIframeElements || 0; + + console.log( + `[SentienceAPI] ✓ Complete: ${totalCount} Smart Elements, ${totalRaw} Raw Elements (includes ${iframeCount} from iframes) (WASM took ${processed.duration?.toFixed(1)}ms)` + ); + + return { + status: 'success', + url: window.location.href, + viewport: { + width: window.innerWidth, + height: window.innerHeight, + }, + elements: cleanedElements, + raw_elements: cleanedRawElements, + screenshot, + }; + } catch (error) { + console.error('[SentienceAPI] snapshot() failed:', error); + console.error('[SentienceAPI] Error stack:', error.stack); + return { + status: 'error', + error: error.message || 'Unknown error', + stack: error.stack, + }; } - function read(options = {}) { - const format = options.format || "raw"; - let content; - return content = "raw" === format ? getRawHTML(document.body) : "markdown" === format ? function(root) { - const rawHTML = getRawHTML(root), tempDiv = document.createElement("div"); - tempDiv.innerHTML = rawHTML; - let markdown = "", insideLink = !1; - return function walk(node) { - if (node.nodeType === Node.TEXT_NODE) { - const text = node.textContent.replace(/[\r\n]+/g, " ").replace(/\s+/g, " "); - return void (text.trim() && (markdown += text)); - } - if (node.nodeType !== Node.ELEMENT_NODE) return; - const tag = node.tagName.toLowerCase(); - if ("h1" === tag && (markdown += "\n# "), "h2" === tag && (markdown += "\n## "), - "h3" === tag && (markdown += "\n### "), "li" === tag && (markdown += "\n- "), insideLink || "p" !== tag && "div" !== tag && "br" !== tag || (markdown += "\n"), - "strong" !== tag && "b" !== tag || (markdown += "**"), "em" !== tag && "i" !== tag || (markdown += "_"), - "a" === tag && (markdown += "[", insideLink = !0), node.shadowRoot ? Array.from(node.shadowRoot.childNodes).forEach(walk) : node.childNodes.forEach(walk), - "a" === tag) { - const href = node.getAttribute("href"); - markdown += href ? `](${href})` : "]", insideLink = !1; - } - "strong" !== tag && "b" !== tag || (markdown += "**"), "em" !== tag && "i" !== tag || (markdown += "_"), - insideLink || "h1" !== tag && "h2" !== tag && "h3" !== tag && "p" !== tag && "div" !== tag || (markdown += "\n"); - }(tempDiv), markdown.replace(/\n{3,}/g, "\n\n").trim(); - }(document.body) : function(root) { - let text = ""; - return function walk(node) { - if (node.nodeType !== Node.TEXT_NODE) { - if (node.nodeType === Node.ELEMENT_NODE) { - const tag = node.tagName.toLowerCase(); - if ([ "nav", "footer", "header", "script", "style", "noscript", "iframe", "svg" ].includes(tag)) return; - const style = window.getComputedStyle(node); - if ("none" === style.display || "hidden" === style.visibility) return; - const isBlock = "block" === style.display || "flex" === style.display || "P" === node.tagName || "DIV" === node.tagName; - isBlock && (text += " "), node.shadowRoot ? Array.from(node.shadowRoot.childNodes).forEach(walk) : node.childNodes.forEach(walk), - isBlock && (text += "\n"); - } - } else text += node.textContent; - }(root || document.body), text.replace(/\n{3,}/g, "\n\n").trim(); - }(document.body), { - status: "success", - url: window.location.href, - format: format, - content: content, - length: content.length - }; + } + + // read.js - Content Reading Methods + + // 2. Read Content (unchanged) + function read(options = {}) { + const format = options.format || 'raw'; + let content; + + if (format === 'raw') { + content = getRawHTML(document.body); + } else if (format === 'markdown') { + content = convertToMarkdown(document.body); + } else { + content = convertToText(document.body); } - function findTextRect(options = {}) { - const {text: text, containerElement: containerElement = document.body, caseSensitive: caseSensitive = !1, wholeWord: wholeWord = !1, maxResults: maxResults = 10} = options; - if (!text || 0 === text.trim().length) return { - status: "error", - error: "Text parameter is required" - }; - const results = [], searchText = caseSensitive ? text : text.toLowerCase(); - function findInTextNode(textNode) { - const nodeText = textNode.nodeValue, searchableText = caseSensitive ? nodeText : nodeText.toLowerCase(); - let startIndex = 0; - for (;startIndex < nodeText.length && results.length < maxResults; ) { - const foundIndex = searchableText.indexOf(searchText, startIndex); - if (-1 === foundIndex) break; - if (wholeWord) { - const before = foundIndex > 0 ? nodeText[foundIndex - 1] : " ", after = foundIndex + text.length < nodeText.length ? nodeText[foundIndex + text.length] : " "; - if (!/\s/.test(before) || !/\s/.test(after)) { - startIndex = foundIndex + 1; - continue; - } - } - try { - const range = document.createRange(); - range.setStart(textNode, foundIndex), range.setEnd(textNode, foundIndex + text.length); - const rect = range.getBoundingClientRect(); - rect.width > 0 && rect.height > 0 && results.push({ - text: nodeText.substring(foundIndex, foundIndex + text.length), - rect: { - x: rect.left + window.scrollX, - y: rect.top + window.scrollY, - width: rect.width, - height: rect.height, - left: rect.left + window.scrollX, - top: rect.top + window.scrollY, - right: rect.right + window.scrollX, - bottom: rect.bottom + window.scrollY - }, - viewport_rect: { - x: rect.left, - y: rect.top, - width: rect.width, - height: rect.height - }, - context: { - before: nodeText.substring(Math.max(0, foundIndex - 20), foundIndex), - after: nodeText.substring(foundIndex + text.length, Math.min(nodeText.length, foundIndex + text.length + 20)) - }, - in_viewport: rect.top >= 0 && rect.left >= 0 && rect.bottom <= window.innerHeight && rect.right <= window.innerWidth - }); - } catch (e) {} - startIndex = foundIndex + 1; - } + + return { + status: 'success', + url: window.location.href, + format, + content, + length: content.length, + }; + } + + // 2b. Find Text Rectangle - Get exact pixel coordinates of specific text + function findTextRect(options = {}) { + const { + text, + containerElement = document.body, + caseSensitive = false, + wholeWord = false, + maxResults = 10, + } = options; + + if (!text || text.trim().length === 0) { + return { + status: 'error', + error: 'Text parameter is required', + }; + } + + const results = []; + const searchText = caseSensitive ? text : text.toLowerCase(); + + // Helper function to find text in a single text node + function findInTextNode(textNode) { + const nodeText = textNode.nodeValue; + const searchableText = caseSensitive ? nodeText : nodeText.toLowerCase(); + + let startIndex = 0; + while (startIndex < nodeText.length && results.length < maxResults) { + const foundIndex = searchableText.indexOf(searchText, startIndex); + + if (foundIndex === -1) break; + + // Check whole word matching if required + if (wholeWord) { + const before = foundIndex > 0 ? nodeText[foundIndex - 1] : ' '; + const after = + foundIndex + text.length < nodeText.length ? nodeText[foundIndex + text.length] : ' '; + + // Check if surrounded by word boundaries + if (!/\s/.test(before) || !/\s/.test(after)) { + startIndex = foundIndex + 1; + continue; + } } - const walker = document.createTreeWalker(containerElement, NodeFilter.SHOW_TEXT, { - acceptNode(node) { - const parent = node.parentElement; - if (!parent) return NodeFilter.FILTER_REJECT; - const tagName = parent.tagName.toLowerCase(); - if ("script" === tagName || "style" === tagName || "noscript" === tagName) return NodeFilter.FILTER_REJECT; - if (!node.nodeValue || 0 === node.nodeValue.trim().length) return NodeFilter.FILTER_REJECT; - const computedStyle = window.getComputedStyle(parent); - return "none" === computedStyle.display || "hidden" === computedStyle.visibility || "0" === computedStyle.opacity ? NodeFilter.FILTER_REJECT : NodeFilter.FILTER_ACCEPT; - } - }); - let currentNode; - for (;(currentNode = walker.nextNode()) && results.length < maxResults; ) findInTextNode(currentNode); - return { - status: "success", - query: text, - case_sensitive: caseSensitive, - whole_word: wholeWord, - matches: results.length, - results: results, - viewport: { - width: window.innerWidth, - height: window.innerHeight, - scroll_x: window.scrollX, - scroll_y: window.scrollY - } - }; + + try { + // Create range for this occurrence + const range = document.createRange(); + range.setStart(textNode, foundIndex); + range.setEnd(textNode, foundIndex + text.length); + + const rect = range.getBoundingClientRect(); + + // Only include visible rectangles + if (rect.width > 0 && rect.height > 0) { + results.push({ + text: nodeText.substring(foundIndex, foundIndex + text.length), + rect: { + x: rect.left + window.scrollX, + y: rect.top + window.scrollY, + width: rect.width, + height: rect.height, + left: rect.left + window.scrollX, + top: rect.top + window.scrollY, + right: rect.right + window.scrollX, + bottom: rect.bottom + window.scrollY, + }, + viewport_rect: { + x: rect.left, + y: rect.top, + width: rect.width, + height: rect.height, + }, + context: { + before: nodeText.substring(Math.max(0, foundIndex - 20), foundIndex), + after: nodeText.substring( + foundIndex + text.length, + Math.min(nodeText.length, foundIndex + text.length + 20) + ), + }, + in_viewport: + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= window.innerHeight && + rect.right <= window.innerWidth, + }); + } + } catch (e) { + console.warn('[SentienceAPI] Failed to get rect for text:', e); + } + + startIndex = foundIndex + 1; + } + } + + // Tree walker to find all text nodes + const walker = document.createTreeWalker(containerElement, NodeFilter.SHOW_TEXT, { + acceptNode(node) { + // Skip script, style, and empty text nodes + const parent = node.parentElement; + if (!parent) return NodeFilter.FILTER_REJECT; + + const tagName = parent.tagName.toLowerCase(); + if (tagName === 'script' || tagName === 'style' || tagName === 'noscript') { + return NodeFilter.FILTER_REJECT; + } + + // Skip whitespace-only nodes + if (!node.nodeValue || node.nodeValue.trim().length === 0) { + return NodeFilter.FILTER_REJECT; + } + + // Check if element is visible + const computedStyle = window.getComputedStyle(parent); + if ( + computedStyle.display === 'none' || + computedStyle.visibility === 'hidden' || + computedStyle.opacity === '0' + ) { + return NodeFilter.FILTER_REJECT; + } + + return NodeFilter.FILTER_ACCEPT; + }, + }); + + // Walk through all text nodes + let currentNode; + while ((currentNode = walker.nextNode()) && results.length < maxResults) { + findInTextNode(currentNode); + } + + return { + status: 'success', + query: text, + case_sensitive: caseSensitive, + whole_word: wholeWord, + matches: results.length, + results, + viewport: { + width: window.innerWidth, + height: window.innerHeight, + scroll_x: window.scrollX, + scroll_y: window.scrollY, + }, + }; + } + + // click.js - Click Action Method + + // 3. Click Action (unchanged) + function click(id) { + const el = window.sentience_registry[id]; + if (el) { + el.click(); + el.focus(); + return true; } - function click(id) { - const el = window.sentience_registry[id]; - return !!el && (el.click(), el.focus(), !0); + return false; + } + + // registry.js - Inspector Mode / Golden Set Collection + + // 4. Inspector Mode: Start Recording for Golden Set Collection + function startRecording(options = {}) { + const { + highlightColor = '#ff0000', + successColor = '#00ff00', + autoDisableTimeout = 30 * 60 * 1000, // 30 minutes default + keyboardShortcut = 'Ctrl+Shift+I', + } = options; + + console.log( + '🔴 [Sentience] Recording Mode STARTED. Click an element to copy its Ground Truth JSON.' + ); + console.log(` Press ${keyboardShortcut} or call stopRecording() to stop.`); + + // Validate registry is populated + if (!window.sentience_registry || window.sentience_registry.length === 0) { + console.warn( + '⚠️ Registry empty. Call `await window.sentience.snapshot()` first to populate registry.' + ); + alert('Registry empty. Run `await window.sentience.snapshot()` first!'); + return () => {}; // Return no-op cleanup function } - function startRecording(options = {}) { - const {highlightColor: highlightColor = "#ff0000", successColor: successColor = "#00ff00", autoDisableTimeout: autoDisableTimeout = 18e5, keyboardShortcut: keyboardShortcut = "Ctrl+Shift+I"} = options; - if (!window.sentience_registry || 0 === window.sentience_registry.length) return alert("Registry empty. Run `await window.sentience.snapshot()` first!"), - () => {}; - window.sentience_registry_map = new Map, window.sentience_registry.forEach((el, idx) => { - el && window.sentience_registry_map.set(el, idx); + + // Create reverse mapping for O(1) lookup (fixes registry lookup bug) + window.sentience_registry_map = new Map(); + window.sentience_registry.forEach((el, idx) => { + if (el) window.sentience_registry_map.set(el, idx); + }); + + // Create highlight box overlay + let highlightBox = document.getElementById('sentience-highlight-box'); + if (!highlightBox) { + highlightBox = document.createElement('div'); + highlightBox.id = 'sentience-highlight-box'; + highlightBox.style.cssText = ` + position: fixed; + pointer-events: none; + z-index: 2147483647; + border: 2px solid ${highlightColor}; + background: rgba(255, 0, 0, 0.1); + display: none; + transition: all 0.1s ease; + box-sizing: border-box; + `; + document.body.appendChild(highlightBox); + } + + // Create visual indicator (red border on page when recording) + let recordingIndicator = document.getElementById('sentience-recording-indicator'); + if (!recordingIndicator) { + recordingIndicator = document.createElement('div'); + recordingIndicator.id = 'sentience-recording-indicator'; + recordingIndicator.style.cssText = ` + position: fixed; + top: 0; + left: 0; + right: 0; + height: 3px; + background: ${highlightColor}; + z-index: 2147483646; + pointer-events: none; + `; + document.body.appendChild(recordingIndicator); + } + recordingIndicator.style.display = 'block'; + + // Hover handler (visual feedback) + const mouseOverHandler = (e) => { + const el = e.target; + if (!el || el === highlightBox || el === recordingIndicator) return; + + const rect = el.getBoundingClientRect(); + highlightBox.style.display = 'block'; + highlightBox.style.top = rect.top + window.scrollY + 'px'; + highlightBox.style.left = rect.left + window.scrollX + 'px'; + highlightBox.style.width = rect.width + 'px'; + highlightBox.style.height = rect.height + 'px'; + }; + + // Click handler (capture ground truth data) + const clickHandler = (e) => { + e.preventDefault(); + e.stopPropagation(); + + const el = e.target; + if (!el || el === highlightBox || el === recordingIndicator) return; + + // Use Map for reliable O(1) lookup + const sentienceId = window.sentience_registry_map.get(el); + if (sentienceId === undefined) { + console.warn('⚠️ Element not found in Sentience Registry. Did you run snapshot() first?'); + alert('Element not in registry. Run `await window.sentience.snapshot()` first!'); + return; + } + + // Extract raw data (ground truth + raw signals, NOT model outputs) + const rawData = extractRawElementData(el); + const selector = getUniqueSelector(el); + const role = el.getAttribute('role') || el.tagName.toLowerCase(); + const text = getText(el); + + // Build golden set JSON (ground truth + raw signals only) + const snippet = { + task: `Interact with ${text.substring(0, 20)}${text.length > 20 ? '...' : ''}`, + url: window.location.href, + timestamp: new Date().toISOString(), + target_criteria: { + id: sentienceId, + selector, + role, + text: text.substring(0, 50), + }, + debug_snapshot: rawData, + }; + + // Copy to clipboard + const jsonString = JSON.stringify(snippet, null, 2); + navigator.clipboard + .writeText(jsonString) + .then(() => { + console.log('✅ Copied Ground Truth to clipboard:', snippet); + + // Flash green to indicate success + highlightBox.style.border = `2px solid ${successColor}`; + highlightBox.style.background = 'rgba(0, 255, 0, 0.2)'; + setTimeout(() => { + highlightBox.style.border = `2px solid ${highlightColor}`; + highlightBox.style.background = 'rgba(255, 0, 0, 0.1)'; + }, 500); + }) + .catch((err) => { + console.error('❌ Failed to copy to clipboard:', err); + alert('Failed to copy to clipboard. Check console for JSON.'); }); - let highlightBox = document.getElementById("sentience-highlight-box"); - highlightBox || (highlightBox = document.createElement("div"), highlightBox.id = "sentience-highlight-box", - highlightBox.style.cssText = `\n position: fixed;\n pointer-events: none;\n z-index: 2147483647;\n border: 2px solid ${highlightColor};\n background: rgba(255, 0, 0, 0.1);\n display: none;\n transition: all 0.1s ease;\n box-sizing: border-box;\n `, - document.body.appendChild(highlightBox)); - let recordingIndicator = document.getElementById("sentience-recording-indicator"); - recordingIndicator || (recordingIndicator = document.createElement("div"), recordingIndicator.id = "sentience-recording-indicator", - recordingIndicator.style.cssText = `\n position: fixed;\n top: 0;\n left: 0;\n right: 0;\n height: 3px;\n background: ${highlightColor};\n z-index: 2147483646;\n pointer-events: none;\n `, - document.body.appendChild(recordingIndicator)), recordingIndicator.style.display = "block"; - const mouseOverHandler = e => { - const el = e.target; - if (!el || el === highlightBox || el === recordingIndicator) return; - const rect = el.getBoundingClientRect(); - highlightBox.style.display = "block", highlightBox.style.top = rect.top + window.scrollY + "px", - highlightBox.style.left = rect.left + window.scrollX + "px", highlightBox.style.width = rect.width + "px", - highlightBox.style.height = rect.height + "px"; - }, clickHandler = e => { - e.preventDefault(), e.stopPropagation(); - const el = e.target; - if (!el || el === highlightBox || el === recordingIndicator) return; - const sentienceId = window.sentience_registry_map.get(el); - if (void 0 === sentienceId) return void alert("Element not in registry. Run `await window.sentience.snapshot()` first!"); - const rawData = function(el) { - const style = window.getComputedStyle(el), rect = el.getBoundingClientRect(); - return { - tag: el.tagName, - rect: { - x: Math.round(rect.x), - y: Math.round(rect.y), - width: Math.round(rect.width), - height: Math.round(rect.height) - }, - styles: { - cursor: style.cursor || null, - backgroundColor: style.backgroundColor || null, - color: style.color || null, - fontWeight: style.fontWeight || null, - fontSize: style.fontSize || null, - display: style.display || null, - position: style.position || null, - zIndex: style.zIndex || null, - opacity: style.opacity || null, - visibility: style.visibility || null - }, - attributes: { - role: el.getAttribute("role") || null, - type: el.getAttribute("type") || null, - ariaLabel: el.getAttribute("aria-label") || null, - id: el.id || null, - className: el.className || null - } - }; - }(el), selector = function(el) { - if (!el || !el.tagName) return ""; - if (el.id) return `#${el.id}`; - for (const attr of el.attributes) if (attr.name.startsWith("data-") || "aria-label" === attr.name) { - const value = attr.value ? attr.value.replace(/"/g, '\\"') : ""; - return `${el.tagName.toLowerCase()}[${attr.name}="${value}"]`; - } - const path = []; - let current = el; - for (;current && current !== document.body && current !== document.documentElement; ) { - let selector = current.tagName.toLowerCase(); - if (current.id) { - selector = `#${current.id}`, path.unshift(selector); - break; - } - if (current.className && "string" == typeof current.className) { - const classes = current.className.trim().split(/\s+/).filter(c => c); - classes.length > 0 && (selector += `.${classes[0]}`); - } - if (current.parentElement) { - const sameTagSiblings = Array.from(current.parentElement.children).filter(s => s.tagName === current.tagName), index = sameTagSiblings.indexOf(current); - (index > 0 || sameTagSiblings.length > 1) && (selector += `:nth-of-type(${index + 1})`); - } - path.unshift(selector), current = current.parentElement; - } - return path.join(" > ") || el.tagName.toLowerCase(); - }(el), role = el.getAttribute("role") || el.tagName.toLowerCase(), text = getText(el), snippet = { - task: `Interact with ${text.substring(0, 20)}${text.length > 20 ? "..." : ""}`, - url: window.location.href, - timestamp: (new Date).toISOString(), - target_criteria: { - id: sentienceId, - selector: selector, - role: role, - text: text.substring(0, 50) - }, - debug_snapshot: rawData - }, jsonString = JSON.stringify(snippet, null, 2); - navigator.clipboard.writeText(jsonString).then(() => { - highlightBox.style.border = `2px solid ${successColor}`, highlightBox.style.background = "rgba(0, 255, 0, 0.2)", - setTimeout(() => { - highlightBox.style.border = `2px solid ${highlightColor}`, highlightBox.style.background = "rgba(255, 0, 0, 0.1)"; - }, 500); - }).catch(err => { - alert("Failed to copy to clipboard. Check console for JSON."); - }); - }; - let timeoutId = null; - const stopRecording = () => { - document.removeEventListener("mouseover", mouseOverHandler, !0), document.removeEventListener("click", clickHandler, !0), - document.removeEventListener("keydown", keyboardHandler, !0), timeoutId && (clearTimeout(timeoutId), - timeoutId = null), highlightBox && (highlightBox.style.display = "none"), recordingIndicator && (recordingIndicator.style.display = "none"), - window.sentience_registry_map && window.sentience_registry_map.clear(), window.sentience_stopRecording === stopRecording && delete window.sentience_stopRecording; - }, keyboardHandler = e => { - (e.ctrlKey || e.metaKey) && e.shiftKey && "I" === e.key && (e.preventDefault(), - stopRecording()); - }; - return document.addEventListener("mouseover", mouseOverHandler, !0), document.addEventListener("click", clickHandler, !0), - document.addEventListener("keydown", keyboardHandler, !0), autoDisableTimeout > 0 && (timeoutId = setTimeout(() => { - stopRecording(); - }, autoDisableTimeout)), window.sentience_stopRecording = stopRecording, stopRecording; - } - function showOverlay(elements, targetElementId = null) { - elements && Array.isArray(elements) && window.postMessage({ - type: "SENTIENCE_SHOW_OVERLAY", - elements: elements, - targetElementId: targetElementId, - timestamp: Date.now() - }, "*"); - } - function clearOverlay() { - window.postMessage({ - type: "SENTIENCE_CLEAR_OVERLAY" - }, "*"); - } - (async () => { - const getExtensionId = () => document.documentElement.dataset.sentienceExtensionId; - let extId = getExtensionId(); - extId || await new Promise(resolve => { - const check = setInterval(() => { - extId = getExtensionId(), extId && (clearInterval(check), resolve()); - }, 50); - setTimeout(() => resolve(), 5e3); - }), extId && (window.sentience_registry = [], window.sentience = { - snapshot: snapshot, - read: read, - findTextRect: findTextRect, - click: click, - startRecording: startRecording, - showOverlay: showOverlay, - clearOverlay: clearOverlay - }, window.sentience_iframe_handler_setup || (window.addEventListener("message", async event => { - if ("SENTIENCE_IFRAME_SNAPSHOT_REQUEST" === event.data?.type) { - const {requestId: requestId, options: options} = event.data; - try { - const snapshotOptions = { - ...options, - collectIframes: !0, - waitForStability: (options.waitForStability, !1) - }, snapshot = await window.sentience.snapshot(snapshotOptions); - event.source && event.source.postMessage && event.source.postMessage({ - type: "SENTIENCE_IFRAME_SNAPSHOT_RESPONSE", - requestId: requestId, - snapshot: snapshot, - error: null - }, "*"); - } catch (error) { - event.source && event.source.postMessage && event.source.postMessage({ - type: "SENTIENCE_IFRAME_SNAPSHOT_RESPONSE", - requestId: requestId, - snapshot: null, - error: error.message - }, "*"); - } - } - }), window.sentience_iframe_handler_setup = !0)); - })(); -}(); \ No newline at end of file + }; + + // Auto-disable timeout + let timeoutId = null; + + // Cleanup function to stop recording (defined before use) + const stopRecording = () => { + document.removeEventListener('mouseover', mouseOverHandler, true); + document.removeEventListener('click', clickHandler, true); + document.removeEventListener('keydown', keyboardHandler, true); + + if (timeoutId) { + clearTimeout(timeoutId); + timeoutId = null; + } + + if (highlightBox) { + highlightBox.style.display = 'none'; + } + + if (recordingIndicator) { + recordingIndicator.style.display = 'none'; + } + + // Clean up registry map (optional, but good practice) + if (window.sentience_registry_map) { + window.sentience_registry_map.clear(); + } + + // Remove global reference + if (window.sentience_stopRecording === stopRecording) { + delete window.sentience_stopRecording; + } + + console.log('⚪ [Sentience] Recording Mode STOPPED.'); + }; + + // Keyboard shortcut handler (defined after stopRecording) + const keyboardHandler = (e) => { + // Ctrl+Shift+I or Cmd+Shift+I + if ((e.ctrlKey || e.metaKey) && e.shiftKey && e.key === 'I') { + e.preventDefault(); + stopRecording(); + } + }; + + // Attach event listeners (use capture phase to intercept early) + document.addEventListener('mouseover', mouseOverHandler, true); + document.addEventListener('click', clickHandler, true); + document.addEventListener('keydown', keyboardHandler, true); + + // Set up auto-disable timeout + if (autoDisableTimeout > 0) { + timeoutId = setTimeout(() => { + console.log('⏰ [Sentience] Recording Mode auto-disabled after timeout.'); + stopRecording(); + }, autoDisableTimeout); + } + + // Store stop function globally for keyboard shortcut access + window.sentience_stopRecording = stopRecording; + + return stopRecording; + } + + // overlay.js - Visual Overlay Methods + + /** + * Show overlay highlighting specific elements with Shadow DOM + * @param {Array} elements - List of elements with bbox, importance, visual_cues + * @param {number} targetElementId - Optional ID of target element (shown in red) + */ + function showOverlay(elements, targetElementId = null) { + if (!elements || !Array.isArray(elements)) { + console.warn('[Sentience] showOverlay: elements must be an array'); + return; + } + + window.postMessage( + { + type: 'SENTIENCE_SHOW_OVERLAY', + elements, + targetElementId, + timestamp: Date.now(), + }, + '*' + ); + + console.log(`[Sentience] Overlay requested for ${elements.length} elements`); + } + + /** + * Clear overlay manually + */ + function clearOverlay() { + window.postMessage( + { + type: 'SENTIENCE_CLEAR_OVERLAY', + }, + '*' + ); + console.log('[Sentience] Overlay cleared'); + } + + // index.js - Main Entry Point for Injected API + // This script ONLY collects raw DOM data and sends it to background for processing + + + (async () => { + // console.log('[SentienceAPI] Initializing (CSP-Resistant Mode)...'); + + // Wait for Extension ID from content.js + const getExtensionId = () => document.documentElement.dataset.sentienceExtensionId; + let extId = getExtensionId(); + + if (!extId) { + await new Promise((resolve) => { + const check = setInterval(() => { + extId = getExtensionId(); + if (extId) { + clearInterval(check); + resolve(); + } + }, 50); + setTimeout(() => resolve(), 5000); // Max 5s wait + }); + } + + if (!extId) { + console.error('[SentienceAPI] Failed to get extension ID'); + return; + } + + // console.log('[SentienceAPI] Extension ID:', extId); + + // Registry for click actions (still needed for click() function) + window.sentience_registry = []; + + // --- GLOBAL API --- + window.sentience = { + snapshot, + read, + findTextRect, + click, + startRecording, + showOverlay, + clearOverlay, + }; + + // Setup iframe handler when script loads (only once) + if (!window.sentience_iframe_handler_setup) { + setupIframeSnapshotHandler(); + window.sentience_iframe_handler_setup = true; + } + + console.log('[SentienceAPI] ✓ Ready! (CSP-Resistant - WASM runs in background)'); + })(); + +})(); diff --git a/sentience/extension/pkg/sentience_core.js b/sentience/extension/pkg/sentience_core.js index ecba479..b232d13 100644 --- a/sentience/extension/pkg/sentience_core.js +++ b/sentience/extension/pkg/sentience_core.js @@ -1,70 +1,112 @@ let wasm; function addHeapObject(obj) { - heap_next === heap.length && heap.push(heap.length + 1); + if (heap_next === heap.length) heap.push(heap.length + 1); const idx = heap_next; - return heap_next = heap[idx], heap[idx] = obj, idx; + heap_next = heap[idx]; + + heap[idx] = obj; + return idx; } function debugString(val) { + // primitive types const type = typeof val; - if ("number" == type || "boolean" == type || null == val) return `${val}`; - if ("string" == type) return `"${val}"`; - if ("symbol" == type) { + if (type == 'number' || type == 'boolean' || val == null) { + return `${val}`; + } + if (type == 'string') { + return `"${val}"`; + } + if (type == 'symbol') { const description = val.description; - return null == description ? "Symbol" : `Symbol(${description})`; + if (description == null) { + return 'Symbol'; + } else { + return `Symbol(${description})`; + } } - if ("function" == type) { + if (type == 'function') { const name = val.name; - return "string" == typeof name && name.length > 0 ? `Function(${name})` : "Function"; + if (typeof name == 'string' && name.length > 0) { + return `Function(${name})`; + } else { + return 'Function'; + } } + // objects if (Array.isArray(val)) { const length = val.length; - let debug = "["; - length > 0 && (debug += debugString(val[0])); - for (let i = 1; i < length; i++) debug += ", " + debugString(val[i]); - return debug += "]", debug; + let debug = '['; + if (length > 0) { + debug += debugString(val[0]); + } + for(let i = 1; i < length; i++) { + debug += ', ' + debugString(val[i]); + } + debug += ']'; + return debug; } + // Test for built-in const builtInMatches = /\[object ([^\]]+)\]/.exec(toString.call(val)); let className; - if (!(builtInMatches && builtInMatches.length > 1)) return toString.call(val); - if (className = builtInMatches[1], "Object" == className) try { - return "Object(" + JSON.stringify(val) + ")"; - } catch (_) { - return "Object"; + if (builtInMatches && builtInMatches.length > 1) { + className = builtInMatches[1]; + } else { + // Failed to match the standard '[object ClassName]' + return toString.call(val); } - return val instanceof Error ? `${val.name}: ${val.message}\n${val.stack}` : className; + if (className == 'Object') { + // we're a user defined class or Object + // JSON.stringify avoids problems with cycles, and is generally much + // easier than looping through ownProperties of `val`. + try { + return 'Object(' + JSON.stringify(val) + ')'; + } catch (_) { + return 'Object'; + } + } + // errors + if (val instanceof Error) { + return `${val.name}: ${val.message}\n${val.stack}`; + } + // TODO we could test for more things here, like `Set`s and `Map`s. + return className; } function dropObject(idx) { - idx < 132 || (heap[idx] = heap_next, heap_next = idx); + if (idx < 132) return; + heap[idx] = heap_next; + heap_next = idx; } function getArrayU8FromWasm0(ptr, len) { - return ptr >>>= 0, getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); + ptr = ptr >>> 0; + return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); } let cachedDataViewMemory0 = null; - function getDataViewMemory0() { - return (null === cachedDataViewMemory0 || !0 === cachedDataViewMemory0.buffer.detached || void 0 === cachedDataViewMemory0.buffer.detached && cachedDataViewMemory0.buffer !== wasm.memory.buffer) && (cachedDataViewMemory0 = new DataView(wasm.memory.buffer)), - cachedDataViewMemory0; + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; } function getStringFromWasm0(ptr, len) { - return decodeText(ptr >>>= 0, len); + ptr = ptr >>> 0; + return decodeText(ptr, len); } let cachedUint8ArrayMemory0 = null; - function getUint8ArrayMemory0() { - return null !== cachedUint8ArrayMemory0 && 0 !== cachedUint8ArrayMemory0.byteLength || (cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer)), - cachedUint8ArrayMemory0; + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; } -function getObject(idx) { - return heap[idx]; -} +function getObject(idx) { return heap[idx]; } function handleError(f, args) { try { @@ -74,250 +116,414 @@ function handleError(f, args) { } } -let heap = new Array(128).fill(void 0); - -heap.push(void 0, null, !0, !1); +let heap = new Array(128).fill(undefined); +heap.push(undefined, null, true, false); let heap_next = heap.length; function isLikeNone(x) { - return null == x; + return x === undefined || x === null; } function passStringToWasm0(arg, malloc, realloc) { - if (void 0 === realloc) { - const buf = cachedTextEncoder.encode(arg), ptr = malloc(buf.length, 1) >>> 0; - return getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf), WASM_VECTOR_LEN = buf.length, - ptr; + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; } - let len = arg.length, ptr = malloc(len, 1) >>> 0; + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + const mem = getUint8ArrayMemory0(); + let offset = 0; - for (;offset < len; offset++) { + + for (; offset < len; offset++) { const code = arg.charCodeAt(offset); - if (code > 127) break; + if (code > 0x7F) break; mem[ptr + offset] = code; } if (offset !== len) { - 0 !== offset && (arg = arg.slice(offset)), ptr = realloc(ptr, len, len = offset + 3 * arg.length, 1) >>> 0; + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); - offset += cachedTextEncoder.encodeInto(arg, view).written, ptr = realloc(ptr, len, offset, 1) >>> 0; + const ret = cachedTextEncoder.encodeInto(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; } - return WASM_VECTOR_LEN = offset, ptr; + + WASM_VECTOR_LEN = offset; + return ptr; } function takeObject(idx) { const ret = getObject(idx); - return dropObject(idx), ret; + dropObject(idx); + return ret; } -let cachedTextDecoder = new TextDecoder("utf-8", { - ignoreBOM: !0, - fatal: !0 -}); - +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); cachedTextDecoder.decode(); - const MAX_SAFARI_DECODE_BYTES = 2146435072; - let numBytesDecoded = 0; - function decodeText(ptr, len) { - return numBytesDecoded += len, numBytesDecoded >= MAX_SAFARI_DECODE_BYTES && (cachedTextDecoder = new TextDecoder("utf-8", { - ignoreBOM: !0, - fatal: !0 - }), cachedTextDecoder.decode(), numBytesDecoded = len), cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); } -const cachedTextEncoder = new TextEncoder; +const cachedTextEncoder = new TextEncoder(); -"encodeInto" in cachedTextEncoder || (cachedTextEncoder.encodeInto = function(arg, view) { - const buf = cachedTextEncoder.encode(arg); - return view.set(buf), { - read: arg.length, - written: buf.length - }; -}); +if (!('encodeInto' in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + } +} let WASM_VECTOR_LEN = 0; +/** + * @param {any} val + * @returns {any} + */ export function analyze_page(val) { - return takeObject(wasm.analyze_page(addHeapObject(val))); + const ret = wasm.analyze_page(addHeapObject(val)); + return takeObject(ret); } +/** + * @param {any} val + * @param {any} options + * @returns {any} + */ export function analyze_page_with_options(val, options) { - return takeObject(wasm.analyze_page_with_options(addHeapObject(val), addHeapObject(options))); + const ret = wasm.analyze_page_with_options(addHeapObject(val), addHeapObject(options)); + return takeObject(ret); } +/** + * @param {any} _raw_elements + */ export function decide_and_act(_raw_elements) { wasm.decide_and_act(addHeapObject(_raw_elements)); } +/** + * Prune raw elements before sending to API + * This is a "dumb" filter that reduces payload size without leaking proprietary IP + * Filters out: tiny elements, invisible elements, non-interactive wrapper divs + * Amazon: 5000-6000 elements -> ~200-400 elements (~95% reduction) + * @param {any} val + * @returns {any} + */ export function prune_for_api(val) { - return takeObject(wasm.prune_for_api(addHeapObject(val))); + const ret = wasm.prune_for_api(addHeapObject(val)); + return takeObject(ret); } -const EXPECTED_RESPONSE_TYPES = new Set([ "basic", "cors", "default" ]); +const EXPECTED_RESPONSE_TYPES = new Set(['basic', 'cors', 'default']); async function __wbg_load(module, imports) { - if ("function" == typeof Response && module instanceof Response) { - if ("function" == typeof WebAssembly.instantiateStreaming) try { - return await WebAssembly.instantiateStreaming(module, imports); - } catch (e) { - if (!(module.ok && EXPECTED_RESPONSE_TYPES.has(module.type)) || "application/wasm" === module.headers.get("Content-Type")) throw e; + if (typeof Response === 'function' && module instanceof Response) { + if (typeof WebAssembly.instantiateStreaming === 'function') { + try { + return await WebAssembly.instantiateStreaming(module, imports); + } catch (e) { + const validResponse = module.ok && EXPECTED_RESPONSE_TYPES.has(module.type); + + if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { + throw e; + } + } } + const bytes = await module.arrayBuffer(); return await WebAssembly.instantiate(bytes, imports); - } - { + } else { const instance = await WebAssembly.instantiate(module, imports); - return instance instanceof WebAssembly.Instance ? { - instance: instance, - module: module - } : instance; + + if (instance instanceof WebAssembly.Instance) { + return { instance, module }; + } else { + return instance; + } } } function __wbg_get_imports() { - const imports = { - wbg: {} - }; - return imports.wbg.__wbg_Error_52673b7de5a0ca89 = function(arg0, arg1) { - return addHeapObject(Error(getStringFromWasm0(arg0, arg1))); - }, imports.wbg.__wbg_Number_2d1dcfcf4ec51736 = function(arg0) { - return Number(getObject(arg0)); - }, imports.wbg.__wbg___wbindgen_bigint_get_as_i64_6e32f5e6aff02e1d = function(arg0, arg1) { - const v = getObject(arg1), ret = "bigint" == typeof v ? v : void 0; - getDataViewMemory0().setBigInt64(arg0 + 8, isLikeNone(ret) ? BigInt(0) : ret, !0), - getDataViewMemory0().setInt32(arg0 + 0, !isLikeNone(ret), !0); - }, imports.wbg.__wbg___wbindgen_boolean_get_dea25b33882b895b = function(arg0) { - const v = getObject(arg0), ret = "boolean" == typeof v ? v : void 0; - return isLikeNone(ret) ? 16777215 : ret ? 1 : 0; - }, imports.wbg.__wbg___wbindgen_debug_string_adfb662ae34724b6 = function(arg0, arg1) { - const ptr1 = passStringToWasm0(debugString(getObject(arg1)), wasm.__wbindgen_export, wasm.__wbindgen_export2), len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4, len1, !0), getDataViewMemory0().setInt32(arg0 + 0, ptr1, !0); - }, imports.wbg.__wbg___wbindgen_in_0d3e1e8f0c669317 = function(arg0, arg1) { - return getObject(arg0) in getObject(arg1); - }, imports.wbg.__wbg___wbindgen_is_bigint_0e1a2e3f55cfae27 = function(arg0) { - return "bigint" == typeof getObject(arg0); - }, imports.wbg.__wbg___wbindgen_is_function_8d400b8b1af978cd = function(arg0) { - return "function" == typeof getObject(arg0); - }, imports.wbg.__wbg___wbindgen_is_object_ce774f3490692386 = function(arg0) { + const imports = {}; + imports.wbg = {}; + imports.wbg.__wbg_Error_52673b7de5a0ca89 = function(arg0, arg1) { + const ret = Error(getStringFromWasm0(arg0, arg1)); + return addHeapObject(ret); + }; + imports.wbg.__wbg_Number_2d1dcfcf4ec51736 = function(arg0) { + const ret = Number(getObject(arg0)); + return ret; + }; + imports.wbg.__wbg___wbindgen_bigint_get_as_i64_6e32f5e6aff02e1d = function(arg0, arg1) { + const v = getObject(arg1); + const ret = typeof(v) === 'bigint' ? v : undefined; + getDataViewMemory0().setBigInt64(arg0 + 8 * 1, isLikeNone(ret) ? BigInt(0) : ret, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); + }; + imports.wbg.__wbg___wbindgen_boolean_get_dea25b33882b895b = function(arg0) { + const v = getObject(arg0); + const ret = typeof(v) === 'boolean' ? v : undefined; + return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0; + }; + imports.wbg.__wbg___wbindgen_debug_string_adfb662ae34724b6 = function(arg0, arg1) { + const ret = debugString(getObject(arg1)); + const ptr1 = passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); + }; + imports.wbg.__wbg___wbindgen_in_0d3e1e8f0c669317 = function(arg0, arg1) { + const ret = getObject(arg0) in getObject(arg1); + return ret; + }; + imports.wbg.__wbg___wbindgen_is_bigint_0e1a2e3f55cfae27 = function(arg0) { + const ret = typeof(getObject(arg0)) === 'bigint'; + return ret; + }; + imports.wbg.__wbg___wbindgen_is_function_8d400b8b1af978cd = function(arg0) { + const ret = typeof(getObject(arg0)) === 'function'; + return ret; + }; + imports.wbg.__wbg___wbindgen_is_object_ce774f3490692386 = function(arg0) { const val = getObject(arg0); - return "object" == typeof val && null !== val; - }, imports.wbg.__wbg___wbindgen_is_undefined_f6b95eab589e0269 = function(arg0) { - return void 0 === getObject(arg0); - }, imports.wbg.__wbg___wbindgen_jsval_eq_b6101cc9cef1fe36 = function(arg0, arg1) { - return getObject(arg0) === getObject(arg1); - }, imports.wbg.__wbg___wbindgen_jsval_loose_eq_766057600fdd1b0d = function(arg0, arg1) { - return getObject(arg0) == getObject(arg1); - }, imports.wbg.__wbg___wbindgen_number_get_9619185a74197f95 = function(arg0, arg1) { - const obj = getObject(arg1), ret = "number" == typeof obj ? obj : void 0; - getDataViewMemory0().setFloat64(arg0 + 8, isLikeNone(ret) ? 0 : ret, !0), getDataViewMemory0().setInt32(arg0 + 0, !isLikeNone(ret), !0); - }, imports.wbg.__wbg___wbindgen_string_get_a2a31e16edf96e42 = function(arg0, arg1) { - const obj = getObject(arg1), ret = "string" == typeof obj ? obj : void 0; - var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2), len1 = WASM_VECTOR_LEN; - getDataViewMemory0().setInt32(arg0 + 4, len1, !0), getDataViewMemory0().setInt32(arg0 + 0, ptr1, !0); - }, imports.wbg.__wbg___wbindgen_throw_dd24417ed36fc46e = function(arg0, arg1) { + const ret = typeof(val) === 'object' && val !== null; + return ret; + }; + imports.wbg.__wbg___wbindgen_is_undefined_f6b95eab589e0269 = function(arg0) { + const ret = getObject(arg0) === undefined; + return ret; + }; + imports.wbg.__wbg___wbindgen_jsval_eq_b6101cc9cef1fe36 = function(arg0, arg1) { + const ret = getObject(arg0) === getObject(arg1); + return ret; + }; + imports.wbg.__wbg___wbindgen_jsval_loose_eq_766057600fdd1b0d = function(arg0, arg1) { + const ret = getObject(arg0) == getObject(arg1); + return ret; + }; + imports.wbg.__wbg___wbindgen_number_get_9619185a74197f95 = function(arg0, arg1) { + const obj = getObject(arg1); + const ret = typeof(obj) === 'number' ? obj : undefined; + getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true); + }; + imports.wbg.__wbg___wbindgen_string_get_a2a31e16edf96e42 = function(arg0, arg1) { + const obj = getObject(arg1); + const ret = typeof(obj) === 'string' ? obj : undefined; + var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2); + var len1 = WASM_VECTOR_LEN; + getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true); + getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true); + }; + imports.wbg.__wbg___wbindgen_throw_dd24417ed36fc46e = function(arg0, arg1) { throw new Error(getStringFromWasm0(arg0, arg1)); - }, imports.wbg.__wbg_call_abb4ff46ce38be40 = function() { - return handleError(function(arg0, arg1) { - return addHeapObject(getObject(arg0).call(getObject(arg1))); - }, arguments); - }, imports.wbg.__wbg_done_62ea16af4ce34b24 = function(arg0) { - return getObject(arg0).done; - }, imports.wbg.__wbg_error_7bc7d576a6aaf855 = function(arg0) {}, imports.wbg.__wbg_get_6b7bd52aca3f9671 = function(arg0, arg1) { - return addHeapObject(getObject(arg0)[arg1 >>> 0]); - }, imports.wbg.__wbg_get_af9dab7e9603ea93 = function() { - return handleError(function(arg0, arg1) { - return addHeapObject(Reflect.get(getObject(arg0), getObject(arg1))); - }, arguments); - }, imports.wbg.__wbg_get_with_ref_key_1dc361bd10053bfe = function(arg0, arg1) { - return addHeapObject(getObject(arg0)[getObject(arg1)]); - }, imports.wbg.__wbg_instanceof_ArrayBuffer_f3320d2419cd0355 = function(arg0) { + }; + imports.wbg.__wbg_call_abb4ff46ce38be40 = function() { return handleError(function (arg0, arg1) { + const ret = getObject(arg0).call(getObject(arg1)); + return addHeapObject(ret); + }, arguments) }; + imports.wbg.__wbg_done_62ea16af4ce34b24 = function(arg0) { + const ret = getObject(arg0).done; + return ret; + }; + imports.wbg.__wbg_error_7bc7d576a6aaf855 = function(arg0) { + console.error(getObject(arg0)); + }; + imports.wbg.__wbg_get_6b7bd52aca3f9671 = function(arg0, arg1) { + const ret = getObject(arg0)[arg1 >>> 0]; + return addHeapObject(ret); + }; + imports.wbg.__wbg_get_af9dab7e9603ea93 = function() { return handleError(function (arg0, arg1) { + const ret = Reflect.get(getObject(arg0), getObject(arg1)); + return addHeapObject(ret); + }, arguments) }; + imports.wbg.__wbg_get_with_ref_key_1dc361bd10053bfe = function(arg0, arg1) { + const ret = getObject(arg0)[getObject(arg1)]; + return addHeapObject(ret); + }; + imports.wbg.__wbg_instanceof_ArrayBuffer_f3320d2419cd0355 = function(arg0) { let result; try { result = getObject(arg0) instanceof ArrayBuffer; } catch (_) { - result = !1; + result = false; } - return result; - }, imports.wbg.__wbg_instanceof_Uint8Array_da54ccc9d3e09434 = function(arg0) { + const ret = result; + return ret; + }; + imports.wbg.__wbg_instanceof_Uint8Array_da54ccc9d3e09434 = function(arg0) { let result; try { result = getObject(arg0) instanceof Uint8Array; } catch (_) { - result = !1; + result = false; } - return result; - }, imports.wbg.__wbg_isArray_51fd9e6422c0a395 = function(arg0) { - return Array.isArray(getObject(arg0)); - }, imports.wbg.__wbg_isSafeInteger_ae7d3f054d55fa16 = function(arg0) { - return Number.isSafeInteger(getObject(arg0)); - }, imports.wbg.__wbg_iterator_27b7c8b35ab3e86b = function() { - return addHeapObject(Symbol.iterator); - }, imports.wbg.__wbg_js_click_element_2fe1e774f3d232c7 = function(arg0) { + const ret = result; + return ret; + }; + imports.wbg.__wbg_isArray_51fd9e6422c0a395 = function(arg0) { + const ret = Array.isArray(getObject(arg0)); + return ret; + }; + imports.wbg.__wbg_isSafeInteger_ae7d3f054d55fa16 = function(arg0) { + const ret = Number.isSafeInteger(getObject(arg0)); + return ret; + }; + imports.wbg.__wbg_iterator_27b7c8b35ab3e86b = function() { + const ret = Symbol.iterator; + return addHeapObject(ret); + }; + imports.wbg.__wbg_js_click_element_2fe1e774f3d232c7 = function(arg0) { js_click_element(arg0); - }, imports.wbg.__wbg_length_22ac23eaec9d8053 = function(arg0) { - return getObject(arg0).length; - }, imports.wbg.__wbg_length_d45040a40c570362 = function(arg0) { - return getObject(arg0).length; - }, imports.wbg.__wbg_new_1ba21ce319a06297 = function() { - return addHeapObject(new Object); - }, imports.wbg.__wbg_new_25f239778d6112b9 = function() { - return addHeapObject(new Array); - }, imports.wbg.__wbg_new_6421f6084cc5bc5a = function(arg0) { - return addHeapObject(new Uint8Array(getObject(arg0))); - }, imports.wbg.__wbg_next_138a17bbf04e926c = function(arg0) { - return addHeapObject(getObject(arg0).next); - }, imports.wbg.__wbg_next_3cfe5c0fe2a4cc53 = function() { - return handleError(function(arg0) { - return addHeapObject(getObject(arg0).next()); - }, arguments); - }, imports.wbg.__wbg_prototypesetcall_dfe9b766cdc1f1fd = function(arg0, arg1, arg2) { + }; + imports.wbg.__wbg_length_22ac23eaec9d8053 = function(arg0) { + const ret = getObject(arg0).length; + return ret; + }; + imports.wbg.__wbg_length_d45040a40c570362 = function(arg0) { + const ret = getObject(arg0).length; + return ret; + }; + imports.wbg.__wbg_new_1ba21ce319a06297 = function() { + const ret = new Object(); + return addHeapObject(ret); + }; + imports.wbg.__wbg_new_25f239778d6112b9 = function() { + const ret = new Array(); + return addHeapObject(ret); + }; + imports.wbg.__wbg_new_6421f6084cc5bc5a = function(arg0) { + const ret = new Uint8Array(getObject(arg0)); + return addHeapObject(ret); + }; + imports.wbg.__wbg_next_138a17bbf04e926c = function(arg0) { + const ret = getObject(arg0).next; + return addHeapObject(ret); + }; + imports.wbg.__wbg_next_3cfe5c0fe2a4cc53 = function() { return handleError(function (arg0) { + const ret = getObject(arg0).next(); + return addHeapObject(ret); + }, arguments) }; + imports.wbg.__wbg_prototypesetcall_dfe9b766cdc1f1fd = function(arg0, arg1, arg2) { Uint8Array.prototype.set.call(getArrayU8FromWasm0(arg0, arg1), getObject(arg2)); - }, imports.wbg.__wbg_set_3f1d0b984ed272ed = function(arg0, arg1, arg2) { + }; + imports.wbg.__wbg_set_3f1d0b984ed272ed = function(arg0, arg1, arg2) { getObject(arg0)[takeObject(arg1)] = takeObject(arg2); - }, imports.wbg.__wbg_set_7df433eea03a5c14 = function(arg0, arg1, arg2) { + }; + imports.wbg.__wbg_set_7df433eea03a5c14 = function(arg0, arg1, arg2) { getObject(arg0)[arg1 >>> 0] = takeObject(arg2); - }, imports.wbg.__wbg_value_57b7b035e117f7ee = function(arg0) { - return addHeapObject(getObject(arg0).value); - }, imports.wbg.__wbindgen_cast_2241b6af4c4b2941 = function(arg0, arg1) { - return addHeapObject(getStringFromWasm0(arg0, arg1)); - }, imports.wbg.__wbindgen_cast_4625c577ab2ec9ee = function(arg0) { - return addHeapObject(BigInt.asUintN(64, arg0)); - }, imports.wbg.__wbindgen_cast_d6cd19b81560fd6e = function(arg0) { - return addHeapObject(arg0); - }, imports.wbg.__wbindgen_object_clone_ref = function(arg0) { - return addHeapObject(getObject(arg0)); - }, imports.wbg.__wbindgen_object_drop_ref = function(arg0) { + }; + imports.wbg.__wbg_value_57b7b035e117f7ee = function(arg0) { + const ret = getObject(arg0).value; + return addHeapObject(ret); + }; + imports.wbg.__wbindgen_cast_2241b6af4c4b2941 = function(arg0, arg1) { + // Cast intrinsic for `Ref(String) -> Externref`. + const ret = getStringFromWasm0(arg0, arg1); + return addHeapObject(ret); + }; + imports.wbg.__wbindgen_cast_4625c577ab2ec9ee = function(arg0) { + // Cast intrinsic for `U64 -> Externref`. + const ret = BigInt.asUintN(64, arg0); + return addHeapObject(ret); + }; + imports.wbg.__wbindgen_cast_d6cd19b81560fd6e = function(arg0) { + // Cast intrinsic for `F64 -> Externref`. + const ret = arg0; + return addHeapObject(ret); + }; + imports.wbg.__wbindgen_object_clone_ref = function(arg0) { + const ret = getObject(arg0); + return addHeapObject(ret); + }; + imports.wbg.__wbindgen_object_drop_ref = function(arg0) { takeObject(arg0); - }, imports; + }; + + return imports; } function __wbg_finalize_init(instance, module) { - return wasm = instance.exports, __wbg_init.__wbindgen_wasm_module = module, cachedDataViewMemory0 = null, - cachedUint8ArrayMemory0 = null, wasm; + wasm = instance.exports; + __wbg_init.__wbindgen_wasm_module = module; + cachedDataViewMemory0 = null; + cachedUint8ArrayMemory0 = null; + + + + return wasm; } function initSync(module) { - if (void 0 !== wasm) return wasm; - void 0 !== module && Object.getPrototypeOf(module) === Object.prototype && ({module: module} = module); + if (wasm !== undefined) return wasm; + + + if (typeof module !== 'undefined') { + if (Object.getPrototypeOf(module) === Object.prototype) { + ({module} = module) + } else { + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') + } + } + const imports = __wbg_get_imports(); - module instanceof WebAssembly.Module || (module = new WebAssembly.Module(module)); - return __wbg_finalize_init(new WebAssembly.Instance(module, imports), module); + if (!(module instanceof WebAssembly.Module)) { + module = new WebAssembly.Module(module); + } + const instance = new WebAssembly.Instance(module, imports); + return __wbg_finalize_init(instance, module); } async function __wbg_init(module_or_path) { - if (void 0 !== wasm) return wasm; - void 0 !== module_or_path && Object.getPrototypeOf(module_or_path) === Object.prototype && ({module_or_path: module_or_path} = module_or_path), - void 0 === module_or_path && (module_or_path = new URL("sentience_core_bg.wasm", import.meta.url)); + if (wasm !== undefined) return wasm; + + + if (typeof module_or_path !== 'undefined') { + if (Object.getPrototypeOf(module_or_path) === Object.prototype) { + ({module_or_path} = module_or_path) + } else { + console.warn('using deprecated parameters for the initialization function; pass a single object instead') + } + } + + if (typeof module_or_path === 'undefined') { + module_or_path = new URL('sentience_core_bg.wasm', import.meta.url); + } const imports = __wbg_get_imports(); - ("string" == typeof module_or_path || "function" == typeof Request && module_or_path instanceof Request || "function" == typeof URL && module_or_path instanceof URL) && (module_or_path = fetch(module_or_path)); - const {instance: instance, module: module} = await __wbg_load(await module_or_path, imports); + + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { + module_or_path = fetch(module_or_path); + } + + const { instance, module } = await __wbg_load(await module_or_path, imports); + return __wbg_finalize_init(instance, module); } export { initSync }; - -export default __wbg_init; \ No newline at end of file +export default __wbg_init; diff --git a/sentience/extension/pkg/sentience_core_bg.wasm b/sentience/extension/pkg/sentience_core_bg.wasm index 259298c..b001b68 100644 Binary files a/sentience/extension/pkg/sentience_core_bg.wasm and b/sentience/extension/pkg/sentience_core_bg.wasm differ diff --git a/sentience/models.py b/sentience/models.py index 985a264..c286b85 100644 --- a/sentience/models.py +++ b/sentience/models.py @@ -54,6 +54,21 @@ class Element(BaseModel): # Diff status for frontend Diff Overlay feature diff_status: Literal["ADDED", "REMOVED", "MODIFIED", "MOVED"] | None = None + # Phase 1: Ordinal support fields for position-based selection + center_x: float | None = None # X coordinate of element center (viewport coords) + center_y: float | None = None # Y coordinate of element center (viewport coords) + doc_y: float | None = None # Y coordinate in document (center_y + scroll_y) + group_key: str | None = None # Geometric bucket key for ordinal grouping + group_index: int | None = None # Position within group (0-indexed, sorted by doc_y) + + # Hyperlink URL (for link elements) + href: str | None = None + + # Phase 3.2: Pre-computed dominant group membership (uses fuzzy matching) + # This field is computed by the gateway so downstream consumers don't need to + # implement fuzzy matching logic themselves. + in_dominant_group: bool | None = None + class Snapshot(BaseModel): """Snapshot response from extension""" @@ -67,6 +82,8 @@ class Snapshot(BaseModel): screenshot_format: Literal["png", "jpeg"] | None = None error: str | None = None requires_license: bool | None = None + # Phase 2: Dominant group key for ordinal selection + dominant_group_key: str | None = None # The most common group_key (main content group) def save(self, filepath: str) -> None: """Save snapshot as JSON file""" diff --git a/sentience/ordinal.py b/sentience/ordinal.py new file mode 100644 index 0000000..ee66a37 --- /dev/null +++ b/sentience/ordinal.py @@ -0,0 +1,280 @@ +""" +Phase 3: Ordinal Intent Detection for Semantic Search + +This module provides functions to detect ordinal intent in natural language goals +and select elements based on their position within groups. + +Ordinal operators supported: +- Position-based: "first", "second", "third", "1st", "2nd", "3rd", etc. +- Relative: "top", "bottom", "last", "next", "previous" +- Numeric: "#1", "#2", "number 1", "item 3" + +Example usage: + from sentience.ordinal import detect_ordinal_intent, select_by_ordinal + + intent = detect_ordinal_intent("click the first search result") + # OrdinalIntent(kind='nth', n=1, detected=True) + + element = select_by_ordinal(elements, dominant_group_key, intent) +""" + +import re +from dataclasses import dataclass +from typing import Literal + +from sentience.models import Element + + +@dataclass +class OrdinalIntent: + """Detected ordinal intent from a goal string.""" + + detected: bool + kind: Literal["first", "last", "nth", "top_k", "next", "previous"] | None = None + n: int | None = None # For "nth" kind: 1-indexed position (1=first, 2=second) + k: int | None = None # For "top_k" kind: number of items + + +# Ordinal word to number mapping +ORDINAL_WORDS = { + "first": 1, + "second": 2, + "third": 3, + "fourth": 4, + "fifth": 5, + "sixth": 6, + "seventh": 7, + "eighth": 8, + "ninth": 9, + "tenth": 10, + "1st": 1, + "2nd": 2, + "3rd": 3, + "4th": 4, + "5th": 5, + "6th": 6, + "7th": 7, + "8th": 8, + "9th": 9, + "10th": 10, +} + +# Patterns for detecting ordinal intent +ORDINAL_PATTERNS = [ + # "first", "second", etc. + ( + r"\b(first|second|third|fourth|fifth|sixth|seventh|eighth|ninth|tenth)\b", + "ordinal_word", + ), + # "1st", "2nd", "3rd", etc. + (r"\b(\d+)(st|nd|rd|th)\b", "ordinal_suffix"), + # "#1", "#2", etc. + (r"#(\d+)\b", "hash_number"), + # "number 1", "item 3", "result 5" + (r"\b(?:number|item|result|option|choice)\s*(\d+)\b", "labeled_number"), + # "top" (implies first/best) + (r"\btop\b(?!\s*\d)", "top"), + # "top 3", "top 5" + (r"\btop\s+(\d+)\b", "top_k"), + # "last", "final", "bottom" + (r"\b(last|final|bottom)\b", "last"), + # "next", "following" + (r"\b(next|following)\b", "next"), + # "previous", "preceding", "prior" + (r"\b(previous|preceding|prior)\b", "previous"), +] + + +def detect_ordinal_intent(goal: str) -> OrdinalIntent: + """ + Detect ordinal intent from a goal string. + + Args: + goal: Natural language goal (e.g., "click the first search result") + + Returns: + OrdinalIntent with detected=True if ordinal intent found, False otherwise. + + Examples: + >>> detect_ordinal_intent("click the first item") + OrdinalIntent(detected=True, kind='nth', n=1) + + >>> detect_ordinal_intent("select the 3rd option") + OrdinalIntent(detected=True, kind='nth', n=3) + + >>> detect_ordinal_intent("show top 5 results") + OrdinalIntent(detected=True, kind='top_k', k=5) + + >>> detect_ordinal_intent("click the last button") + OrdinalIntent(detected=True, kind='last') + + >>> detect_ordinal_intent("find the submit button") + OrdinalIntent(detected=False) + """ + goal_lower = goal.lower() + + for pattern, pattern_type in ORDINAL_PATTERNS: + match = re.search(pattern, goal_lower, re.IGNORECASE) + if match: + if pattern_type == "ordinal_word": + word = match.group(1).lower() + n = ORDINAL_WORDS.get(word) + if n: + return OrdinalIntent(detected=True, kind="nth", n=n) + + elif pattern_type == "ordinal_suffix": + n = int(match.group(1)) + return OrdinalIntent(detected=True, kind="nth", n=n) + + elif pattern_type == "hash_number": + n = int(match.group(1)) + return OrdinalIntent(detected=True, kind="nth", n=n) + + elif pattern_type == "labeled_number": + n = int(match.group(1)) + return OrdinalIntent(detected=True, kind="nth", n=n) + + elif pattern_type == "top": + # "top" without a number means "first/best" + return OrdinalIntent(detected=True, kind="first") + + elif pattern_type == "top_k": + k = int(match.group(1)) + return OrdinalIntent(detected=True, kind="top_k", k=k) + + elif pattern_type == "last": + return OrdinalIntent(detected=True, kind="last") + + elif pattern_type == "next": + return OrdinalIntent(detected=True, kind="next") + + elif pattern_type == "previous": + return OrdinalIntent(detected=True, kind="previous") + + return OrdinalIntent(detected=False) + + +def select_by_ordinal( + elements: list[Element], + dominant_group_key: str | None, + intent: OrdinalIntent, + current_element_id: int | None = None, +) -> Element | list[Element] | None: + """ + Select element(s) from a list based on ordinal intent. + + Uses the dominant_group_key to filter to the "main content" group, + then selects by group_index based on the ordinal intent. + + Args: + elements: List of elements with group_key and group_index populated + dominant_group_key: The most common group key (main content group) + intent: Detected ordinal intent + current_element_id: Current element ID (for next/previous navigation) + + Returns: + Single Element for nth/first/last, list of Elements for top_k, + or None if no matching element found. + + Examples: + >>> intent = OrdinalIntent(detected=True, kind='nth', n=1) + >>> element = select_by_ordinal(elements, "x5-w2-h1", intent) + # Returns element with group_key="x5-w2-h1" and group_index=0 + """ + if not intent.detected: + return None + + # Filter to dominant group if available + if dominant_group_key: + group_elements = [e for e in elements if e.group_key == dominant_group_key] + else: + # Fallback: use all elements with group_index + group_elements = [e for e in elements if e.group_index is not None] + + if not group_elements: + return None + + # Sort by group_index to ensure correct ordering + group_elements.sort(key=lambda e: e.group_index if e.group_index is not None else 0) + + if intent.kind == "first" or (intent.kind == "nth" and intent.n == 1): + # First element (group_index=0) + return group_elements[0] if group_elements else None + + elif intent.kind == "nth" and intent.n is not None: + # Nth element (1-indexed, so n=2 means group_index=1) + target_index = intent.n - 1 + if 0 <= target_index < len(group_elements): + return group_elements[target_index] + return None + + elif intent.kind == "last": + # Last element + return group_elements[-1] if group_elements else None + + elif intent.kind == "top_k" and intent.k is not None: + # Top K elements + return group_elements[: intent.k] + + elif intent.kind == "next" and current_element_id is not None: + # Next element after current + for i, elem in enumerate(group_elements): + if elem.id == current_element_id and i + 1 < len(group_elements): + return group_elements[i + 1] + return None + + elif intent.kind == "previous" and current_element_id is not None: + # Previous element before current + for i, elem in enumerate(group_elements): + if elem.id == current_element_id and i > 0: + return group_elements[i - 1] + return None + + return None + + +def boost_ordinal_elements( + elements: list[Element], + dominant_group_key: str | None, + intent: OrdinalIntent, + boost_factor: int = 10000, +) -> list[Element]: + """ + Boost the importance of elements matching ordinal intent. + + This is useful for integrating ordinal selection with existing + importance-based ranking. Elements matching the ordinal intent + get a significant importance boost. + + Args: + elements: List of elements (not modified) + dominant_group_key: The most common group key + intent: Detected ordinal intent + boost_factor: Amount to add to importance (default: 10000) + + Returns: + A new list with copies of elements, with boosted importance for matches. + """ + if not intent.detected or not dominant_group_key: + return [e.model_copy() for e in elements] + + target = select_by_ordinal(elements, dominant_group_key, intent) + + if target is None: + return [e.model_copy() for e in elements] + + # Handle single element or list + if isinstance(target, list): + target_ids = {e.id for e in target} + else: + target_ids = {target.id} + + # Create copies and boost matching elements + result = [] + for elem in elements: + copy = elem.model_copy() + if copy.id in target_ids: + copy.importance = (copy.importance or 0) + boost_factor + result.append(copy) + + return result diff --git a/sentience/snapshot.py b/sentience/snapshot.py index 3366141..5ebc412 100644 --- a/sentience/snapshot.py +++ b/sentience/snapshot.py @@ -132,6 +132,8 @@ def _merge_api_result_with_local( "screenshot": raw_result.get("screenshot"), # Keep local screenshot "screenshot_format": raw_result.get("screenshot_format"), "error": api_result.get("error"), + # Phase 2: Ordinal support - dominant group key from Gateway + "dominant_group_key": api_result.get("dominant_group_key"), } diff --git a/tests/test_ordinal.py b/tests/test_ordinal.py new file mode 100644 index 0000000..5474851 --- /dev/null +++ b/tests/test_ordinal.py @@ -0,0 +1,442 @@ +""" +Unit tests for ordinal intent detection and selection. + +Tests the detect_ordinal_intent, select_by_ordinal, and boost_ordinal_elements functions. +""" + +import pytest + +from sentience.models import BBox, Element, VisualCues +from sentience.ordinal import ( + OrdinalIntent, + boost_ordinal_elements, + detect_ordinal_intent, + select_by_ordinal, +) + + +class TestDetectOrdinalIntent: + """Tests for detect_ordinal_intent function.""" + + # Ordinal words + def test_first(self): + result = detect_ordinal_intent("Click the first result") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 1 + + def test_second(self): + result = detect_ordinal_intent("Select the second item") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 2 + + def test_third(self): + result = detect_ordinal_intent("Click the third option") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 3 + + def test_fourth(self): + result = detect_ordinal_intent("Choose the fourth link") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 4 + + def test_fifth(self): + result = detect_ordinal_intent("Click the fifth button") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 5 + + def test_tenth(self): + result = detect_ordinal_intent("Select the tenth item") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 10 + + # Ordinal suffixes + def test_1st(self): + result = detect_ordinal_intent("Click the 1st result") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 1 + + def test_2nd(self): + result = detect_ordinal_intent("Select the 2nd item") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 2 + + def test_3rd(self): + result = detect_ordinal_intent("Click the 3rd option") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 3 + + def test_4th(self): + result = detect_ordinal_intent("Choose the 4th link") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 4 + + def test_21st(self): + result = detect_ordinal_intent("Select the 21st item") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 21 + + def test_22nd(self): + result = detect_ordinal_intent("Click the 22nd result") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 22 + + def test_33rd(self): + result = detect_ordinal_intent("Choose the 33rd option") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 33 + + def test_100th(self): + result = detect_ordinal_intent("Select the 100th item") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 100 + + # Hash numbers + def test_hash_1(self): + result = detect_ordinal_intent("Click item #1") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 1 + + def test_hash_3(self): + result = detect_ordinal_intent("Select result #3") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 3 + + def test_hash_10(self): + result = detect_ordinal_intent("Choose option #10") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 10 + + # Labeled numbers + def test_item_number(self): + result = detect_ordinal_intent("Click item 5") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 5 + + def test_result_number(self): + result = detect_ordinal_intent("Select result 3") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 3 + + def test_option_number(self): + result = detect_ordinal_intent("Choose option 2") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 2 + + def test_number_word(self): + result = detect_ordinal_intent("Click number 4") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 4 + + def test_choice_number(self): + result = detect_ordinal_intent("Select choice 1") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 1 + + # Top/first keywords + def test_top(self): + result = detect_ordinal_intent("Click the top result") + assert result.detected is True + assert result.kind == "first" + + def test_top_case_insensitive(self): + result = detect_ordinal_intent("Click the TOP result") + assert result.detected is True + assert result.kind == "first" + + # Top K + def test_top_3(self): + result = detect_ordinal_intent("Select the top 3 items") + assert result.detected is True + assert result.kind == "top_k" + assert result.k == 3 + + def test_top_5(self): + result = detect_ordinal_intent("View top 5 results") + assert result.detected is True + assert result.kind == "top_k" + assert result.k == 5 + + def test_top_10(self): + result = detect_ordinal_intent("Show top 10 products") + assert result.detected is True + assert result.kind == "top_k" + assert result.k == 10 + + # Last keywords + def test_last(self): + result = detect_ordinal_intent("Click the last item") + assert result.detected is True + assert result.kind == "last" + + def test_final(self): + result = detect_ordinal_intent("Select the final option") + assert result.detected is True + assert result.kind == "last" + + def test_bottom(self): + result = detect_ordinal_intent("Click the bottom result") + assert result.detected is True + assert result.kind == "last" + + # Next keywords + def test_next(self): + result = detect_ordinal_intent("Click the next button") + assert result.detected is True + assert result.kind == "next" + + def test_following(self): + result = detect_ordinal_intent("Go to the following item") + assert result.detected is True + assert result.kind == "next" + + # Previous keywords + def test_previous(self): + result = detect_ordinal_intent("Click the previous button") + assert result.detected is True + assert result.kind == "previous" + + def test_preceding(self): + result = detect_ordinal_intent("Go to the preceding item") + assert result.detected is True + assert result.kind == "previous" + + def test_prior(self): + result = detect_ordinal_intent("Select the prior option") + assert result.detected is True + assert result.kind == "previous" + + # No ordinal intent + def test_no_ordinal_click_button(self): + result = detect_ordinal_intent("Click the submit button") + assert result.detected is False + assert result.kind is None + + def test_no_ordinal_search(self): + result = detect_ordinal_intent("Search for laptops") + assert result.detected is False + + def test_no_ordinal_type(self): + result = detect_ordinal_intent("Type hello in the input") + assert result.detected is False + + def test_empty_string(self): + result = detect_ordinal_intent("") + assert result.detected is False + + # Case insensitivity + def test_case_insensitive_first(self): + result = detect_ordinal_intent("Click the FIRST result") + assert result.detected is True + assert result.kind == "nth" + assert result.n == 1 + + def test_case_insensitive_last(self): + result = detect_ordinal_intent("Select the LAST item") + assert result.detected is True + assert result.kind == "last" + + +def _make_element( + id: int, + text: str, + group_key: str | None = None, + group_index: int | None = None, + importance: int = 100, +) -> Element: + """Helper to create test elements.""" + return Element( + id=id, + role="button", + text=text, + importance=importance, + bbox=BBox(x=0, y=id * 50, width=100, height=40), + visual_cues=VisualCues(is_primary=False, background_color_name=None, is_clickable=True), + in_viewport=True, + is_occluded=False, + z_index=0, + group_key=group_key, + group_index=group_index, + ) + + +class TestSelectByOrdinal: + """Tests for select_by_ordinal function.""" + + @pytest.fixture + def elements(self) -> list[Element]: + """Create a list of test elements in the dominant group.""" + return [ + _make_element(1, "Item A", "x100-w200-h40", 0), + _make_element(2, "Item B", "x100-w200-h40", 1), + _make_element(3, "Item C", "x100-w200-h40", 2), + _make_element(4, "Item D", "x100-w200-h40", 3), + _make_element(5, "Item E", "x100-w200-h40", 4), + _make_element(6, "Other", "x500-w100-h30", 0), # Different group + ] + + def test_select_first(self, elements): + intent = OrdinalIntent(detected=True, kind="first") + result = select_by_ordinal(elements, "x100-w200-h40", intent) + assert result is not None + assert result.id == 1 + assert result.text == "Item A" + + def test_select_nth_2(self, elements): + intent = OrdinalIntent(detected=True, kind="nth", n=2) + result = select_by_ordinal(elements, "x100-w200-h40", intent) + assert result is not None + assert result.id == 2 + assert result.text == "Item B" + + def test_select_nth_5(self, elements): + intent = OrdinalIntent(detected=True, kind="nth", n=5) + result = select_by_ordinal(elements, "x100-w200-h40", intent) + assert result is not None + assert result.id == 5 + assert result.text == "Item E" + + def test_select_last(self, elements): + intent = OrdinalIntent(detected=True, kind="last") + result = select_by_ordinal(elements, "x100-w200-h40", intent) + assert result is not None + assert result.id == 5 + assert result.text == "Item E" + + def test_select_top_k(self, elements): + intent = OrdinalIntent(detected=True, kind="top_k", k=3) + result = select_by_ordinal(elements, "x100-w200-h40", intent) + assert isinstance(result, list) + assert len(result) == 3 + assert [e.id for e in result] == [1, 2, 3] + + def test_select_out_of_bounds(self, elements): + intent = OrdinalIntent(detected=True, kind="nth", n=100) + result = select_by_ordinal(elements, "x100-w200-h40", intent) + assert result is None + + def test_select_no_dominant_group(self, elements): + intent = OrdinalIntent(detected=True, kind="first") + result = select_by_ordinal(elements, None, intent) + # Should fall back to all elements sorted by group_index + assert result is not None + + def test_select_not_detected(self, elements): + intent = OrdinalIntent(detected=False) + result = select_by_ordinal(elements, "x100-w200-h40", intent) + assert result is None + + +class TestBoostOrdinalElements: + """Tests for boost_ordinal_elements function.""" + + @pytest.fixture + def elements(self) -> list[Element]: + """Create a list of test elements.""" + return [ + _make_element(1, "Item A", "x100-w200-h40", 0, importance=100), + _make_element(2, "Item B", "x100-w200-h40", 1, importance=90), + _make_element(3, "Item C", "x100-w200-h40", 2, importance=80), + _make_element(4, "Item D", "x100-w200-h40", 3, importance=70), + _make_element(5, "Other", "x500-w100-h30", 0, importance=200), + ] + + def test_boost_first(self, elements): + intent = OrdinalIntent(detected=True, kind="first") + result = boost_ordinal_elements(elements, "x100-w200-h40", intent, boost_factor=10000) + + # First element should be boosted + boosted = [e for e in result if e.id == 1][0] + assert boosted.importance == 100 + 10000 + + # Other elements unchanged + other = [e for e in result if e.id == 2][0] + assert other.importance == 90 + + def test_boost_nth(self, elements): + intent = OrdinalIntent(detected=True, kind="nth", n=3) + result = boost_ordinal_elements(elements, "x100-w200-h40", intent, boost_factor=5000) + + # Third element should be boosted + boosted = [e for e in result if e.id == 3][0] + assert boosted.importance == 80 + 5000 + + def test_boost_last(self, elements): + intent = OrdinalIntent(detected=True, kind="last") + result = boost_ordinal_elements(elements, "x100-w200-h40", intent, boost_factor=10000) + + # Last element in dominant group should be boosted (id=4, group_index=3) + boosted = [e for e in result if e.id == 4][0] + assert boosted.importance == 70 + 10000 + + def test_boost_top_k(self, elements): + intent = OrdinalIntent(detected=True, kind="top_k", k=2) + result = boost_ordinal_elements(elements, "x100-w200-h40", intent, boost_factor=10000) + + # First two elements should be boosted + first = [e for e in result if e.id == 1][0] + second = [e for e in result if e.id == 2][0] + third = [e for e in result if e.id == 3][0] + + assert first.importance == 100 + 10000 + assert second.importance == 90 + 10000 + assert third.importance == 80 # Not boosted + + def test_boost_not_detected(self, elements): + intent = OrdinalIntent(detected=False) + result = boost_ordinal_elements(elements, "x100-w200-h40", intent) + + # No elements should be boosted + for orig, boosted in zip(elements, result): + assert orig.importance == boosted.importance + + def test_boost_returns_copy(self, elements): + intent = OrdinalIntent(detected=True, kind="first") + result = boost_ordinal_elements(elements, "x100-w200-h40", intent) + + # Original elements should not be modified + assert elements[0].importance == 100 + + +class TestOrdinalIntent: + """Tests for OrdinalIntent dataclass.""" + + def test_default_values(self): + intent = OrdinalIntent(detected=False) + assert intent.detected is False + assert intent.kind is None + assert intent.n is None + assert intent.k is None + + def test_with_nth(self): + intent = OrdinalIntent(detected=True, kind="nth", n=5) + assert intent.detected is True + assert intent.kind == "nth" + assert intent.n == 5 + + def test_with_top_k(self): + intent = OrdinalIntent(detected=True, kind="top_k", k=3) + assert intent.detected is True + assert intent.kind == "top_k" + assert intent.k == 3 diff --git a/tests/test_video_recording.py b/tests/test_video_recording.py index 0b49553..9a069da 100644 --- a/tests/test_video_recording.py +++ b/tests/test_video_recording.py @@ -22,9 +22,10 @@ def test_video_recording_basic(): try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("domcontentloaded") - + # Small delay to ensure page is fully loaded and video recording is stable import time + time.sleep(0.5) video_path = browser.close()