From 08996d75fc8665e1e63650475f4a231ee35600c0 Mon Sep 17 00:00:00 2001 From: rcholic Date: Mon, 29 Dec 2025 17:38:44 -0800 Subject: [PATCH 1/5] browser video recording --- .github/workflows/release.yml | 30 ++-- examples/video_recording_demo.py | 49 ++++++ pyproject.toml | 2 +- sentience/__init__.py | 2 +- sentience/browser.py | 85 ++++++++- sentience/extension/background.js | 6 +- sentience/extension/content.js | 2 +- sentience/extension/injected_api.js | 214 +++++++++++------------ sentience/text_search.py | 4 +- sentience/trace_indexing/__init__.py | 12 +- sentience/trace_indexing/index_schema.py | 28 +-- sentience/trace_indexing/indexer.py | 32 ++-- tests/test_trace_indexing.py | 20 +-- 13 files changed, 301 insertions(+), 185 deletions(-) create mode 100644 examples/video_recording_demo.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 256e138..de0fcb0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,7 @@ jobs: - name: Verify extension files are present run: | echo "šŸ” Verifying extension files are included..." - + # Check required extension files exist REQUIRED_FILES=( "sentience/extension/manifest.json" @@ -63,14 +63,14 @@ jobs: "sentience/extension/pkg/sentience_core.js" "sentience/extension/pkg/sentience_core_bg.wasm" ) - + MISSING_FILES=() for file in "${REQUIRED_FILES[@]}"; do if [ ! -f "$file" ]; then MISSING_FILES+=("$file") fi done - + if [ ${#MISSING_FILES[@]} -ne 0 ]; then echo "āŒ Error: Missing required extension files:" printf ' - %s\n' "${MISSING_FILES[@]}" @@ -79,14 +79,14 @@ jobs: echo "Run the sync-extension workflow or manually sync extension files." exit 1 fi - + # Verify findTextRect function exists in injected_api.js if ! grep -q "findTextRect:" sentience/extension/injected_api.js; then echo "āŒ Error: findTextRect function not found in injected_api.js" echo "The extension may be out of date. Please sync the extension before releasing." exit 1 fi - + echo "āœ… All extension files verified" echo "šŸ“¦ Extension files that will be included:" find sentience/extension -type f | sort @@ -98,23 +98,23 @@ jobs: - name: Check package run: | twine check dist/* - + - name: Verify extension files in built package run: | echo "šŸ” Verifying extension files are included in the built package..." - + # Extract wheel to check contents WHEEL_FILE=$(ls dist/*.whl | head -1) WHEEL_PATH=$(realpath "$WHEEL_FILE") echo "Checking wheel: $WHEEL_PATH" - + # Create temp directory for extraction TEMP_DIR=$(mktemp -d) cd "$TEMP_DIR" - + # Extract wheel (it's a zip file) unzip -q "$WHEEL_PATH" - + # Check for required extension files in the wheel REQUIRED_IN_WHEEL=( "sentience/extension/manifest.json" @@ -122,14 +122,14 @@ jobs: "sentience/extension/pkg/sentience_core.js" "sentience/extension/pkg/sentience_core_bg.wasm" ) - + MISSING_IN_WHEEL=() for file in "${REQUIRED_IN_WHEEL[@]}"; do if [ ! -f "$file" ]; then MISSING_IN_WHEEL+=("$file") fi done - + if [ ${#MISSING_IN_WHEEL[@]} -ne 0 ]; then echo "āŒ Error: Extension files missing from built wheel:" printf ' - %s\n' "${MISSING_IN_WHEEL[@]}" @@ -138,17 +138,17 @@ jobs: echo "Check MANIFEST.in and pyproject.toml package-data settings." exit 1 fi - + # Verify findTextRect is in the packaged injected_api.js if ! grep -q "findTextRect:" sentience/extension/injected_api.js; then echo "āŒ Error: findTextRect not found in packaged injected_api.js" exit 1 fi - + echo "āœ… All extension files verified in built package" echo "šŸ“¦ Extension files found in wheel:" find sentience/extension -type f | sort - + # Cleanup rm -rf "$TEMP_DIR" diff --git a/examples/video_recording_demo.py b/examples/video_recording_demo.py new file mode 100644 index 0000000..d1edae0 --- /dev/null +++ b/examples/video_recording_demo.py @@ -0,0 +1,49 @@ +""" +Video Recording Demo - Record browser sessions with SentienceBrowser + +This example demonstrates how to use the video recording feature +to capture browser automation sessions. +""" + +from sentience import SentienceBrowser +from pathlib import Path + + +def main(): + # Create output directory for videos + video_dir = Path("./recordings") + video_dir.mkdir(exist_ok=True) + + print("\n" + "=" * 60) + print("Video Recording Demo") + print("=" * 60 + "\n") + + # Create browser with video recording enabled + with SentienceBrowser(record_video_dir=str(video_dir)) as browser: + print("šŸŽ„ Video recording enabled") + print(f"šŸ“ Videos will be saved to: {video_dir.absolute()}\n") + + # Navigate to example.com + print("Navigating to example.com...") + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + # Perform some actions + print("Taking screenshot...") + browser.page.screenshot(path="example_screenshot.png") + + print("Scrolling page...") + browser.page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + browser.page.wait_for_timeout(1000) + + print("\nāœ… Recording complete!") + print("Video will be saved when browser closes...\n") + + # Video is automatically saved when context manager exits + print("=" * 60) + print(f"Check {video_dir.absolute()} for the recorded video (.webm)") + print("=" * 60 + "\n") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 928c2eb..db6537f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sentienceapi" -version = "0.90.11" +version = "0.90.12" description = "Python SDK for Sentience AI Agent Browser Automation" readme = "README.md" requires-python = ">=3.11" diff --git a/sentience/__init__.py b/sentience/__init__.py index dd64f83..ca126be 100644 --- a/sentience/__init__.py +++ b/sentience/__init__.py @@ -70,7 +70,7 @@ ) from .wait import wait_for -__version__ = "0.90.11" +__version__ = "0.90.12" __all__ = [ # Core SDK diff --git a/sentience/browser.py b/sentience/browser.py index efc41a0..271cc86 100644 --- a/sentience/browser.py +++ b/sentience/browser.py @@ -33,6 +33,8 @@ def __init__( proxy: str | None = None, user_data_dir: str | None = None, storage_state: str | Path | StorageState | dict | None = None, + record_video_dir: str | Path | None = None, + record_video_size: dict[str, int] | None = None, ): """ Initialize Sentience browser @@ -57,6 +59,14 @@ def __init__( - StorageState object - Dictionary with 'cookies' and/or 'origins' keys If provided, browser starts with pre-injected authentication. + record_video_dir: Optional directory path to save video recordings. + If provided, browser will record video of all pages. + Videos are saved as .webm files in the specified directory. + If None, no video recording is performed. + record_video_size: Optional video resolution as dict with 'width' and 'height' keys. + Examples: {"width": 1280, "height": 800} (default) + {"width": 1920, "height": 1080} (1080p) + If None, defaults to 1280x800. """ self.api_key = api_key # Only set api_url if api_key is provided, otherwise None (free tier) @@ -80,6 +90,10 @@ def __init__( self.user_data_dir = user_data_dir self.storage_state = storage_state + # Video recording support + self.record_video_dir = record_video_dir + self.record_video_size = record_video_size or {"width": 1280, "height": 800} + self.playwright: Playwright | None = None self.context: BrowserContext | None = None self.page: Page | None = None @@ -209,6 +223,17 @@ def start(self) -> None: launch_params["ignore_https_errors"] = True print(f"šŸŒ [Sentience] Using proxy: {proxy_config.server}") + # Add video recording if configured + if self.record_video_dir: + video_dir = Path(self.record_video_dir) + video_dir.mkdir(parents=True, exist_ok=True) + launch_params["record_video_dir"] = str(video_dir) + launch_params["record_video_size"] = self.record_video_size + print(f"šŸŽ„ [Sentience] Recording video to: {video_dir}") + print( + f" Resolution: {self.record_video_size['width']}x{self.record_video_size['height']}" + ) + # Launch persistent context (required for extensions) # Note: We pass headless=False to launch_persistent_context because we handle # headless mode via the --headless=new arg above. This is a Playwright workaround. @@ -390,15 +415,71 @@ def _wait_for_extension(self, timeout_sec: float = 5.0) -> bool: return False - def close(self) -> None: - """Close browser and cleanup""" + def close(self, output_path: str | Path | None = None) -> str | None: + """ + Close browser and cleanup + + Args: + output_path: Optional path to rename the video file to. + If provided, the recorded video will be moved to this location. + Useful for giving videos meaningful names instead of random hashes. + + Returns: + Path to video file if recording was enabled, None otherwise + Note: Video files are saved automatically by Playwright when context closes. + If multiple pages exist, returns the path to the first page's video. + """ + temp_video_path = None + + # Get video path before closing (if recording was enabled) + # Note: Playwright saves videos when pages/context close, but we can get the + # expected path before closing. The actual file will be available after close. + if self.record_video_dir: + try: + # Try to get video path from the first page + if self.page and self.page.video: + temp_video_path = self.page.video.path() + # If that fails, check all pages in the context + elif self.context: + for page in self.context.pages: + if page.video: + temp_video_path = page.video.path() + break + except Exception: + # Video path might not be available until after close + # In that case, we'll return None and user can check the directory + pass + + # Close context (this triggers video file finalization) if self.context: self.context.close() + + # Close playwright if self.playwright: self.playwright.stop() + + # Clean up extension directory if self._extension_path and os.path.exists(self._extension_path): shutil.rmtree(self._extension_path) + # Rename/move video if output_path is specified + final_path = temp_video_path + if temp_video_path and output_path and os.path.exists(temp_video_path): + try: + output_path = str(output_path) + # Ensure parent directory exists + Path(output_path).parent.mkdir(parents=True, exist_ok=True) + shutil.move(temp_video_path, output_path) + final_path = output_path + except Exception as e: + import warnings + + warnings.warn(f"Failed to rename video file: {e}") + # Return original path if rename fails + final_path = temp_video_path + + return final_path + def __enter__(self): """Context manager entry""" self.start() diff --git a/sentience/extension/background.js b/sentience/extension/background.js index 811303f..f359ba6 100644 --- a/sentience/extension/background.js +++ b/sentience/extension/background.js @@ -144,13 +144,13 @@ async function handleScreenshotCapture(_tabId, options = {}) { async function handleSnapshotProcessing(rawData, options = {}) { const MAX_ELEMENTS = 10000; // Safety limit to prevent hangs const startTime = performance.now(); - + try { // Safety check: limit element count to prevent hangs if (!Array.isArray(rawData)) { throw new Error('rawData must be an array'); } - + if (rawData.length > MAX_ELEMENTS) { console.warn(`[Sentience Background] āš ļø Large dataset: ${rawData.length} elements. Limiting to ${MAX_ELEMENTS} to prevent hangs.`); rawData = rawData.slice(0, MAX_ELEMENTS); @@ -186,7 +186,7 @@ async function handleSnapshotProcessing(rawData, options = {}) { // Add timeout protection (18 seconds - less than content.js timeout) analyzedElements = await Promise.race([ wasmPromise, - new Promise((_, reject) => + new Promise((_, reject) => setTimeout(() => reject(new Error('WASM processing timeout (>18s)')), 18000) ) ]); diff --git a/sentience/extension/content.js b/sentience/extension/content.js index 62ae408..8d3b0d4 100644 --- a/sentience/extension/content.js +++ b/sentience/extension/content.js @@ -92,7 +92,7 @@ function handleSnapshotRequest(data) { if (responded) return; // Already responded via timeout responded = true; clearTimeout(timeoutId); - + const duration = performance.now() - startTime; // Handle Chrome extension errors (e.g., background script crashed) diff --git a/sentience/extension/injected_api.js b/sentience/extension/injected_api.js index 45c4337..e81c9be 100644 --- a/sentience/extension/injected_api.js +++ b/sentience/extension/injected_api.js @@ -66,10 +66,10 @@ // --- HELPER: Safe Class Name Extractor (Handles SVGAnimatedString) --- function getClassName(el) { if (!el || !el.className) return ''; - + // Handle string (HTML elements) if (typeof el.className === 'string') return el.className; - + // Handle SVGAnimatedString (SVG elements) if (typeof el.className === 'object') { if ('baseVal' in el.className && typeof el.className.baseVal === 'string') { @@ -85,17 +85,17 @@ return ''; } } - + return ''; } // --- HELPER: Paranoid String Converter (Handles SVGAnimatedString) --- function toSafeString(value) { if (value === null || value === undefined) return null; - + // 1. If it's already a primitive string, return it if (typeof value === 'string') return value; - + // 2. Handle SVG objects (SVGAnimatedString, SVGAnimatedNumber, etc.) if (typeof value === 'object') { // Try extracting baseVal (standard SVG property) @@ -114,7 +114,7 @@ return null; } } - + // 3. Last resort cast for primitives try { return String(value); @@ -127,9 +127,9 @@ // For SVG elements, get the fill or stroke color (SVGs use fill/stroke, not backgroundColor) function getSVGColor(el) { if (!el || el.tagName !== 'SVG') return null; - + const style = window.getComputedStyle(el); - + // Try fill first (most common for SVG icons) const fill = style.fill; if (fill && fill !== 'none' && fill !== 'transparent' && fill !== 'rgba(0, 0, 0, 0)') { @@ -144,7 +144,7 @@ return fill; } } - + // Fallback to stroke if fill is not available const stroke = style.stroke; if (stroke && stroke !== 'none' && stroke !== 'transparent' && stroke !== 'rgba(0, 0, 0, 0)') { @@ -158,7 +158,7 @@ return stroke; } } - + return null; } @@ -168,28 +168,28 @@ // This handles rgba(0,0,0,0) and transparent values that browsers commonly return function getEffectiveBackgroundColor(el) { if (!el) return null; - + // For SVG elements, use fill/stroke instead of backgroundColor if (el.tagName === 'SVG') { const svgColor = getSVGColor(el); if (svgColor) return svgColor; } - + let current = el; const maxDepth = 10; // Prevent infinite loops let depth = 0; - + while (current && depth < maxDepth) { const style = window.getComputedStyle(current); - + // For SVG elements in the tree, also check fill/stroke if (current.tagName === 'SVG') { const svgColor = getSVGColor(current); if (svgColor) return svgColor; } - + const bgColor = style.backgroundColor; - + if (bgColor && bgColor !== 'transparent' && bgColor !== 'rgba(0, 0, 0, 0)') { // Check if it's rgba with alpha < 1 (semi-transparent) const rgbaMatch = bgColor.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/); @@ -209,12 +209,12 @@ return bgColor; } } - + // Move up the DOM tree current = current.parentElement; depth++; } - + // Fallback: return null if nothing found return null; } @@ -235,7 +235,7 @@ // Only check for elements that are likely to be occluded (overlays, modals, tooltips) const zIndex = parseInt(style.zIndex, 10); const position = style.position; - + // Skip occlusion check for normal flow elements (vast majority) // Only check for positioned elements or high z-index (likely overlays) if (position === 'static' && (isNaN(zIndex) || zIndex <= 10)) { @@ -308,7 +308,7 @@ }; window.addEventListener('message', listener); - + try { window.postMessage({ type: 'SENTIENCE_SNAPSHOT_REQUEST', @@ -514,7 +514,7 @@ function extractRawElementData(el) { const style = window.getComputedStyle(el); const rect = el.getBoundingClientRect(); - + return { tag: el.tagName, rect: { @@ -548,12 +548,12 @@ // --- HELPER: Generate Unique CSS Selector (for Golden Set) --- function getUniqueSelector(el) { if (!el || !el.tagName) return ''; - + // If element has a unique ID, use it if (el.id) { return `#${el.id}`; } - + // Try data attributes or aria-label for uniqueness for (const attr of el.attributes) { if (attr.name.startsWith('data-') || attr.name === 'aria-label') { @@ -561,21 +561,21 @@ return `${el.tagName.toLowerCase()}[${attr.name}="${value}"]`; } } - + // Build path with classes and nth-child for uniqueness const path = []; let current = el; - + while (current && current !== document.body && current !== document.documentElement) { let selector = current.tagName.toLowerCase(); - + // If current element has ID, use it and stop if (current.id) { selector = `#${current.id}`; path.unshift(selector); break; } - + // Add class if available if (current.className && typeof current.className === 'string') { const classes = current.className.trim().split(/\s+/).filter(c => c); @@ -584,7 +584,7 @@ selector += `.${classes[0]}`; } } - + // Add nth-of-type if needed for uniqueness if (current.parentElement) { const siblings = Array.from(current.parentElement.children); @@ -594,11 +594,11 @@ selector += `:nth-of-type(${index + 1})`; } } - + path.unshift(selector); current = current.parentElement; } - + return path.join(' > ') || el.tagName.toLowerCase(); } @@ -613,7 +613,7 @@ } = options; const startTime = Date.now(); - + return new Promise((resolve) => { // Check if DOM already has enough nodes const nodeCount = document.querySelectorAll('*').length; @@ -623,17 +623,17 @@ const observer = new MutationObserver(() => { lastChange = Date.now(); }); - + observer.observe(document.body, { childList: true, subtree: true, attributes: false }); - + const checkStable = () => { const timeSinceLastChange = Date.now() - lastChange; const totalWait = Date.now() - startTime; - + if (timeSinceLastChange >= quietPeriod) { observer.disconnect(); resolve(); @@ -645,14 +645,14 @@ setTimeout(checkStable, 50); } }; - + checkStable(); } else { // DOM doesn't have enough nodes yet, wait for them const observer = new MutationObserver(() => { const currentCount = document.querySelectorAll('*').length; const totalWait = Date.now() - startTime; - + if (currentCount >= minNodeCount) { observer.disconnect(); // Now wait for quiet period @@ -660,17 +660,17 @@ const quietObserver = new MutationObserver(() => { lastChange = Date.now(); }); - + quietObserver.observe(document.body, { childList: true, subtree: true, attributes: false }); - + const checkQuiet = () => { const timeSinceLastChange = Date.now() - lastChange; const totalWait = Date.now() - startTime; - + if (timeSinceLastChange >= quietPeriod) { quietObserver.disconnect(); resolve(); @@ -682,7 +682,7 @@ setTimeout(checkQuiet, 50); } }; - + checkQuiet(); } else if (totalWait >= maxWait) { observer.disconnect(); @@ -690,13 +690,13 @@ resolve(); } }); - + observer.observe(document.body, { childList: true, subtree: true, attributes: false }); - + // Timeout fallback setTimeout(() => { observer.disconnect(); @@ -710,21 +710,21 @@ // --- HELPER: Collect Iframe Snapshots (Frame Stitching) --- // Recursively collects snapshot data from all child iframes // This enables detection of elements inside iframes (e.g., Stripe forms) - // + // // NOTE: Cross-origin iframes cannot be accessed due to browser security (Same-Origin Policy). // Only same-origin iframes will return snapshot data. Cross-origin iframes will be skipped // with a warning. For cross-origin iframes, users must manually switch frames using // Playwright's page.frame() API. async function collectIframeSnapshots(options = {}) { const iframeData = new Map(); // Map of iframe element -> snapshot data - + // Find all iframe elements in current document const iframes = Array.from(document.querySelectorAll('iframe')); - + if (iframes.length === 0) { return iframeData; } - + console.log(`[SentienceAPI] Found ${iframes.length} iframe(s), requesting snapshots...`); // Request snapshot from each iframe const iframePromises = iframes.map((iframe, idx) => { @@ -737,13 +737,13 @@ return new Promise((resolve) => { const requestId = `iframe-${idx}-${Date.now()}`; - + // 1. EXTENDED TIMEOUT (Handle slow children) const timeout = setTimeout(() => { console.warn(`[SentienceAPI] āš ļø Iframe ${idx} snapshot TIMEOUT (id: ${requestId})`); resolve(null); }, 5000); // Increased to 5s to handle slow processing - + // 2. ROBUST LISTENER with debugging const listener = (event) => { // Debug: Log all SENTIENCE_IFRAME_SNAPSHOT_RESPONSE messages to see what's happening @@ -753,14 +753,14 @@ // console.log(`[SentienceAPI] Received response for different request: ${event.data.requestId} (expected: ${requestId})`); } } - + // Check if this is the response we're waiting for - if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE' && + if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_RESPONSE' && event.data?.requestId === requestId) { - + clearTimeout(timeout); window.removeEventListener('message', listener); - + if (event.data.error) { console.warn(`[SentienceAPI] Iframe ${idx} returned error:`, event.data.error); resolve(null); @@ -775,9 +775,9 @@ } } }; - + window.addEventListener('message', listener); - + // 3. SEND REQUEST with error handling try { if (iframe.contentWindow) { @@ -785,8 +785,8 @@ iframe.contentWindow.postMessage({ type: 'SENTIENCE_IFRAME_SNAPSHOT_REQUEST', requestId: requestId, - options: { - ...options, + options: { + ...options, collectIframes: true // Enable recursion for nested iframes } }, '*'); // Use '*' for cross-origin, but browser will enforce same-origin policy @@ -804,10 +804,10 @@ } }); }); - + // Wait for all iframe responses const results = await Promise.all(iframePromises); - + // Store iframe data results.forEach((result, idx) => { if (result && result.data && !result.error) { @@ -819,7 +819,7 @@ console.warn(`[SentienceAPI] Iframe ${idx} returned no data (timeout or error)`); } }); - + return iframeData; } @@ -832,7 +832,7 @@ // Security: only respond to snapshot requests from parent frames if (event.data?.type === 'SENTIENCE_IFRAME_SNAPSHOT_REQUEST') { const { requestId, options } = event.data; - + try { // Generate snapshot for this iframe's content // Allow recursive collection - querySelectorAll('iframe') only finds direct children, @@ -840,7 +840,7 @@ // waitForStability: false makes performance better - i.e. don't wait for children frames const snapshotOptions = { ...options, collectIframes: true, waitForStability: options.waitForStability === false ? false : false }; const snapshot = await window.sentience.snapshot(snapshotOptions); - + // Send response back to parent if (event.source && event.source.postMessage) { event.source.postMessage({ @@ -864,7 +864,7 @@ } }); } - + // Setup iframe handler when script loads (only once) if (!window.sentience_iframe_handler_setup) { setupIframeSnapshotHandler(); @@ -880,7 +880,7 @@ if (options.waitForStability !== false) { await waitForStability(options.waitForStability || {}); } - + // Step 1: Collect raw DOM data (Main World - CSP can't block this!) const rawData = []; window.sentience_registry = []; @@ -896,17 +896,17 @@ const textVal = getText(el); const inView = isInViewport(rect); - + // Get computed style once (needed for both occlusion check and data collection) const style = window.getComputedStyle(el); - + // Only check occlusion for elements likely to be occluded (optimized) // This avoids layout thrashing for the vast majority of elements const occluded = inView ? isOccluded(el, rect, style) : false; - + // Get effective background color (traverses DOM to find non-transparent color) const effectiveBgColor = getEffectiveBackgroundColor(el); - + rawData.push({ id: idx, tag: el.tagName.toLowerCase(), @@ -946,26 +946,26 @@ // This allows WASM to process all elements uniformly (no recursion needed) let allRawElements = [...rawData]; // Start with main frame elements let totalIframeElements = 0; - + if (options.collectIframes !== false) { try { console.log(`[SentienceAPI] Starting iframe collection...`); const iframeSnapshots = await collectIframeSnapshots(options); console.log(`[SentienceAPI] Iframe collection complete. Received ${iframeSnapshots.size} snapshot(s)`); - + if (iframeSnapshots.size > 0) { // FLATTEN IMMEDIATELY: Don't nest them. Just append them with coordinate translation. iframeSnapshots.forEach((iframeSnapshot, iframeEl) => { // Debug: Log structure to verify data is correct // console.log(`[SentienceAPI] Processing iframe snapshot:`, iframeSnapshot); - + if (iframeSnapshot && iframeSnapshot.raw_elements) { const rawElementsCount = iframeSnapshot.raw_elements.length; console.log(`[SentienceAPI] Processing ${rawElementsCount} elements from iframe (src: ${iframeEl.src || 'unknown'})`); // Get iframe's bounding rect (offset for coordinate translation) const iframeRect = iframeEl.getBoundingClientRect(); const offset = { x: iframeRect.x, y: iframeRect.y }; - + // Get iframe context for frame switching (Playwright needs this) const iframeSrc = iframeEl.src || iframeEl.getAttribute('src') || ''; let isSameOrigin = false; @@ -975,11 +975,11 @@ } catch (e) { isSameOrigin = false; } - + // Adjust coordinates and add iframe context to each element const adjustedElements = iframeSnapshot.raw_elements.map(el => { const adjusted = { ...el }; - + // Adjust rect coordinates to parent viewport if (adjusted.rect) { adjusted.rect = { @@ -988,22 +988,22 @@ y: adjusted.rect.y + offset.y }; } - + // Add iframe context so agents can switch frames in Playwright adjusted.iframe_context = { src: iframeSrc, is_same_origin: isSameOrigin }; - + return adjusted; }); - + // Append flattened iframe elements to main array allRawElements.push(...adjustedElements); totalIframeElements += adjustedElements.length; } }); - + // console.log(`[SentienceAPI] Merged ${iframeSnapshots.size} iframe(s). Total elements: ${allRawElements.length} (${rawData.length} main + ${totalIframeElements} iframe)`); } } catch (error) { @@ -1016,7 +1016,7 @@ // No recursion needed - everything is already flat console.log(`[SentienceAPI] Sending ${allRawElements.length} total elements to WASM (${rawData.length} main + ${totalIframeElements} iframe)`); const processed = await processSnapshotInBackground(allRawElements, options); - + if (!processed || !processed.elements) { throw new Error('WASM processing returned invalid result'); } @@ -1032,10 +1032,10 @@ const cleanedRawElements = cleanElement(processed.raw_elements); // FIXED: Removed undefined 'totalIframeRawElements' - // FIXED: Logic updated for "Flatten Early" architecture. + // FIXED: Logic updated for "Flatten Early" architecture. // processed.elements ALREADY contains the merged iframe elements, // so we simply use .length. No addition needed. - + const totalCount = cleanedElements.length; const totalRaw = cleanedRawElements.length; const iframeCount = totalIframeElements || 0; @@ -1253,23 +1253,23 @@ autoDisableTimeout = 30 * 60 * 1000, // 30 minutes default keyboardShortcut = 'Ctrl+Shift+I' } = options; - + console.log("šŸ”“ [Sentience] Recording Mode STARTED. Click an element to copy its Ground Truth JSON."); console.log(` Press ${keyboardShortcut} or call stopRecording() to stop.`); - + // Validate registry is populated if (!window.sentience_registry || window.sentience_registry.length === 0) { console.warn("āš ļø Registry empty. Call `await window.sentience.snapshot()` first to populate registry."); alert("Registry empty. Run `await window.sentience.snapshot()` first!"); return () => {}; // Return no-op cleanup function } - + // Create reverse mapping for O(1) lookup (fixes registry lookup bug) window.sentience_registry_map = new Map(); window.sentience_registry.forEach((el, idx) => { if (el) window.sentience_registry_map.set(el, idx); }); - + // Create highlight box overlay let highlightBox = document.getElementById('sentience-highlight-box'); if (!highlightBox) { @@ -1287,7 +1287,7 @@ `; document.body.appendChild(highlightBox); } - + // Create visual indicator (red border on page when recording) let recordingIndicator = document.getElementById('sentience-recording-indicator'); if (!recordingIndicator) { @@ -1306,12 +1306,12 @@ document.body.appendChild(recordingIndicator); } recordingIndicator.style.display = 'block'; - + // Hover handler (visual feedback) const mouseOverHandler = (e) => { const el = e.target; if (!el || el === highlightBox || el === recordingIndicator) return; - + const rect = el.getBoundingClientRect(); highlightBox.style.display = 'block'; highlightBox.style.top = (rect.top + window.scrollY) + 'px'; @@ -1319,15 +1319,15 @@ highlightBox.style.width = rect.width + 'px'; highlightBox.style.height = rect.height + 'px'; }; - + // Click handler (capture ground truth data) const clickHandler = (e) => { e.preventDefault(); e.stopPropagation(); - + const el = e.target; if (!el || el === highlightBox || el === recordingIndicator) return; - + // Use Map for reliable O(1) lookup const sentienceId = window.sentience_registry_map.get(el); if (sentienceId === undefined) { @@ -1335,13 +1335,13 @@ alert("Element not in registry. Run `await window.sentience.snapshot()` first!"); return; } - + // Extract raw data (ground truth + raw signals, NOT model outputs) const rawData = extractRawElementData(el); const selector = getUniqueSelector(el); const role = el.getAttribute('role') || el.tagName.toLowerCase(); const text = getText(el); - + // Build golden set JSON (ground truth + raw signals only) const snippet = { task: `Interact with ${text.substring(0, 20)}${text.length > 20 ? '...' : ''}`, @@ -1355,12 +1355,12 @@ }, debug_snapshot: rawData }; - + // Copy to clipboard const jsonString = JSON.stringify(snippet, null, 2); navigator.clipboard.writeText(jsonString).then(() => { console.log("āœ… Copied Ground Truth to clipboard:", snippet); - + // Flash green to indicate success highlightBox.style.border = `2px solid ${successColor}`; highlightBox.style.background = 'rgba(0, 255, 0, 0.2)'; @@ -1373,42 +1373,42 @@ alert("Failed to copy to clipboard. Check console for JSON."); }); }; - + // Auto-disable timeout let timeoutId = null; - + // Cleanup function to stop recording (defined before use) const stopRecording = () => { document.removeEventListener('mouseover', mouseOverHandler, true); document.removeEventListener('click', clickHandler, true); document.removeEventListener('keydown', keyboardHandler, true); - + if (timeoutId) { clearTimeout(timeoutId); timeoutId = null; } - + if (highlightBox) { highlightBox.style.display = 'none'; } - + if (recordingIndicator) { recordingIndicator.style.display = 'none'; } - + // Clean up registry map (optional, but good practice) if (window.sentience_registry_map) { window.sentience_registry_map.clear(); } - + // Remove global reference if (window.sentience_stopRecording === stopRecording) { delete window.sentience_stopRecording; } - + console.log("āšŖ [Sentience] Recording Mode STOPPED."); }; - + // Keyboard shortcut handler (defined after stopRecording) const keyboardHandler = (e) => { // Ctrl+Shift+I or Cmd+Shift+I @@ -1417,12 +1417,12 @@ stopRecording(); } }; - + // Attach event listeners (use capture phase to intercept early) document.addEventListener('mouseover', mouseOverHandler, true); document.addEventListener('click', clickHandler, true); document.addEventListener('keydown', keyboardHandler, true); - + // Set up auto-disable timeout if (autoDisableTimeout > 0) { timeoutId = setTimeout(() => { @@ -1430,10 +1430,10 @@ stopRecording(); }, autoDisableTimeout); } - + // Store stop function globally for keyboard shortcut access window.sentience_stopRecording = stopRecording; - + return stopRecording; } }; diff --git a/sentience/text_search.py b/sentience/text_search.py index 2a2bf70..42b9309 100644 --- a/sentience/text_search.py +++ b/sentience/text_search.py @@ -127,9 +127,7 @@ def find_text_rect( except RuntimeError: raise except Exception as e: - raise RuntimeError( - f"Failed to verify findTextRect availability: {e}" - ) from e + raise RuntimeError(f"Failed to verify findTextRect availability: {e}") from e # Call the extension's findTextRect method result_dict = browser.page.evaluate( diff --git a/sentience/trace_indexing/__init__.py b/sentience/trace_indexing/__init__.py index dfe9066..746ee59 100644 --- a/sentience/trace_indexing/__init__.py +++ b/sentience/trace_indexing/__init__.py @@ -2,16 +2,16 @@ Trace indexing module for Sentience SDK. """ -from .indexer import build_trace_index, write_trace_index, read_step_events from .index_schema import ( - TraceIndex, - StepIndex, - TraceSummary, - TraceFileInfo, - SnapshotInfo, ActionInfo, + SnapshotInfo, StepCounters, + StepIndex, + TraceFileInfo, + TraceIndex, + TraceSummary, ) +from .indexer import build_trace_index, read_step_events, write_trace_index __all__ = [ "build_trace_index", diff --git a/sentience/trace_indexing/index_schema.py b/sentience/trace_indexing/index_schema.py index 4439d82..f630b1b 100644 --- a/sentience/trace_indexing/index_schema.py +++ b/sentience/trace_indexing/index_schema.py @@ -2,8 +2,8 @@ Type definitions for trace index schema using concrete classes. """ -from dataclasses import dataclass, field, asdict -from typing import Optional, List, Literal +from dataclasses import asdict, dataclass, field +from typing import List, Literal, Optional @dataclass @@ -27,7 +27,7 @@ class TraceSummary: event_count: int step_count: int error_count: int - final_url: Optional[str] + final_url: str | None def to_dict(self) -> dict: return asdict(self) @@ -37,9 +37,9 @@ def to_dict(self) -> dict: class SnapshotInfo: """Snapshot metadata for index.""" - snapshot_id: Optional[str] = None - digest: Optional[str] = None - url: Optional[str] = None + snapshot_id: str | None = None + digest: str | None = None + url: str | None = None def to_dict(self) -> dict: return asdict(self) @@ -49,10 +49,10 @@ def to_dict(self) -> dict: class ActionInfo: """Action metadata for index.""" - type: Optional[str] = None - target_element_id: Optional[int] = None - args_digest: Optional[str] = None - success: Optional[bool] = None + type: str | None = None + target_element_id: int | None = None + args_digest: str | None = None + success: bool | None = None def to_dict(self) -> dict: return asdict(self) @@ -77,14 +77,14 @@ class StepIndex: step_index: int step_id: str - goal: Optional[str] + goal: str | None status: Literal["ok", "error", "partial"] ts_start: str ts_end: str offset_start: int offset_end: int - url_before: Optional[str] - url_after: Optional[str] + url_before: str | None + url_after: str | None snapshot_before: SnapshotInfo snapshot_after: SnapshotInfo action: ActionInfo @@ -104,7 +104,7 @@ class TraceIndex: created_at: str trace_file: TraceFileInfo summary: TraceSummary - steps: List[StepIndex] = field(default_factory=list) + steps: list[StepIndex] = field(default_factory=list) def to_dict(self) -> dict: """Convert to dictionary for JSON serialization.""" diff --git a/sentience/trace_indexing/indexer.py b/sentience/trace_indexing/indexer.py index e793f7c..52f3fab 100644 --- a/sentience/trace_indexing/indexer.py +++ b/sentience/trace_indexing/indexer.py @@ -10,13 +10,13 @@ from typing import Any, Dict, List from .index_schema import ( - TraceIndex, - StepIndex, - TraceSummary, - TraceFileInfo, - SnapshotInfo, ActionInfo, + SnapshotInfo, StepCounters, + StepIndex, + TraceFileInfo, + TraceIndex, + TraceSummary, ) @@ -34,7 +34,7 @@ def _normalize_text(text: str | None, max_len: int = 80) -> str: return normalized -def _round_bbox(bbox: Dict[str, float], precision: int = 2) -> Dict[str, int]: +def _round_bbox(bbox: dict[str, float], precision: int = 2) -> dict[str, int]: """Round bbox coordinates to reduce noise (default: 2px precision).""" return { "x": round(bbox.get("x", 0) / precision) * precision, @@ -44,7 +44,7 @@ def _round_bbox(bbox: Dict[str, float], precision: int = 2) -> Dict[str, int]: } -def _compute_snapshot_digest(snapshot_data: Dict[str, Any]) -> str: +def _compute_snapshot_digest(snapshot_data: dict[str, Any]) -> str: """ Compute stable digest of snapshot for diffing. @@ -62,9 +62,7 @@ def _compute_snapshot_digest(snapshot_data: Dict[str, Any]) -> str: "id": elem.get("id"), "role": elem.get("role", ""), "text_norm": _normalize_text(elem.get("text")), - "bbox": _round_bbox( - elem.get("bbox", {"x": 0, "y": 0, "width": 0, "height": 0}) - ), + "bbox": _round_bbox(elem.get("bbox", {"x": 0, "y": 0, "width": 0, "height": 0})), "is_primary": elem.get("is_primary", False), "is_clickable": elem.get("is_clickable", False), } @@ -89,7 +87,7 @@ def _compute_snapshot_digest(snapshot_data: Dict[str, Any]) -> str: return f"sha256:{digest}" -def _compute_action_digest(action_data: Dict[str, Any]) -> str: +def _compute_action_digest(action_data: dict[str, Any]) -> str: """ Compute digest of action args for privacy + determinism. @@ -152,8 +150,8 @@ def build_trace_index(trace_path: str) -> TraceIndex: error_count = 0 final_url = None - steps_by_id: Dict[str, StepIndex] = {} - step_order: List[str] = [] # Track order of first appearance + steps_by_id: dict[str, StepIndex] = {} + step_order: list[str] = [] # Track order of first appearance # Stream through file, tracking byte offsets with open(trace_path, "rb") as f: @@ -228,9 +226,7 @@ def build_trace_index(trace_path: str) -> TraceIndex: ) step.url_before = step.url_before or url - step.snapshot_after = SnapshotInfo( - snapshot_id=snapshot_id, digest=digest, url=url - ) + step.snapshot_after = SnapshotInfo(snapshot_id=snapshot_id, digest=digest, url=url) step.url_after = url step.counters.snapshots += 1 final_url = url @@ -311,9 +307,7 @@ def write_trace_index(trace_path: str, index_path: str | None = None) -> str: return index_path -def read_step_events( - trace_path: str, offset_start: int, offset_end: int -) -> List[Dict[str, Any]]: +def read_step_events(trace_path: str, offset_start: int, offset_end: int) -> list[dict[str, Any]]: """ Read events for a specific step using byte offsets from index. diff --git a/tests/test_trace_indexing.py b/tests/test_trace_indexing.py index bcadffb..927a25e 100644 --- a/tests/test_trace_indexing.py +++ b/tests/test_trace_indexing.py @@ -10,11 +10,11 @@ import pytest from sentience.trace_indexing import ( + StepIndex, + TraceIndex, build_trace_index, - write_trace_index, read_step_events, - TraceIndex, - StepIndex, + write_trace_index, ) @@ -180,9 +180,7 @@ def test_byte_offset_accuracy(self): # Read step-1 events using offset step1 = index.steps[0] - step1_events = read_step_events( - str(trace_path), step1.offset_start, step1.offset_end - ) + step1_events = read_step_events(str(trace_path), step1.offset_start, step1.offset_end) assert len(step1_events) == 2 assert step1_events[0]["step_id"] == "step-1" @@ -192,9 +190,7 @@ def test_byte_offset_accuracy(self): # Read step-2 events using offset step2 = index.steps[1] - step2_events = read_step_events( - str(trace_path), step2.offset_start, step2.offset_end - ) + step2_events = read_step_events(str(trace_path), step2.offset_start, step2.offset_end) assert len(step2_events) == 2 assert step2_events[0]["step_id"] == "step-2" @@ -421,9 +417,7 @@ def test_write_trace_index(self): with tempfile.TemporaryDirectory() as tmpdir: trace_path = Path(tmpdir) / "test.jsonl" - events = [ - {"v": 1, "type": "run_start", "ts": "2025-12-29T10:00:00.000Z", "data": {}} - ] + events = [{"v": 1, "type": "run_start", "ts": "2025-12-29T10:00:00.000Z", "data": {}}] with open(trace_path, "w") as f: for event in events: @@ -435,7 +429,7 @@ def test_write_trace_index(self): assert index_path.endswith(".index.json") # Verify index content - with open(index_path, "r") as f: + with open(index_path) as f: index_data = json.load(f) assert index_data["version"] == 1 From c175f9ec905d9a2df80e2bb311705dad1e9644f2 Mon Sep 17 00:00:00 2001 From: rcholic Date: Mon, 29 Dec 2025 17:42:42 -0800 Subject: [PATCH 2/5] examples & tests --- examples/video_recording_advanced.py | 90 +++++++++++++ tests/test_video_recording.py | 191 +++++++++++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 examples/video_recording_advanced.py create mode 100644 tests/test_video_recording.py diff --git a/examples/video_recording_advanced.py b/examples/video_recording_advanced.py new file mode 100644 index 0000000..e471829 --- /dev/null +++ b/examples/video_recording_advanced.py @@ -0,0 +1,90 @@ +""" +Advanced Video Recording Demo + +Demonstrates advanced video recording features: +- Custom resolution (1080p) +- Custom output filename +- Multiple recordings in one session +""" + +from sentience import SentienceBrowser +from pathlib import Path +from datetime import datetime + + +def main(): + print("\n" + "=" * 60) + print("Advanced Video Recording Demo") + print("=" * 60 + "\n") + + video_dir = Path("./recordings") + video_dir.mkdir(exist_ok=True) + + # Example 1: Custom Resolution (1080p) + print("šŸ“¹ Example 1: Recording in 1080p (Full HD)\n") + + with SentienceBrowser( + record_video_dir=str(video_dir), + record_video_size={"width": 1920, "height": 1080} # 1080p resolution + ) as browser: + print(" Resolution: 1920x1080") + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_timeout(2000) + + # Close with custom filename + video_path = browser.close(output_path=video_dir / "example_1080p.webm") + print(f" āœ… Saved: {video_path}\n") + + # Example 2: Custom Filename with Timestamp + print("šŸ“¹ Example 2: Recording with timestamp filename\n") + + timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + custom_filename = f"recording_{timestamp}.webm" + + with SentienceBrowser(record_video_dir=str(video_dir)) as browser: + browser.page.goto("https://example.com") + browser.page.click("text=More information") + browser.page.wait_for_timeout(2000) + + video_path = browser.close(output_path=video_dir / custom_filename) + print(f" āœ… Saved: {video_path}\n") + + # Example 3: Organized by Project + print("šŸ“¹ Example 3: Organized directory structure\n") + + project_dir = Path("./recordings/my_project/tutorials") + + with SentienceBrowser(record_video_dir=str(project_dir)) as browser: + print(f" Saving to: {project_dir}") + browser.page.goto("https://example.com") + browser.page.wait_for_timeout(2000) + + video_path = browser.close(output_path=project_dir / "tutorial_01.webm") + print(f" āœ… Saved: {video_path}\n") + + # Example 4: Multiple videos with descriptive names + print("šŸ“¹ Example 4: Tutorial series with descriptive names\n") + + tutorials = [ + ("intro", "https://example.com"), + ("navigation", "https://example.com"), + ("features", "https://example.com"), + ] + + for name, url in tutorials: + with SentienceBrowser(record_video_dir=str(video_dir)) as browser: + browser.page.goto(url) + browser.page.wait_for_timeout(1000) + + video_path = browser.close(output_path=video_dir / f"{name}.webm") + print(f" āœ… {name}: {video_path}") + + print("\n" + "=" * 60) + print("All recordings completed!") + print(f"Check {video_dir.absolute()} for all videos") + print("=" * 60 + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/test_video_recording.py b/tests/test_video_recording.py new file mode 100644 index 0000000..f919c31 --- /dev/null +++ b/tests/test_video_recording.py @@ -0,0 +1,191 @@ +""" +Tests for video recording functionality +""" + +import os +import pytest +from pathlib import Path +import tempfile +import shutil + +from sentience import SentienceBrowser + + +def test_video_recording_basic(): + """Test basic video recording functionality""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + + with SentienceBrowser( + headless=True, + record_video_dir=str(video_dir) + ) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + video_path = browser.close() + + # Verify video was created + assert video_path is not None + assert os.path.exists(video_path) + assert video_path.endswith(".webm") + + # Verify file has content + file_size = os.path.getsize(video_path) + assert file_size > 0 + + +def test_video_recording_custom_resolution(): + """Test video recording with custom resolution""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + + with SentienceBrowser( + headless=True, + record_video_dir=str(video_dir), + record_video_size={"width": 1920, "height": 1080} + ) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + video_path = browser.close() + + assert video_path is not None + assert os.path.exists(video_path) + + +def test_video_recording_custom_output_path(): + """Test video recording with custom output path""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + custom_path = video_dir / "my_recording.webm" + + with SentienceBrowser( + headless=True, + record_video_dir=str(video_dir) + ) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + video_path = browser.close(output_path=str(custom_path)) + + # Verify video was renamed to custom path + assert video_path == str(custom_path) + assert os.path.exists(custom_path) + + +def test_video_recording_nested_output_path(): + """Test video recording with nested directory in output path""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + nested_path = video_dir / "project" / "tutorials" / "video1.webm" + + with SentienceBrowser( + headless=True, + record_video_dir=str(video_dir) + ) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + video_path = browser.close(output_path=str(nested_path)) + + # Verify nested directories were created + assert video_path == str(nested_path) + assert os.path.exists(nested_path) + assert nested_path.parent.exists() + + +def test_no_video_recording_when_disabled(): + """Test that no video is created when recording is disabled""" + with SentienceBrowser(headless=True) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + video_path = browser.close() + + # Should return None when recording is disabled + assert video_path is None + + +def test_video_recording_directory_auto_created(): + """Test that video directory is automatically created""" + with tempfile.TemporaryDirectory() as temp_dir: + # Use a non-existent directory + video_dir = Path(temp_dir) / "new_recordings" / "subdir" + + with SentienceBrowser( + headless=True, + record_video_dir=str(video_dir) + ) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + video_path = browser.close() + + # Verify directory was created + assert video_dir.exists() + assert video_path is not None + assert os.path.exists(video_path) + + +def test_video_recording_with_pathlib(): + """Test video recording using pathlib.Path objects""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + output_path = video_dir / "test_video.webm" + + with SentienceBrowser( + headless=True, + record_video_dir=video_dir # Pass Path object + ) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + video_path = browser.close(output_path=output_path) # Pass Path object + + assert os.path.exists(output_path) + assert video_path == str(output_path) + + +def test_video_recording_multiple_sessions(): + """Test creating multiple video recordings in sequence""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + + video_paths = [] + + # Create 3 video recordings + for i in range(3): + with SentienceBrowser( + headless=True, + record_video_dir=str(video_dir) + ) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + + output_path = video_dir / f"video_{i}.webm" + video_path = browser.close(output_path=str(output_path)) + video_paths.append(video_path) + + # Verify all videos were created + for video_path in video_paths: + assert os.path.exists(video_path) + + +def test_video_recording_default_resolution(): + """Test that default resolution is 1280x800""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + + browser = SentienceBrowser( + headless=True, + record_video_dir=str(video_dir) + ) + + # Verify default resolution + assert browser.record_video_size == {"width": 1280, "height": 800} + + browser.start() + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + browser.close() From cb669ee7f99a1c48216050669034c4101ac9db4e Mon Sep 17 00:00:00 2001 From: rcholic Date: Mon, 29 Dec 2025 17:42:52 -0800 Subject: [PATCH 3/5] examples & tests --- examples/video_recording_advanced.py | 7 +++-- examples/video_recording_demo.py | 3 +- tests/test_video_recording.py | 42 ++++++++-------------------- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/examples/video_recording_advanced.py b/examples/video_recording_advanced.py index e471829..c001490 100644 --- a/examples/video_recording_advanced.py +++ b/examples/video_recording_advanced.py @@ -7,9 +7,10 @@ - Multiple recordings in one session """ -from sentience import SentienceBrowser -from pathlib import Path from datetime import datetime +from pathlib import Path + +from sentience import SentienceBrowser def main(): @@ -25,7 +26,7 @@ def main(): with SentienceBrowser( record_video_dir=str(video_dir), - record_video_size={"width": 1920, "height": 1080} # 1080p resolution + record_video_size={"width": 1920, "height": 1080}, # 1080p resolution ) as browser: print(" Resolution: 1920x1080") browser.page.goto("https://example.com") diff --git a/examples/video_recording_demo.py b/examples/video_recording_demo.py index d1edae0..4e78eb7 100644 --- a/examples/video_recording_demo.py +++ b/examples/video_recording_demo.py @@ -5,9 +5,10 @@ to capture browser automation sessions. """ -from sentience import SentienceBrowser from pathlib import Path +from sentience import SentienceBrowser + def main(): # Create output directory for videos diff --git a/tests/test_video_recording.py b/tests/test_video_recording.py index f919c31..77984dc 100644 --- a/tests/test_video_recording.py +++ b/tests/test_video_recording.py @@ -3,10 +3,11 @@ """ import os -import pytest -from pathlib import Path -import tempfile import shutil +import tempfile +from pathlib import Path + +import pytest from sentience import SentienceBrowser @@ -16,10 +17,7 @@ def test_video_recording_basic(): with tempfile.TemporaryDirectory() as temp_dir: video_dir = Path(temp_dir) / "recordings" - with SentienceBrowser( - headless=True, - record_video_dir=str(video_dir) - ) as browser: + with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -43,7 +41,7 @@ def test_video_recording_custom_resolution(): with SentienceBrowser( headless=True, record_video_dir=str(video_dir), - record_video_size={"width": 1920, "height": 1080} + record_video_size={"width": 1920, "height": 1080}, ) as browser: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -60,10 +58,7 @@ def test_video_recording_custom_output_path(): video_dir = Path(temp_dir) / "recordings" custom_path = video_dir / "my_recording.webm" - with SentienceBrowser( - headless=True, - record_video_dir=str(video_dir) - ) as browser: + with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -80,10 +75,7 @@ def test_video_recording_nested_output_path(): video_dir = Path(temp_dir) / "recordings" nested_path = video_dir / "project" / "tutorials" / "video1.webm" - with SentienceBrowser( - headless=True, - record_video_dir=str(video_dir) - ) as browser: + with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -113,10 +105,7 @@ def test_video_recording_directory_auto_created(): # Use a non-existent directory video_dir = Path(temp_dir) / "new_recordings" / "subdir" - with SentienceBrowser( - headless=True, - record_video_dir=str(video_dir) - ) as browser: + with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -135,8 +124,7 @@ def test_video_recording_with_pathlib(): output_path = video_dir / "test_video.webm" with SentienceBrowser( - headless=True, - record_video_dir=video_dir # Pass Path object + headless=True, record_video_dir=video_dir # Pass Path object ) as browser: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -156,10 +144,7 @@ def test_video_recording_multiple_sessions(): # Create 3 video recordings for i in range(3): - with SentienceBrowser( - headless=True, - record_video_dir=str(video_dir) - ) as browser: + with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -177,10 +162,7 @@ def test_video_recording_default_resolution(): with tempfile.TemporaryDirectory() as temp_dir: video_dir = Path(temp_dir) / "recordings" - browser = SentienceBrowser( - headless=True, - record_video_dir=str(video_dir) - ) + browser = SentienceBrowser(headless=True, record_video_dir=str(video_dir)) # Verify default resolution assert browser.record_video_size == {"width": 1280, "height": 800} From d71c96d9f6bd29d5fa786181373cdb469413a8a2 Mon Sep 17 00:00:00 2001 From: rcholic Date: Mon, 29 Dec 2025 17:51:35 -0800 Subject: [PATCH 4/5] fix tests --- tests/test_video_recording.py | 103 ++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 16 deletions(-) diff --git a/tests/test_video_recording.py b/tests/test_video_recording.py index 77984dc..445bc25 100644 --- a/tests/test_video_recording.py +++ b/tests/test_video_recording.py @@ -3,7 +3,6 @@ """ import os -import shutil import tempfile from pathlib import Path @@ -17,7 +16,10 @@ def test_video_recording_basic(): with tempfile.TemporaryDirectory() as temp_dir: video_dir = Path(temp_dir) / "recordings" - with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: + browser = SentienceBrowser(headless=True, record_video_dir=str(video_dir)) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -31,6 +33,9 @@ def test_video_recording_basic(): # Verify file has content file_size = os.path.getsize(video_path) assert file_size > 0 + except Exception: + browser.close() + raise def test_video_recording_custom_resolution(): @@ -38,11 +43,14 @@ def test_video_recording_custom_resolution(): with tempfile.TemporaryDirectory() as temp_dir: video_dir = Path(temp_dir) / "recordings" - with SentienceBrowser( + browser = SentienceBrowser( headless=True, record_video_dir=str(video_dir), - record_video_size={"width": 1920, "height": 1080}, - ) as browser: + record_video_size={"width": 1920, "height": 1080} + ) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -50,6 +58,9 @@ def test_video_recording_custom_resolution(): assert video_path is not None assert os.path.exists(video_path) + except Exception: + browser.close() + raise def test_video_recording_custom_output_path(): @@ -58,7 +69,10 @@ def test_video_recording_custom_output_path(): video_dir = Path(temp_dir) / "recordings" custom_path = video_dir / "my_recording.webm" - with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: + browser = SentienceBrowser(headless=True, record_video_dir=str(video_dir)) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -67,6 +81,9 @@ def test_video_recording_custom_output_path(): # Verify video was renamed to custom path assert video_path == str(custom_path) assert os.path.exists(custom_path) + except Exception: + browser.close() + raise def test_video_recording_nested_output_path(): @@ -75,7 +92,10 @@ def test_video_recording_nested_output_path(): video_dir = Path(temp_dir) / "recordings" nested_path = video_dir / "project" / "tutorials" / "video1.webm" - with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: + browser = SentienceBrowser(headless=True, record_video_dir=str(video_dir)) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -85,11 +105,17 @@ def test_video_recording_nested_output_path(): assert video_path == str(nested_path) assert os.path.exists(nested_path) assert nested_path.parent.exists() + except Exception: + browser.close() + raise def test_no_video_recording_when_disabled(): """Test that no video is created when recording is disabled""" - with SentienceBrowser(headless=True) as browser: + browser = SentienceBrowser(headless=True) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -97,6 +123,9 @@ def test_no_video_recording_when_disabled(): # Should return None when recording is disabled assert video_path is None + except Exception: + browser.close() + raise def test_video_recording_directory_auto_created(): @@ -105,7 +134,10 @@ def test_video_recording_directory_auto_created(): # Use a non-existent directory video_dir = Path(temp_dir) / "new_recordings" / "subdir" - with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: + browser = SentienceBrowser(headless=True, record_video_dir=str(video_dir)) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -115,6 +147,9 @@ def test_video_recording_directory_auto_created(): assert video_dir.exists() assert video_path is not None assert os.path.exists(video_path) + except Exception: + browser.close() + raise def test_video_recording_with_pathlib(): @@ -123,9 +158,13 @@ def test_video_recording_with_pathlib(): video_dir = Path(temp_dir) / "recordings" output_path = video_dir / "test_video.webm" - with SentienceBrowser( - headless=True, record_video_dir=video_dir # Pass Path object - ) as browser: + browser = SentienceBrowser( + headless=True, + record_video_dir=video_dir # Pass Path object + ) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") @@ -133,6 +172,9 @@ def test_video_recording_with_pathlib(): assert os.path.exists(output_path) assert video_path == str(output_path) + except Exception: + browser.close() + raise def test_video_recording_multiple_sessions(): @@ -144,13 +186,19 @@ def test_video_recording_multiple_sessions(): # Create 3 video recordings for i in range(3): - with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: + browser = SentienceBrowser(headless=True, record_video_dir=str(video_dir)) + browser.start() + + try: browser.page.goto("https://example.com") browser.page.wait_for_load_state("networkidle") output_path = video_dir / f"video_{i}.webm" video_path = browser.close(output_path=str(output_path)) video_paths.append(video_path) + except Exception: + browser.close() + raise # Verify all videos were created for video_path in video_paths: @@ -168,6 +216,29 @@ def test_video_recording_default_resolution(): assert browser.record_video_size == {"width": 1280, "height": 800} browser.start() - browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") - browser.close() + + try: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + browser.close() + except Exception: + browser.close() + raise + + +def test_video_recording_with_context_manager(): + """Test that context manager works when NOT calling close() manually""" + with tempfile.TemporaryDirectory() as temp_dir: + video_dir = Path(temp_dir) / "recordings" + + # Use context manager WITHOUT calling close() manually + with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: + browser.page.goto("https://example.com") + browser.page.wait_for_load_state("networkidle") + # Don't call browser.close() - let context manager handle it + + # Verify video was created after context manager exits + # Find the .webm file in the directory + webm_files = list(video_dir.glob("*.webm")) + assert len(webm_files) > 0 + assert os.path.exists(webm_files[0]) From 1c5f0697ffc50589591214f6f2d56ca77b3e1738 Mon Sep 17 00:00:00 2001 From: rcholic Date: Mon, 29 Dec 2025 18:09:56 -0800 Subject: [PATCH 5/5] fix tests --- tests/test_video_recording.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_video_recording.py b/tests/test_video_recording.py index 445bc25..db64f5c 100644 --- a/tests/test_video_recording.py +++ b/tests/test_video_recording.py @@ -21,7 +21,7 @@ def test_video_recording_basic(): try: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") video_path = browser.close() @@ -52,7 +52,7 @@ def test_video_recording_custom_resolution(): try: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") video_path = browser.close() @@ -74,7 +74,7 @@ def test_video_recording_custom_output_path(): try: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") video_path = browser.close(output_path=str(custom_path)) @@ -97,7 +97,7 @@ def test_video_recording_nested_output_path(): try: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") video_path = browser.close(output_path=str(nested_path)) @@ -139,7 +139,7 @@ def test_video_recording_directory_auto_created(): try: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") video_path = browser.close() @@ -166,7 +166,7 @@ def test_video_recording_with_pathlib(): try: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") video_path = browser.close(output_path=output_path) # Pass Path object @@ -219,7 +219,7 @@ def test_video_recording_default_resolution(): try: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") browser.close() except Exception: browser.close() @@ -234,7 +234,7 @@ def test_video_recording_with_context_manager(): # Use context manager WITHOUT calling close() manually with SentienceBrowser(headless=True, record_video_dir=str(video_dir)) as browser: browser.page.goto("https://example.com") - browser.page.wait_for_load_state("networkidle") + browser.page.wait_for_load_state("domcontentloaded") # Don't call browser.close() - let context manager handle it # Verify video was created after context manager exits