make jpeg default frame format in artifacts

examples/agent_runtime_captcha_strategies.py

@@ -30,18 +30,16 @@ async def main() -> None:
         )
 
         # Option 1: Human-in-loop
-        runtime.set_captcha_options(
-            CaptchaOptions(policy="callback", handler=HumanHandoffSolver())
-        )
+        runtime.set_captcha_options(CaptchaOptions(policy="callback", handler=HumanHandoffSolver()))
 
         # Option 2: Vision-only verification (no actions)
-        runtime.set_captcha_options(
-            CaptchaOptions(policy="callback", handler=VisionSolver())
-        )
+        runtime.set_captcha_options(CaptchaOptions(policy="callback", handler=VisionSolver()))
 
         # Option 3: External resolver orchestration
         runtime.set_captcha_options(
-            CaptchaOptions(policy="callback", handler=ExternalSolver(lambda ctx: notify_webhook(ctx)))
+            CaptchaOptions(
+                policy="callback", handler=ExternalSolver(lambda ctx: notify_webhook(ctx))
+            )
         )
 
         await page.goto(os.environ.get("CAPTCHA_TEST_URL", "https://example.com"))

sentience/agent_runtime.py

@@ -426,10 +426,14 @@ async def _capture_artifact_frame(self) -> None:
         if not self._artifact_buffer:
             return
         try:
-            image_bytes = await self.backend.screenshot_png()
+            fmt = self._artifact_buffer.options.frame_format
+            if fmt == "jpeg":
+                image_bytes = await self.backend.screenshot_jpeg()
+            else:
+                image_bytes = await self.backend.screenshot_png()
         except Exception:
             return
-        self._artifact_buffer.add_frame(image_bytes, fmt="png")
+        self._artifact_buffer.add_frame(image_bytes, fmt=fmt)
 
     async def _artifact_timer_loop(self) -> None:
         if not self._artifact_buffer:

sentience/backends/cdp_backend.py

@@ -256,6 +256,20 @@ async def screenshot_png(self) -> bytes:
         data = result.get("data", "")
         return base64.b64decode(data)
 
+    async def screenshot_jpeg(self, quality: int | None = None) -> bytes:
+        """Capture viewport screenshot as JPEG bytes."""
+        q = 80 if quality is None else max(1, min(int(quality), 100))
+        result = await self._transport.send(
+            "Page.captureScreenshot",
+            {
+                "format": "jpeg",
+                "quality": q,
+                "captureBeyondViewport": False,
+            },
+        )
+        data = result.get("data", "")
+        return base64.b64decode(data)
+
     async def mouse_move(self, x: float, y: float) -> None:
         """Move mouse to viewport coordinates."""
         await self._transport.send(

sentience/backends/playwright_backend.py

@@ -126,6 +126,11 @@ async def screenshot_png(self) -> bytes:
         """Capture viewport screenshot as PNG bytes."""
         return await self._page.screenshot(type="png")
 
+    async def screenshot_jpeg(self, quality: int | None = None) -> bytes:
+        """Capture viewport screenshot as JPEG bytes."""
+        q = 80 if quality is None else max(1, min(int(quality), 100))
+        return await self._page.screenshot(type="jpeg", quality=q)
+
     async def mouse_move(self, x: float, y: float) -> None:
         """Move mouse to viewport coordinates."""
         await self._page.mouse.move(x, y)

sentience/backends/protocol.py

@@ -126,6 +126,18 @@ async def screenshot_png(self) -> bytes:
         """
         ...
 
+    async def screenshot_jpeg(self, quality: int | None = None) -> bytes:
+        """
+        Capture viewport screenshot as JPEG bytes.
+
+        Args:
+            quality: Optional JPEG quality (1-100)
+
+        Returns:
+            JPEG image bytes
+        """
+        ...
+
     async def mouse_move(self, x: float, y: float) -> None:
         """
         Move mouse to viewport coordinates.

sentience/captcha.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+from collections.abc import Awaitable, Callable
 from dataclasses import dataclass
-from typing import Awaitable, Callable, Literal, Optional
+from typing import Literal, Optional
 
 from .models import CaptchaDiagnostics
 
@@ -17,20 +18,20 @@ class CaptchaContext:
     url: str
     source: CaptchaSource
     captcha: CaptchaDiagnostics
-    screenshot_path: Optional[str] = None
-    frames_dir: Optional[str] = None
-    snapshot_path: Optional[str] = None
-    live_session_url: Optional[str] = None
-    meta: Optional[dict[str, str]] = None
+    screenshot_path: str | None = None
+    frames_dir: str | None = None
+    snapshot_path: str | None = None
+    live_session_url: str | None = None
+    meta: dict[str, str] | None = None
 
 
 @dataclass
 class CaptchaResolution:
     action: CaptchaAction
-    message: Optional[str] = None
-    handled_by: Optional[Literal["human", "customer_system", "unknown"]] = None
-    timeout_ms: Optional[int] = None
-    poll_ms: Optional[int] = None
+    message: str | None = None
+    handled_by: Literal["human", "customer_system", "unknown"] | None = None
+    timeout_ms: int | None = None
+    poll_ms: int | None = None
 
 
 CaptchaHandler = Callable[[CaptchaContext], CaptchaResolution | Awaitable[CaptchaResolution]]
@@ -43,8 +44,8 @@ class CaptchaOptions:
     timeout_ms: int = 120_000
     poll_ms: int = 1_000
     max_retries_new_session: int = 1
-    handler: Optional[CaptchaHandler] = None
-    reset_session: Optional[Callable[[], Awaitable[None]]] = None
+    handler: CaptchaHandler | None = None
+    reset_session: Callable[[], Awaitable[None]] | None = None
 
 
 class CaptchaHandlingError(RuntimeError):

sentience/captcha_strategies.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import inspect
-from typing import Callable
+from collections.abc import Callable
 
 from .captcha import CaptchaContext, CaptchaHandler, CaptchaResolution
 

sentience/extension/background.js

@@ -28,14 +28,14 @@ async function handleSnapshotProcessing(rawData, options = {}) {
     const startTime = performance.now();
     try {
         if (!Array.isArray(rawData)) throw new Error("rawData must be an array");
-        if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(), 
+        if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(),
         !wasmReady) throw new Error("WASM module not initialized");
         let analyzedElements, prunedRawData;
         try {
             const wasmPromise = new Promise((resolve, reject) => {
                 try {
                     let result;
-                    result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData), 
+                    result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData),
                     resolve(result);
                 } catch (e) {
                     reject(e);
@@ -101,4 +101,4 @@ initWASM().catch(err => {}), chrome.runtime.onMessage.addListener((request, send
     event.preventDefault();
 }), self.addEventListener("unhandledrejection", event => {
     event.preventDefault();
-});
+});

sentience/extension/content.js

@@ -82,7 +82,7 @@
                 if (!elements || !Array.isArray(elements)) return;
                 removeOverlay();
                 const host = document.createElement("div");
-                host.id = OVERLAY_HOST_ID, host.style.cssText = "\n        position: fixed !important;\n        top: 0 !important;\n        left: 0 !important;\n        width: 100vw !important;\n        height: 100vh !important;\n        pointer-events: none !important;\n        z-index: 2147483647 !important;\n        margin: 0 !important;\n        padding: 0 !important;\n    ", 
+                host.id = OVERLAY_HOST_ID, host.style.cssText = "\n        position: fixed !important;\n        top: 0 !important;\n        left: 0 !important;\n        width: 100vw !important;\n        height: 100vh !important;\n        pointer-events: none !important;\n        z-index: 2147483647 !important;\n        margin: 0 !important;\n        padding: 0 !important;\n    ",
                 document.body.appendChild(host);
                 const shadow = host.attachShadow({
                     mode: "closed"
@@ -94,15 +94,15 @@
                     let color;
                     color = isTarget ? "#FF0000" : isPrimary ? "#0066FF" : "#00FF00";
                     const importanceRatio = maxImportance > 0 ? importance / maxImportance : .5, borderOpacity = isTarget ? 1 : isPrimary ? .9 : Math.max(.4, .5 + .5 * importanceRatio), fillOpacity = .2 * borderOpacity, borderWidth = isTarget ? 2 : isPrimary ? 1.5 : Math.max(.5, Math.round(2 * importanceRatio)), hexOpacity = Math.round(255 * fillOpacity).toString(16).padStart(2, "0"), box = document.createElement("div");
-                    if (box.style.cssText = `\n            position: absolute;\n            left: ${bbox.x}px;\n            top: ${bbox.y}px;\n            width: ${bbox.width}px;\n            height: ${bbox.height}px;\n            border: ${borderWidth}px solid ${color};\n            background-color: ${color}${hexOpacity};\n            box-sizing: border-box;\n            opacity: ${borderOpacity};\n            pointer-events: none;\n        `, 
+                    if (box.style.cssText = `\n            position: absolute;\n            left: ${bbox.x}px;\n            top: ${bbox.y}px;\n            width: ${bbox.width}px;\n            height: ${bbox.height}px;\n            border: ${borderWidth}px solid ${color};\n            background-color: ${color}${hexOpacity};\n            box-sizing: border-box;\n            opacity: ${borderOpacity};\n            pointer-events: none;\n        `,
                     importance > 0 || isPrimary) {
                         const badge = document.createElement("span");
-                        badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n                position: absolute;\n                top: -18px;\n                left: 0;\n                background: ${color};\n                color: white;\n                font-size: 11px;\n                font-weight: bold;\n                padding: 2px 6px;\n                font-family: Arial, sans-serif;\n                border-radius: 3px;\n                opacity: 0.95;\n                white-space: nowrap;\n                pointer-events: none;\n            `, 
+                        badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n                position: absolute;\n                top: -18px;\n                left: 0;\n                background: ${color};\n                color: white;\n                font-size: 11px;\n                font-weight: bold;\n                padding: 2px 6px;\n                font-family: Arial, sans-serif;\n                border-radius: 3px;\n                opacity: 0.95;\n                white-space: nowrap;\n                pointer-events: none;\n            `,
                         box.appendChild(badge);
                     }
                     if (isTarget) {
                         const targetIndicator = document.createElement("span");
-                        targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n                position: absolute;\n                top: -18px;\n                right: 0;\n                font-size: 16px;\n                pointer-events: none;\n            ", 
+                        targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n                position: absolute;\n                top: -18px;\n                right: 0;\n                font-size: 16px;\n                pointer-events: none;\n            ",
                         box.appendChild(targetIndicator);
                     }
                     shadow.appendChild(box);
@@ -122,7 +122,7 @@
                 if (!grids || !Array.isArray(grids)) return;
                 removeOverlay();
                 const host = document.createElement("div");
-                host.id = OVERLAY_HOST_ID, host.style.cssText = "\n        position: fixed !important;\n        top: 0 !important;\n        left: 0 !important;\n        width: 100vw !important;\n        height: 100vh !important;\n        pointer-events: none !important;\n        z-index: 2147483647 !important;\n        margin: 0 !important;\n        padding: 0 !important;\n    ", 
+                host.id = OVERLAY_HOST_ID, host.style.cssText = "\n        position: fixed !important;\n        top: 0 !important;\n        left: 0 !important;\n        width: 100vw !important;\n        height: 100vh !important;\n        pointer-events: none !important;\n        z-index: 2147483647 !important;\n        margin: 0 !important;\n        padding: 0 !important;\n    ",
                 document.body.appendChild(host);
                 const shadow = host.attachShadow({
                     mode: "closed"
@@ -138,10 +138,10 @@
                     let labelText = grid.label ? `Grid ${grid.grid_id}: ${grid.label}` : `Grid ${grid.grid_id}`;
                     grid.is_dominant && (labelText = `⭐ ${labelText} (dominant)`);
                     const badge = document.createElement("span");
-                    if (badge.textContent = labelText, badge.style.cssText = `\n                position: absolute;\n                top: -18px;\n                left: 0;\n                background: ${color};\n                color: white;\n                font-size: 11px;\n                font-weight: bold;\n                padding: 2px 6px;\n                font-family: Arial, sans-serif;\n                border-radius: 3px;\n                opacity: 0.95;\n                white-space: nowrap;\n                pointer-events: none;\n            `, 
+                    if (badge.textContent = labelText, badge.style.cssText = `\n                position: absolute;\n                top: -18px;\n                left: 0;\n                background: ${color};\n                color: white;\n                font-size: 11px;\n                font-weight: bold;\n                padding: 2px 6px;\n                font-family: Arial, sans-serif;\n                border-radius: 3px;\n                opacity: 0.95;\n                white-space: nowrap;\n                pointer-events: none;\n            `,
                     box.appendChild(badge), isTarget) {
                         const targetIndicator = document.createElement("span");
-                        targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n                position: absolute;\n                top: -18px;\n                right: 0;\n                font-size: 16px;\n                pointer-events: none;\n            ", 
+                        targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n                position: absolute;\n                top: -18px;\n                right: 0;\n                font-size: 16px;\n                pointer-events: none;\n            ",
                         box.appendChild(targetIndicator);
                     }
                     shadow.appendChild(box);
@@ -155,7 +155,7 @@
     let overlayTimeout = null;
     function removeOverlay() {
         const existing = document.getElementById(OVERLAY_HOST_ID);
-        existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout), 
+        existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout),
         overlayTimeout = null);
     }
-}();
+}();