Skip to content

Commit e909bec

Browse files
rcholicrcholic
authored
Merge pull request #171 from SentienceAPI/tweaking_fixes
make jpeg default frame format in artifacts
2 parents ece4fb8 + 9e662e8 commit e909bec

File tree

12 files changed

+120
-81
lines changed

12 files changed

+120
-81
lines changed

‎examples/agent_runtime_captcha_strategies.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,18 +30,16 @@ async def main() -> None:
3030
)
3131

3232
# Option 1: Human-in-loop
33-
runtime.set_captcha_options(
34-
CaptchaOptions(policy="callback", handler=HumanHandoffSolver())
35-
)
33+
runtime.set_captcha_options(CaptchaOptions(policy="callback", handler=HumanHandoffSolver()))
3634

3735
# Option 2: Vision-only verification (no actions)
38-
runtime.set_captcha_options(
39-
CaptchaOptions(policy="callback", handler=VisionSolver())
40-
)
36+
runtime.set_captcha_options(CaptchaOptions(policy="callback", handler=VisionSolver()))
4137

4238
# Option 3: External resolver orchestration
4339
runtime.set_captcha_options(
44-
CaptchaOptions(policy="callback", handler=ExternalSolver(lambda ctx: notify_webhook(ctx)))
40+
CaptchaOptions(
41+
policy="callback", handler=ExternalSolver(lambda ctx: notify_webhook(ctx))
42+
)
4543
)
4644

4745
await page.goto(os.environ.get("CAPTCHA_TEST_URL", "https://example.com"))

‎sentience/agent_runtime.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -426,10 +426,14 @@ async def _capture_artifact_frame(self) -> None:
426426
if not self._artifact_buffer:
427427
return
428428
try:
429-
image_bytes = await self.backend.screenshot_png()
429+
fmt = self._artifact_buffer.options.frame_format
430+
if fmt == "jpeg":
431+
image_bytes = await self.backend.screenshot_jpeg()
432+
else:
433+
image_bytes = await self.backend.screenshot_png()
430434
except Exception:
431435
return
432-
self._artifact_buffer.add_frame(image_bytes, fmt="png")
436+
self._artifact_buffer.add_frame(image_bytes, fmt=fmt)
433437

434438
async def _artifact_timer_loop(self) -> None:
435439
if not self._artifact_buffer:

‎sentience/backends/cdp_backend.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,20 @@ async def screenshot_png(self) -> bytes:
256256
data = result.get("data", "")
257257
return base64.b64decode(data)
258258

259+
async def screenshot_jpeg(self, quality: int | None = None) -> bytes:
260+
"""Capture viewport screenshot as JPEG bytes."""
261+
q = 80 if quality is None else max(1, min(int(quality), 100))
262+
result = await self._transport.send(
263+
"Page.captureScreenshot",
264+
{
265+
"format": "jpeg",
266+
"quality": q,
267+
"captureBeyondViewport": False,
268+
},
269+
)
270+
data = result.get("data", "")
271+
return base64.b64decode(data)
272+
259273
async def mouse_move(self, x: float, y: float) -> None:
260274
"""Move mouse to viewport coordinates."""
261275
await self._transport.send(

‎sentience/backends/playwright_backend.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,11 @@ async def screenshot_png(self) -> bytes:
126126
"""Capture viewport screenshot as PNG bytes."""
127127
return await self._page.screenshot(type="png")
128128

129+
async def screenshot_jpeg(self, quality: int | None = None) -> bytes:
130+
"""Capture viewport screenshot as JPEG bytes."""
131+
q = 80 if quality is None else max(1, min(int(quality), 100))
132+
return await self._page.screenshot(type="jpeg", quality=q)
133+
129134
async def mouse_move(self, x: float, y: float) -> None:
130135
"""Move mouse to viewport coordinates."""
131136
await self._page.mouse.move(x, y)

‎sentience/backends/protocol.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,18 @@ async def screenshot_png(self) -> bytes:
126126
"""
127127
...
128128

129+
async def screenshot_jpeg(self, quality: int | None = None) -> bytes:
130+
"""
131+
Capture viewport screenshot as JPEG bytes.
132+
133+
Args:
134+
quality: Optional JPEG quality (1-100)
135+
136+
Returns:
137+
JPEG image bytes
138+
"""
139+
...
140+
129141
async def mouse_move(self, x: float, y: float) -> None:
130142
"""
131143
Move mouse to viewport coordinates.

‎sentience/captcha.py‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
from __future__ import annotations
22

3+
from collections.abc import Awaitable, Callable
34
from dataclasses import dataclass
4-
from typing import Awaitable, Callable, Literal, Optional
5+
from typing import Literal, Optional
56

67
from .models import CaptchaDiagnostics
78

@@ -17,20 +18,20 @@ class CaptchaContext:
1718
url: str
1819
source: CaptchaSource
1920
captcha: CaptchaDiagnostics
20-
screenshot_path: Optional[str] = None
21-
frames_dir: Optional[str] = None
22-
snapshot_path: Optional[str] = None
23-
live_session_url: Optional[str] = None
24-
meta: Optional[dict[str, str]] = None
21+
screenshot_path: str | None = None
22+
frames_dir: str | None = None
23+
snapshot_path: str | None = None
24+
live_session_url: str | None = None
25+
meta: dict[str, str] | None = None
2526

2627

2728
@dataclass
2829
class CaptchaResolution:
2930
action: CaptchaAction
30-
message: Optional[str] = None
31-
handled_by: Optional[Literal["human", "customer_system", "unknown"]] = None
32-
timeout_ms: Optional[int] = None
33-
poll_ms: Optional[int] = None
31+
message: str | None = None
32+
handled_by: Literal["human", "customer_system", "unknown"] | None = None
33+
timeout_ms: int | None = None
34+
poll_ms: int | None = None
3435

3536

3637
CaptchaHandler = Callable[[CaptchaContext], CaptchaResolution | Awaitable[CaptchaResolution]]
@@ -43,8 +44,8 @@ class CaptchaOptions:
4344
timeout_ms: int = 120_000
4445
poll_ms: int = 1_000
4546
max_retries_new_session: int = 1
46-
handler: Optional[CaptchaHandler] = None
47-
reset_session: Optional[Callable[[], Awaitable[None]]] = None
47+
handler: CaptchaHandler | None = None
48+
reset_session: Callable[[], Awaitable[None]] | None = None
4849

4950

5051
class CaptchaHandlingError(RuntimeError):

‎sentience/captcha_strategies.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from __future__ import annotations
22

33
import inspect
4-
from typing import Callable
4+
from collections.abc import Callable
55

66
from .captcha import CaptchaContext, CaptchaHandler, CaptchaResolution
77

‎sentience/extension/background.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@ async function handleSnapshotProcessing(rawData, options = {}) {
2828
const startTime = performance.now();
2929
try {
3030
if (!Array.isArray(rawData)) throw new Error("rawData must be an array");
31-
if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(),
31+
if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(),
3232
!wasmReady) throw new Error("WASM module not initialized");
3333
let analyzedElements, prunedRawData;
3434
try {
3535
const wasmPromise = new Promise((resolve, reject) => {
3636
try {
3737
let result;
38-
result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData),
38+
result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData),
3939
resolve(result);
4040
} catch (e) {
4141
reject(e);
@@ -101,4 +101,4 @@ initWASM().catch(err => {}), chrome.runtime.onMessage.addListener((request, send
101101
event.preventDefault();
102102
}), self.addEventListener("unhandledrejection", event => {
103103
event.preventDefault();
104-
});
104+
});

‎sentience/extension/content.js‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@
8282
if (!elements || !Array.isArray(elements)) return;
8383
removeOverlay();
8484
const host = document.createElement("div");
85-
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
85+
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
8686
document.body.appendChild(host);
8787
const shadow = host.attachShadow({
8888
mode: "closed"
@@ -94,15 +94,15 @@
9494
let color;
9595
color = isTarget ? "#FF0000" : isPrimary ? "#0066FF" : "#00FF00";
9696
const importanceRatio = maxImportance > 0 ? importance / maxImportance : .5, borderOpacity = isTarget ? 1 : isPrimary ? .9 : Math.max(.4, .5 + .5 * importanceRatio), fillOpacity = .2 * borderOpacity, borderWidth = isTarget ? 2 : isPrimary ? 1.5 : Math.max(.5, Math.round(2 * importanceRatio)), hexOpacity = Math.round(255 * fillOpacity).toString(16).padStart(2, "0"), box = document.createElement("div");
97-
if (box.style.cssText = `\n position: absolute;\n left: ${bbox.x}px;\n top: ${bbox.y}px;\n width: ${bbox.width}px;\n height: ${bbox.height}px;\n border: ${borderWidth}px solid ${color};\n background-color: ${color}${hexOpacity};\n box-sizing: border-box;\n opacity: ${borderOpacity};\n pointer-events: none;\n `,
97+
if (box.style.cssText = `\n position: absolute;\n left: ${bbox.x}px;\n top: ${bbox.y}px;\n width: ${bbox.width}px;\n height: ${bbox.height}px;\n border: ${borderWidth}px solid ${color};\n background-color: ${color}${hexOpacity};\n box-sizing: border-box;\n opacity: ${borderOpacity};\n pointer-events: none;\n `,
9898
importance > 0 || isPrimary) {
9999
const badge = document.createElement("span");
100-
badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
100+
badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
101101
box.appendChild(badge);
102102
}
103103
if (isTarget) {
104104
const targetIndicator = document.createElement("span");
105-
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
105+
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
106106
box.appendChild(targetIndicator);
107107
}
108108
shadow.appendChild(box);
@@ -122,7 +122,7 @@
122122
if (!grids || !Array.isArray(grids)) return;
123123
removeOverlay();
124124
const host = document.createElement("div");
125-
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
125+
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
126126
document.body.appendChild(host);
127127
const shadow = host.attachShadow({
128128
mode: "closed"
@@ -138,10 +138,10 @@
138138
let labelText = grid.label ? `Grid ${grid.grid_id}: ${grid.label}` : `Grid ${grid.grid_id}`;
139139
grid.is_dominant && (labelText = `⭐ ${labelText} (dominant)`);
140140
const badge = document.createElement("span");
141-
if (badge.textContent = labelText, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
141+
if (badge.textContent = labelText, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
142142
box.appendChild(badge), isTarget) {
143143
const targetIndicator = document.createElement("span");
144-
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
144+
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
145145
box.appendChild(targetIndicator);
146146
}
147147
shadow.appendChild(box);
@@ -155,7 +155,7 @@
155155
let overlayTimeout = null;
156156
function removeOverlay() {
157157
const existing = document.getElementById(OVERLAY_HOST_ID);
158-
existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout),
158+
existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout),
159159
overlayTimeout = null);
160160
}
161-
}();
161+
}();

0 commit comments

Comments
 (0)