Skip to content

Commit ddd2bbc

Browse files
rcholicrcholic
authored
Merge pull request #99 from SentienceAPI/more_fix_trace
align trace and indexing file data format
2 parents 8f7d226 + 06ca7f7 commit ddd2bbc

File tree

11 files changed

+943
-159
lines changed

11 files changed

+943
-159
lines changed

‎sentience/agent.py‎

Lines changed: 201 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
"""
55

66
import asyncio
7+
import hashlib
78
import re
89
import time
910
from typing import TYPE_CHECKING, Any, Optional
@@ -95,6 +96,24 @@ def __init__(
9596
# Step counter for tracing
9697
self._step_count = 0
9798

99+
def _compute_hash(self, text: str) -> str:
100+
"""Compute SHA256 hash of text."""
101+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
102+
103+
def _get_element_bbox(self, element_id: int | None, snap: Snapshot) -> dict[str, float] | None:
104+
"""Get bounding box for an element from snapshot."""
105+
if element_id is None:
106+
return None
107+
for el in snap.elements:
108+
if el.id == element_id:
109+
return {
110+
"x": el.bbox.x,
111+
"y": el.bbox.y,
112+
"width": el.bbox.width,
113+
"height": el.bbox.height,
114+
}
115+
return None
116+
98117
def act( # noqa: C901
99118
self,
100119
goal: str,
@@ -343,15 +362,99 @@ def act( # noqa: C901
343362

344363
# Emit step completion trace event if tracer is enabled
345364
if self.tracer:
346-
self.tracer.emit(
347-
"step_end",
348-
{
349-
"success": result.success,
350-
"duration_ms": duration_ms,
351-
"action": result.action,
365+
# Get pre_url from step_start (stored in tracer or use current)
366+
pre_url = snap.url
367+
post_url = self.browser.page.url if self.browser.page else None
368+
369+
# Compute snapshot digest (simplified - use URL + timestamp)
370+
snapshot_digest = f"sha256:{self._compute_hash(f'{pre_url}{snap.timestamp}')}"
371+
372+
# Build LLM data
373+
llm_response_text = llm_response.content
374+
llm_response_hash = f"sha256:{self._compute_hash(llm_response_text)}"
375+
llm_data = {
376+
"response_text": llm_response_text,
377+
"response_hash": llm_response_hash,
378+
"usage": {
379+
"prompt_tokens": llm_response.prompt_tokens or 0,
380+
"completion_tokens": llm_response.completion_tokens or 0,
381+
"total_tokens": llm_response.total_tokens or 0,
352382
},
353-
step_id=step_id,
383+
}
384+
385+
# Build exec data
386+
exec_data = {
387+
"success": result.success,
388+
"action": result.action,
389+
"outcome": result.outcome
390+
or (
391+
f"Action {result.action} executed successfully"
392+
if result.success
393+
else f"Action {result.action} failed"
394+
),
395+
"duration_ms": duration_ms,
396+
}
397+
398+
# Add optional exec fields
399+
if result.element_id is not None:
400+
exec_data["element_id"] = result.element_id
401+
# Add bounding box if element found
402+
bbox = self._get_element_bbox(result.element_id, snap)
403+
if bbox:
404+
exec_data["bounding_box"] = bbox
405+
if result.text is not None:
406+
exec_data["text"] = result.text
407+
if result.key is not None:
408+
exec_data["key"] = result.key
409+
if result.error is not None:
410+
exec_data["error"] = result.error
411+
412+
# Build verify data (simplified - based on success and url_changed)
413+
verify_passed = result.success and (
414+
result.url_changed or result.action != "click"
354415
)
416+
verify_signals = {
417+
"url_changed": result.url_changed or False,
418+
}
419+
if result.error:
420+
verify_signals["error"] = result.error
421+
422+
# Add elements_found array if element was targeted
423+
if result.element_id is not None:
424+
bbox = self._get_element_bbox(result.element_id, snap)
425+
if bbox:
426+
verify_signals["elements_found"] = [
427+
{
428+
"label": f"Element {result.element_id}",
429+
"bounding_box": bbox,
430+
}
431+
]
432+
433+
verify_data = {
434+
"passed": verify_passed,
435+
"signals": verify_signals,
436+
}
437+
438+
# Build complete step_end event
439+
step_end_data = {
440+
"v": 1,
441+
"step_id": step_id,
442+
"step_index": self._step_count,
443+
"goal": goal,
444+
"attempt": attempt,
445+
"pre": {
446+
"url": pre_url,
447+
"snapshot_digest": snapshot_digest,
448+
},
449+
"llm": llm_data,
450+
"exec": exec_data,
451+
"post": {
452+
"url": post_url,
453+
},
454+
"verify": verify_data,
455+
}
456+
457+
self.tracer.emit("step_end", step_end_data, step_id=step_id)
355458

356459
return result
357460

@@ -1026,15 +1129,99 @@ async def act( # noqa: C901
10261129

10271130
# Emit step completion trace event if tracer is enabled
10281131
if self.tracer:
1029-
self.tracer.emit(
1030-
"step_end",
1031-
{
1032-
"success": result.success,
1033-
"duration_ms": duration_ms,
1034-
"action": result.action,
1132+
# Get pre_url from step_start (stored in tracer or use current)
1133+
pre_url = snap.url
1134+
post_url = self.browser.page.url if self.browser.page else None
1135+
1136+
# Compute snapshot digest (simplified - use URL + timestamp)
1137+
snapshot_digest = f"sha256:{self._compute_hash(f'{pre_url}{snap.timestamp}')}"
1138+
1139+
# Build LLM data
1140+
llm_response_text = llm_response.content
1141+
llm_response_hash = f"sha256:{self._compute_hash(llm_response_text)}"
1142+
llm_data = {
1143+
"response_text": llm_response_text,
1144+
"response_hash": llm_response_hash,
1145+
"usage": {
1146+
"prompt_tokens": llm_response.prompt_tokens or 0,
1147+
"completion_tokens": llm_response.completion_tokens or 0,
1148+
"total_tokens": llm_response.total_tokens or 0,
10351149
},
1036-
step_id=step_id,
1150+
}
1151+
1152+
# Build exec data
1153+
exec_data = {
1154+
"success": result.success,
1155+
"action": result.action,
1156+
"outcome": result.outcome
1157+
or (
1158+
f"Action {result.action} executed successfully"
1159+
if result.success
1160+
else f"Action {result.action} failed"
1161+
),
1162+
"duration_ms": duration_ms,
1163+
}
1164+
1165+
# Add optional exec fields
1166+
if result.element_id is not None:
1167+
exec_data["element_id"] = result.element_id
1168+
# Add bounding box if element found
1169+
bbox = self._get_element_bbox(result.element_id, snap)
1170+
if bbox:
1171+
exec_data["bounding_box"] = bbox
1172+
if result.text is not None:
1173+
exec_data["text"] = result.text
1174+
if result.key is not None:
1175+
exec_data["key"] = result.key
1176+
if result.error is not None:
1177+
exec_data["error"] = result.error
1178+
1179+
# Build verify data (simplified - based on success and url_changed)
1180+
verify_passed = result.success and (
1181+
result.url_changed or result.action != "click"
10371182
)
1183+
verify_signals = {
1184+
"url_changed": result.url_changed or False,
1185+
}
1186+
if result.error:
1187+
verify_signals["error"] = result.error
1188+
1189+
# Add elements_found array if element was targeted
1190+
if result.element_id is not None:
1191+
bbox = self._get_element_bbox(result.element_id, snap)
1192+
if bbox:
1193+
verify_signals["elements_found"] = [
1194+
{
1195+
"label": f"Element {result.element_id}",
1196+
"bounding_box": bbox,
1197+
}
1198+
]
1199+
1200+
verify_data = {
1201+
"passed": verify_passed,
1202+
"signals": verify_signals,
1203+
}
1204+
1205+
# Build complete step_end event
1206+
step_end_data = {
1207+
"v": 1,
1208+
"step_id": step_id,
1209+
"step_index": self._step_count,
1210+
"goal": goal,
1211+
"attempt": attempt,
1212+
"pre": {
1213+
"url": pre_url,
1214+
"snapshot_digest": snapshot_digest,
1215+
},
1216+
"llm": llm_data,
1217+
"exec": exec_data,
1218+
"post": {
1219+
"url": post_url,
1220+
},
1221+
"verify": verify_data,
1222+
}
1223+
1224+
self.tracer.emit("step_end", step_end_data, step_id=step_id)
10381225

10391226
return result
10401227

‎sentience/extension/background.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -144,13 +144,13 @@ async function handleScreenshotCapture(_tabId, options = {}) {
144144
async function handleSnapshotProcessing(rawData, options = {}) {
145145
const MAX_ELEMENTS = 10000; // Safety limit to prevent hangs
146146
const startTime = performance.now();
147-
147+
148148
try {
149149
// Safety check: limit element count to prevent hangs
150150
if (!Array.isArray(rawData)) {
151151
throw new Error('rawData must be an array');
152152
}
153-
153+
154154
if (rawData.length > MAX_ELEMENTS) {
155155
console.warn(`[Sentience Background] ⚠️ Large dataset: ${rawData.length} elements. Limiting to ${MAX_ELEMENTS} to prevent hangs.`);
156156
rawData = rawData.slice(0, MAX_ELEMENTS);
@@ -186,7 +186,7 @@ async function handleSnapshotProcessing(rawData, options = {}) {
186186
// Add timeout protection (18 seconds - less than content.js timeout)
187187
analyzedElements = await Promise.race([
188188
wasmPromise,
189-
new Promise((_, reject) =>
189+
new Promise((_, reject) =>
190190
setTimeout(() => reject(new Error('WASM processing timeout (>18s)')), 18000)
191191
)
192192
]);

‎sentience/extension/content.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ function handleSnapshotRequest(data) {
9292
if (responded) return; // Already responded via timeout
9393
responded = true;
9494
clearTimeout(timeoutId);
95-
95+
9696
const duration = performance.now() - startTime;
9797

9898
// Handle Chrome extension errors (e.g., background script crashed)

0 commit comments

Comments
 (0)