Skip to content

Commit 24083f2

Browse files
rcholicrcholic
authored
Merge pull request #176 from SentienceAPI/runtime_agent
Agent with integrated Runtime for verification gate
2 parents 1b72c4b + 3582cd6 commit 24083f2

File tree

8 files changed

+913
-59
lines changed

8 files changed

+913
-59
lines changed

‎examples/runtime_agent_minimal.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
"""
2+
Example: RuntimeAgent (AgentRuntime-backed) minimal demo.
3+
4+
This demonstrates the verification-first loop:
5+
snapshot -> propose action (structured executor) -> execute -> verify (AgentRuntime predicates)
6+
7+
Usage:
8+
python examples/runtime_agent_minimal.py
9+
"""
10+
11+
import asyncio
12+
13+
from sentience import AsyncSentienceBrowser
14+
from sentience.agent_runtime import AgentRuntime
15+
from sentience.llm_provider import LLMProvider, LLMResponse
16+
from sentience.runtime_agent import RuntimeAgent, RuntimeStep, StepVerification
17+
from sentience.tracing import JsonlTraceSink, Tracer
18+
from sentience.verification import AssertContext, AssertOutcome, exists, url_contains
19+
20+
21+
class FixedActionProvider(LLMProvider):
22+
"""A tiny in-process provider for examples/tests."""
23+
24+
def __init__(self, action: str):
25+
super().__init__(model="fixed-action")
26+
self._action = action
27+
28+
def generate(self, system_prompt: str, user_prompt: str, **kwargs) -> LLMResponse:
29+
_ = system_prompt, user_prompt, kwargs
30+
return LLMResponse(content=self._action, model_name=self.model_name)
31+
32+
def supports_json_mode(self) -> bool:
33+
return False
34+
35+
@property
36+
def model_name(self) -> str:
37+
return "fixed-action"
38+
39+
40+
async def main() -> None:
41+
# Local trace (viewable in Studio if uploaded later).
42+
run_id = "runtime-agent-minimal"
43+
tracer = Tracer(run_id=run_id, sink=JsonlTraceSink(f"traces/{run_id}.jsonl"))
44+
45+
async with AsyncSentienceBrowser(headless=False) as browser:
46+
page = await browser.new_page()
47+
await page.goto("https://example.com")
48+
await page.wait_for_load_state("networkidle")
49+
50+
runtime = await AgentRuntime.from_sentience_browser(browser=browser, page=page, tracer=tracer)
51+
52+
# Structured executor (for demo, we just return FINISH()).
53+
executor = FixedActionProvider("FINISH()")
54+
55+
agent = RuntimeAgent(
56+
runtime=runtime,
57+
executor=executor,
58+
# vision_executor=... (optional)
59+
# vision_verifier=... (optional, for AgentRuntime assertion vision fallback)
60+
)
61+
62+
# One step: no action needed; we just verify structure + URL.
63+
def has_example_heading(ctx: AssertContext) -> AssertOutcome:
64+
# Demonstrates custom predicates (you can also use exists/url_contains helpers).
65+
snap = ctx.snapshot
66+
ok = bool(snap and any((el.role == "heading" and (el.text or "").startswith("Example")) for el in snap.elements))
67+
return AssertOutcome(passed=ok, reason="" if ok else "missing heading", details={})
68+
69+
step = RuntimeStep(
70+
goal="Confirm Example Domain page is loaded",
71+
verifications=[
72+
StepVerification(predicate=url_contains("example.com"), label="url_contains_example", required=True),
73+
StepVerification(predicate=exists("role=heading"), label="has_heading", required=True),
74+
StepVerification(predicate=has_example_heading, label="heading_text_matches", required=False),
75+
],
76+
max_snapshot_attempts=2,
77+
snapshot_limit_base=60,
78+
)
79+
80+
ok = await agent.run_step(task_goal="Open example.com and verify", step=step)
81+
print(f"step ok: {ok}")
82+
83+
tracer.close()
84+
print(f"trace written to traces/{run_id}.jsonl")
85+
86+
87+
if __name__ == "__main__":
88+
asyncio.run(main())
89+

‎sentience/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
from .query import find, query
9090
from .read import read
9191
from .recorder import Recorder, Trace, TraceStep, record
92+
from .runtime_agent import RuntimeAgent, RuntimeStep, StepVerification
9293
from .screenshot import screenshot
9394
from .sentience_methods import AgentAction, SentienceMethod
9495
from .snapshot import snapshot
@@ -210,6 +211,9 @@
210211
"MLXVLMProvider",
211212
"SentienceAgent",
212213
"SentienceAgentAsync",
214+
"RuntimeAgent",
215+
"RuntimeStep",
216+
"StepVerification",
213217
"SentienceVisualAgent",
214218
"SentienceVisualAgentAsync",
215219
"ConversationalAgent",

‎sentience/extension/background.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@ async function handleSnapshotProcessing(rawData, options = {}) {
2828
const startTime = performance.now();
2929
try {
3030
if (!Array.isArray(rawData)) throw new Error("rawData must be an array");
31-
if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(),
31+
if (rawData.length > 1e4 && (rawData = rawData.slice(0, 1e4)), await initWASM(),
3232
!wasmReady) throw new Error("WASM module not initialized");
3333
let analyzedElements, prunedRawData;
3434
try {
3535
const wasmPromise = new Promise((resolve, reject) => {
3636
try {
3737
let result;
38-
result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData),
38+
result = options.limit || options.filter ? analyze_page_with_options(rawData, options) : analyze_page(rawData),
3939
resolve(result);
4040
} catch (e) {
4141
reject(e);
@@ -101,4 +101,4 @@ initWASM().catch(err => {}), chrome.runtime.onMessage.addListener((request, send
101101
event.preventDefault();
102102
}), self.addEventListener("unhandledrejection", event => {
103103
event.preventDefault();
104-
});
104+
});

‎sentience/extension/content.js‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@
8282
if (!elements || !Array.isArray(elements)) return;
8383
removeOverlay();
8484
const host = document.createElement("div");
85-
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
85+
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
8686
document.body.appendChild(host);
8787
const shadow = host.attachShadow({
8888
mode: "closed"
@@ -94,15 +94,15 @@
9494
let color;
9595
color = isTarget ? "#FF0000" : isPrimary ? "#0066FF" : "#00FF00";
9696
const importanceRatio = maxImportance > 0 ? importance / maxImportance : .5, borderOpacity = isTarget ? 1 : isPrimary ? .9 : Math.max(.4, .5 + .5 * importanceRatio), fillOpacity = .2 * borderOpacity, borderWidth = isTarget ? 2 : isPrimary ? 1.5 : Math.max(.5, Math.round(2 * importanceRatio)), hexOpacity = Math.round(255 * fillOpacity).toString(16).padStart(2, "0"), box = document.createElement("div");
97-
if (box.style.cssText = `\n position: absolute;\n left: ${bbox.x}px;\n top: ${bbox.y}px;\n width: ${bbox.width}px;\n height: ${bbox.height}px;\n border: ${borderWidth}px solid ${color};\n background-color: ${color}${hexOpacity};\n box-sizing: border-box;\n opacity: ${borderOpacity};\n pointer-events: none;\n `,
97+
if (box.style.cssText = `\n position: absolute;\n left: ${bbox.x}px;\n top: ${bbox.y}px;\n width: ${bbox.width}px;\n height: ${bbox.height}px;\n border: ${borderWidth}px solid ${color};\n background-color: ${color}${hexOpacity};\n box-sizing: border-box;\n opacity: ${borderOpacity};\n pointer-events: none;\n `,
9898
importance > 0 || isPrimary) {
9999
const badge = document.createElement("span");
100-
badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
100+
badge.textContent = isPrimary ? `⭐${importance}` : `${importance}`, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
101101
box.appendChild(badge);
102102
}
103103
if (isTarget) {
104104
const targetIndicator = document.createElement("span");
105-
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
105+
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
106106
box.appendChild(targetIndicator);
107107
}
108108
shadow.appendChild(box);
@@ -122,7 +122,7 @@
122122
if (!grids || !Array.isArray(grids)) return;
123123
removeOverlay();
124124
const host = document.createElement("div");
125-
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
125+
host.id = OVERLAY_HOST_ID, host.style.cssText = "\n position: fixed !important;\n top: 0 !important;\n left: 0 !important;\n width: 100vw !important;\n height: 100vh !important;\n pointer-events: none !important;\n z-index: 2147483647 !important;\n margin: 0 !important;\n padding: 0 !important;\n ",
126126
document.body.appendChild(host);
127127
const shadow = host.attachShadow({
128128
mode: "closed"
@@ -138,10 +138,10 @@
138138
let labelText = grid.label ? `Grid ${grid.grid_id}: ${grid.label}` : `Grid ${grid.grid_id}`;
139139
grid.is_dominant && (labelText = `⭐ ${labelText} (dominant)`);
140140
const badge = document.createElement("span");
141-
if (badge.textContent = labelText, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
141+
if (badge.textContent = labelText, badge.style.cssText = `\n position: absolute;\n top: -18px;\n left: 0;\n background: ${color};\n color: white;\n font-size: 11px;\n font-weight: bold;\n padding: 2px 6px;\n font-family: Arial, sans-serif;\n border-radius: 3px;\n opacity: 0.95;\n white-space: nowrap;\n pointer-events: none;\n `,
142142
box.appendChild(badge), isTarget) {
143143
const targetIndicator = document.createElement("span");
144-
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
144+
targetIndicator.textContent = "🎯", targetIndicator.style.cssText = "\n position: absolute;\n top: -18px;\n right: 0;\n font-size: 16px;\n pointer-events: none;\n ",
145145
box.appendChild(targetIndicator);
146146
}
147147
shadow.appendChild(box);
@@ -155,7 +155,7 @@
155155
let overlayTimeout = null;
156156
function removeOverlay() {
157157
const existing = document.getElementById(OVERLAY_HOST_ID);
158-
existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout),
158+
existing && existing.remove(), overlayTimeout && (clearTimeout(overlayTimeout),
159159
overlayTimeout = null);
160160
}
161-
}();
161+
}();

0 commit comments

Comments
 (0)