Merge pull request #189 from SentienceAPI/parity_win2

Phase 2: ToolRegistry + Filesystem + extraction

Author: rcholic

sentience/__init__.py

@@ -19,7 +19,11 @@
     click_rect,
     press,
     scroll_to,
+    search,
+    search_async,
     select_option,
+    send_keys,
+    send_keys_async,
     submit,
     type_text,
     uncheck,
@@ -102,13 +106,14 @@
 from .ordinal import OrdinalIntent, boost_ordinal_elements, detect_ordinal_intent, select_by_ordinal
 from .overlay import clear_overlay, show_overlay
 from .query import find, query
-from .read import read
+from .read import extract, extract_async, read
 from .recorder import Recorder, Trace, TraceStep, record
 from .runtime_agent import RuntimeAgent, RuntimeStep, StepVerification
 from .screenshot import screenshot
 from .sentience_methods import AgentAction, SentienceMethod
 from .snapshot import snapshot
 from .text_search import find_text_rect
+from .tools import BackendCapabilities, ToolContext, ToolRegistry, ToolSpec, register_default_tools
 from .tracer_factory import SENTIENCE_API_URL, create_tracer
 from .tracing import JsonlTraceSink, TraceEvent, Tracer, TraceSink
 

sentience/actions.py

@@ -5,11 +5,12 @@
 import asyncio
 import time
 from pathlib import Path
+from urllib.parse import quote_plus
 
 from .browser import AsyncSentienceBrowser, SentienceBrowser
 from .browser_evaluator import BrowserEvaluator
 from .cursor_policy import CursorPolicy, build_human_cursor_path
-from .models import ActionResult, BBox, Snapshot
+from .models import ActionResult, BBox, Snapshot, SnapshotOptions
 from .sentience_methods import SentienceMethod
 from .snapshot import snapshot, snapshot_async
 
@@ -709,6 +710,146 @@ def press(browser: SentienceBrowser, key: str, take_snapshot: bool = False) -> A
     )
 
 
+def _normalize_key_token(token: str) -> str:
+    lookup = {
+        "CMD": "Meta",
+        "COMMAND": "Meta",
+        "CTRL": "Control",
+        "CONTROL": "Control",
+        "ALT": "Alt",
+        "OPTION": "Alt",
+        "SHIFT": "Shift",
+        "ESC": "Escape",
+        "ESCAPE": "Escape",
+        "ENTER": "Enter",
+        "RETURN": "Enter",
+        "TAB": "Tab",
+        "SPACE": "Space",
+    }
+    upper = token.strip().upper()
+    return lookup.get(upper, token.strip())
+
+
+def _parse_key_sequence(sequence: str) -> list[str]:
+    parts = []
+    for raw in sequence.replace(",", " ").split():
+        raw = raw.strip()
+        if not raw:
+            continue
+        if raw.startswith("{") and raw.endswith("}"):
+            raw = raw[1:-1]
+        if "+" in raw:
+            combo = "+".join(_normalize_key_token(tok) for tok in raw.split("+") if tok)
+            parts.append(combo)
+        else:
+            parts.append(_normalize_key_token(raw))
+    return parts
+
+
+def send_keys(
+    browser: SentienceBrowser,
+    sequence: str,
+    take_snapshot: bool = False,
+    delay_ms: int = 50,
+) -> ActionResult:
+    """
+    Send a sequence of key presses (e.g., "CMD+H", "CTRL+SHIFT+P").
+
+    Supports sequences separated by commas/spaces, and brace-wrapped tokens
+    like "{ENTER}" or "{CTRL+L}".
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    keys = _parse_key_sequence(sequence)
+    if not keys:
+        raise ValueError("send_keys sequence is empty")
+    for key in keys:
+        browser.page.keyboard.press(key)
+        if delay_ms > 0:
+            browser.page.wait_for_timeout(delay_ms)
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = snapshot(browser)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
+def _build_search_url(query: str, engine: str) -> str:
+    q = quote_plus(query)
+    key = engine.strip().lower()
+    if key in {"duckduckgo", "ddg"}:
+        return f"https://duckduckgo.com/?q={q}"
+    if key in {"google.com", "google"}:
+        return f"https://www.google.com/search?q={q}"
+    if key in {"google"}:
+        return f"https://www.google.com/search?q={q}"
+    if key in {"bing"}:
+        return f"https://www.bing.com/search?q={q}"
+    raise ValueError(f"unsupported search engine: {engine}")
+
+
+def search(
+    browser: SentienceBrowser,
+    query: str,
+    engine: str = "duckduckgo",
+    take_snapshot: bool = False,
+    snapshot_options: SnapshotOptions | None = None,
+) -> ActionResult:
+    """
+    Navigate to a search results page for the given query.
+
+    Args:
+        browser: SentienceBrowser instance
+        query: Search query string
+        engine: Search engine name (duckduckgo, google, google.com, bing)
+        take_snapshot: Whether to take snapshot after navigation
+        snapshot_options: Snapshot options passed to snapshot() when take_snapshot is True.
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+    if not query.strip():
+        raise ValueError("search query is empty")
+
+    start_time = time.time()
+    url_before = browser.page.url
+    url = _build_search_url(query, engine)
+    browser.goto(url)
+    browser.page.wait_for_load_state("networkidle")
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = snapshot(browser, snapshot_options)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
 def scroll_to(
     browser: SentienceBrowser,
     element_id: int,
@@ -1698,6 +1839,93 @@ async def press_async(
     )
 
 
+async def send_keys_async(
+    browser: AsyncSentienceBrowser,
+    sequence: str,
+    take_snapshot: bool = False,
+    delay_ms: int = 50,
+) -> ActionResult:
+    """
+    Async version of send_keys().
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+
+    start_time = time.time()
+    url_before = browser.page.url
+
+    keys = _parse_key_sequence(sequence)
+    if not keys:
+        raise ValueError("send_keys sequence is empty")
+    for key in keys:
+        await browser.page.keyboard.press(key)
+        if delay_ms > 0:
+            await browser.page.wait_for_timeout(delay_ms)
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = await snapshot_async(browser)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
+async def search_async(
+    browser: AsyncSentienceBrowser,
+    query: str,
+    engine: str = "duckduckgo",
+    take_snapshot: bool = False,
+    snapshot_options: SnapshotOptions | None = None,
+) -> ActionResult:
+    """
+    Async version of search().
+
+    Args:
+        browser: AsyncSentienceBrowser instance
+        query: Search query string
+        engine: Search engine name (duckduckgo, google, google.com, bing)
+        take_snapshot: Whether to take snapshot after navigation
+        snapshot_options: Snapshot options passed to snapshot_async() when take_snapshot is True.
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call await browser.start() first.")
+    if not query.strip():
+        raise ValueError("search query is empty")
+
+    start_time = time.time()
+    url_before = browser.page.url
+    url = _build_search_url(query, engine)
+    await browser.goto(url)
+    await browser.page.wait_for_load_state("networkidle")
+
+    duration_ms = int((time.time() - start_time) * 1000)
+    url_after = browser.page.url
+    url_changed = url_before != url_after
+    outcome = "navigated" if url_changed else "dom_updated"
+
+    snapshot_after: Snapshot | None = None
+    if take_snapshot:
+        snapshot_after = await snapshot_async(browser, snapshot_options)
+
+    return ActionResult(
+        success=True,
+        duration_ms=duration_ms,
+        outcome=outcome,
+        url_changed=url_changed,
+        snapshot_after=snapshot_after,
+    )
+
+
 async def scroll_to_async(
     browser: AsyncSentienceBrowser,
     element_id: int,

sentience/agent.py

@@ -89,9 +89,7 @@ def _safe_hook_call_sync(
         if verbose:
             print(f"⚠️  Hook error (non-fatal): {hook_error}")
         else:
-            logging.getLogger(__name__).warning(
-                "Hook error (non-fatal): %s", hook_error
-            )
+            logging.getLogger(__name__).warning("Hook error (non-fatal): %s", hook_error)
 
 
 async def _safe_hook_call_async(
@@ -109,9 +107,7 @@ async def _safe_hook_call_async(
         if verbose:
             print(f"⚠️  Hook error (non-fatal): {hook_error}")
         else:
-            logging.getLogger(__name__).warning(
-                "Hook error (non-fatal): %s", hook_error
-            )
+            logging.getLogger(__name__).warning("Hook error (non-fatal): %s", hook_error)
 
 
 class SentienceAgent(BaseAgent):

sentience/agent_runtime.py

@@ -81,6 +81,7 @@
     TabListResult,
     TabOperationResult,
 )
+from .tools import BackendCapabilities, ToolRegistry
 from .trace_event_builder import TraceEventBuilder
 from .verification import AssertContext, AssertOutcome, Predicate
 
@@ -118,6 +119,7 @@ def __init__(
         tracer: Tracer,
         snapshot_options: SnapshotOptions | None = None,
         sentience_api_key: str | None = None,
+        tool_registry: ToolRegistry | None = None,
     ):
         """
         Initialize agent runtime with any BrowserBackend-compatible browser.
@@ -130,9 +132,11 @@ def __init__(
             tracer: Tracer for emitting verification events
             snapshot_options: Default options for snapshots
             sentience_api_key: API key for Pro/Enterprise tier (enables Gateway refinement)
+            tool_registry: Optional ToolRegistry for LLM-callable tools
         """
         self.backend = backend
         self.tracer = tracer
+        self.tool_registry = tool_registry
 
         # Build default snapshot options with API key if provided
         default_opts = snapshot_options or SnapshotOptions()
@@ -372,6 +376,33 @@ def _get_tab_backend(self):
             return None
         return backend
 
+    def capabilities(self) -> BackendCapabilities:
+        backend = getattr(self, "backend", None)
+        if backend is None:
+            return BackendCapabilities()
+        has_eval = hasattr(backend, "eval")
+        has_keyboard = hasattr(backend, "type_text") or bool(
+            getattr(getattr(backend, "_page", None), "keyboard", None)
+        )
+        has_downloads = bool(getattr(backend, "downloads", None))
+        has_files = False
+        if self.tool_registry is not None:
+            try:
+                has_files = self.tool_registry.get("read_file") is not None
+            except Exception:
+                has_files = False
+        return BackendCapabilities(
+            tabs=self._get_tab_backend() is not None,
+            evaluate_js=bool(has_eval),
+            downloads=has_downloads,
+            filesystem_tools=has_files,
+            keyboard=bool(has_keyboard or has_eval),
+        )
+
+    def can(self, capability: str) -> bool:
+        caps = self.capabilities()
+        return bool(getattr(caps, capability, False))
+
     @staticmethod
     def _stringify_eval_value(value: Any) -> str:
         if value is None: