Merge pull request #178 from SentienceAPI/bench_gaps

vision executor + enrich form actions

Author: rcholic

.github/workflows/test.yml

@@ -40,61 +40,8 @@ jobs:
         # Also clean .pyc files in sentience package specifically
         find sentience -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || python -c "import pathlib, shutil; [shutil.rmtree(p) for p in pathlib.Path('sentience').rglob('__pycache__') if p.is_dir()]" || true
         find sentience -name "*.pyc" -delete 2>/dev/null || python -c "import pathlib; [p.unlink() for p in pathlib.Path('sentience').rglob('*.pyc')]" || true
-        # CRITICAL: Fix assertTrue bug if it exists in source (shouldn't happen, but safety check)
-        python << 'PYEOF'
-        import re
-        import os
-        import sys
-
-        # Set UTF-8 encoding for Windows compatibility
-        if sys.platform == 'win32':
-            import io
-            sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
-            sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace')
-
-        file_path = 'sentience/agent_runtime.py'
-        print(f'=== Auto-fix check for {file_path} ===')
-        try:
-            if not os.path.exists(file_path):
-                print(f'ERROR: {file_path} not found!')
-                sys.exit(1)
-
-            with open(file_path, 'r', encoding='utf-8') as f:
-                content = f.read()
-
-            if 'self.assertTrue(' in content:
-                print('WARNING: Found self.assertTrue( in source file! Auto-fixing...')
-                # Count occurrences
-                count = len(re.findall(r'self\.assertTrue\s*\(', content))
-                print(f'Found {count} occurrence(s) of self.assertTrue(')
-
-                # Replace all occurrences
-                new_content = re.sub(r'self\.assertTrue\s*\(', 'self.assert_(', content)
-
-                # Write back
-                with open(file_path, 'w', encoding='utf-8') as f:
-                    f.write(new_content)
-
-                # Verify the fix
-                with open(file_path, 'r', encoding='utf-8') as f:
-                    verify_content = f.read()
-                    if 'self.assertTrue(' in verify_content:
-                        print('ERROR: Auto-fix failed! File still contains self.assertTrue(')
-                        sys.exit(1)
-                    else:
-                        print('OK: Auto-fixed: Replaced self.assertTrue( with self.assert_(')
-                        print('OK: Verified: File no longer contains self.assertTrue(')
-            else:
-                print('OK: Source file is correct (uses self.assert_())')
-        except Exception as e:
-            print(f'ERROR in auto-fix: {e}')
-            import traceback
-            traceback.print_exc()
-            sys.exit(1)
-        PYEOF
-        # Verify source file is fixed before installation
-        echo "=== Verifying source file after auto-fix ==="
-        python -c "import sys; import io; sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') if sys.platform == 'win32' else sys.stdout; content = open('sentience/agent_runtime.py', 'r', encoding='utf-8').read(); assert 'self.assertTrue(' not in content, 'Source file still has self.assertTrue( after auto-fix!'; print('OK: Source file verified: uses self.assert_()')"
+        # Ensure source uses assert_ (no auto-rewrite).
+        python -c "import sys; import io; sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') if sys.platform == 'win32' else sys.stdout; content = open('sentience/agent_runtime.py', 'r', encoding='utf-8').read(); assert 'self.assertTrue(' not in content, 'Source file still has self.assertTrue('; print('OK: Source file verified: uses self.assert_()')"
 
         # Force reinstall to ensure latest code
         pip install --no-cache-dir --force-reinstall -e ".[dev]"

.gitignore

@@ -60,3 +60,7 @@ Thumbs.db
 # Temporary directories from sync workflows
 extension-temp/
 playground/
+
+# Allow bundled extension assets under sentience/extension/dist
+!sentience/extension/dist/
+!sentience/extension/dist/**

examples/runtime_agent_minimal.py

@@ -47,7 +47,9 @@ async def main() -> None:
         await page.goto("https://example.com")
         await page.wait_for_load_state("networkidle")
 
-        runtime = await AgentRuntime.from_sentience_browser(browser=browser, page=page, tracer=tracer)
+        runtime = await AgentRuntime.from_sentience_browser(
+            browser=browser, page=page, tracer=tracer
+        )
 
         # Structured executor (for demo, we just return FINISH()).
         executor = FixedActionProvider("FINISH()")
@@ -63,15 +65,29 @@ async def main() -> None:
         def has_example_heading(ctx: AssertContext) -> AssertOutcome:
             # Demonstrates custom predicates (you can also use exists/url_contains helpers).
             snap = ctx.snapshot
-            ok = bool(snap and any((el.role == "heading" and (el.text or "").startswith("Example")) for el in snap.elements))
+            ok = bool(
+                snap
+                and any(
+                    (el.role == "heading" and (el.text or "").startswith("Example"))
+                    for el in snap.elements
+                )
+            )
             return AssertOutcome(passed=ok, reason="" if ok else "missing heading", details={})
 
         step = RuntimeStep(
             goal="Confirm Example Domain page is loaded",
             verifications=[
-                StepVerification(predicate=url_contains("example.com"), label="url_contains_example", required=True),
-                StepVerification(predicate=exists("role=heading"), label="has_heading", required=True),
-                StepVerification(predicate=has_example_heading, label="heading_text_matches", required=False),
+                StepVerification(
+                    predicate=url_contains("example.com"),
+                    label="url_contains_example",
+                    required=True,
+                ),
+                StepVerification(
+                    predicate=exists("role=heading"), label="has_heading", required=True
+                ),
+                StepVerification(
+                    predicate=has_example_heading, label="heading_text_matches", required=False
+                ),
             ],
             max_snapshot_attempts=2,
             snapshot_limit_base=60,
@@ -86,4 +102,3 @@ def has_example_heading(ctx: AssertContext) -> AssertOutcome:
 
 if __name__ == "__main__":
     asyncio.run(main())
-

scripts/sync_extension.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+CHROME_DIR="${REPO_ROOT}/sentience-chrome"
+SDK_EXT_DIR="${REPO_ROOT}/sdk-python/sentience/extension"
+
+if [[ ! -d "${CHROME_DIR}" ]]; then
+  echo "[sync_extension] sentience-chrome not found at ${CHROME_DIR}"
+  exit 1
+fi
+
+if [[ ! -f "${CHROME_DIR}/package.json" ]]; then
+  echo "[sync_extension] package.json missing in sentience-chrome"
+  exit 1
+fi
+
+echo "[sync_extension] Building sentience-chrome..."
+pushd "${CHROME_DIR}" >/dev/null
+npm run build
+popd >/dev/null
+
+echo "[sync_extension] Syncing dist/ and pkg/ to sdk-python..."
+mkdir -p "${SDK_EXT_DIR}/dist" "${SDK_EXT_DIR}/pkg"
+cp "${CHROME_DIR}/dist/"* "${SDK_EXT_DIR}/dist/"
+cp "${CHROME_DIR}/pkg/"* "${SDK_EXT_DIR}/pkg/"
+
+echo "[sync_extension] Done."

sentience/__init__.py

@@ -11,7 +11,20 @@
     verify_extension_version,
     verify_extension_version_async,
 )
-from .actions import click, click_rect, press, scroll_to, type_text
+from .actions import (
+    back,
+    check,
+    clear,
+    click,
+    click_rect,
+    press,
+    scroll_to,
+    select_option,
+    submit,
+    type_text,
+    uncheck,
+    upload_file,
+)
 from .agent import SentienceAgent, SentienceAgentAsync
 from .agent_config import AgentConfig
 from .agent_runtime import AgentRuntime, AssertionHandle
@@ -118,6 +131,7 @@
     all_of,
     any_of,
     custom,
+    download_completed,
     element_count,
     exists,
     is_checked,
@@ -132,6 +146,13 @@
     value_contains,
     value_equals,
 )
+
+# Vision executor primitives (shared parsing/execution helpers)
+from .vision_executor import (
+    VisionExecutorAction,
+    execute_vision_executor_action,
+    parse_vision_executor_action,
+)
 from .visual_agent import SentienceVisualAgent, SentienceVisualAgentAsync
 from .wait import wait_for