Merge pull request #150 from SentienceAPI/humanize_local_vision

local vision llm provider

Author: rcholic

examples/human_cursor_click_demo.py

@@ -38,4 +38,3 @@ def main() -> None:
 
 if __name__ == "__main__":
     main()
-

pyproject.toml

@@ -43,6 +43,15 @@ sentience = "sentience.cli:main"
 browser-use = [
     "browser-use>=0.1.40",
 ]
+vision-local = [
+    "pillow>=10.0.0",
+    "torch>=2.2.0",
+    "transformers>=4.46.0",
+]
+mlx-vlm = [
+    "pillow>=10.0.0",
+    "mlx-vlm>=0.1.0",
+]
 dev = [
     "pytest>=7.0.0",
     "pytest-asyncio>=0.21.0",

sentience/__init__.py

@@ -52,6 +52,8 @@
     LLMProvider,
     LLMResponse,
     LocalLLMProvider,
+    LocalVisionLLMProvider,
+    MLXVLMProvider,
     OpenAIProvider,
 )
 from .models import (  # Agent Layer Models
@@ -202,6 +204,8 @@
     "OpenAIProvider",
     "AnthropicProvider",
     "LocalLLMProvider",
+    "LocalVisionLLMProvider",
+    "MLXVLMProvider",
     "SentienceAgent",
     "SentienceAgentAsync",
     "SentienceVisualAgent",

sentience/cursor_policy.py

@@ -44,7 +44,13 @@ def _ease_in_out(t: float) -> float:
     return t * t * (3 - 2 * t)
 
 
-def _bezier(p0: tuple[float, float], p1: tuple[float, float], p2: tuple[float, float], p3: tuple[float, float], t: float) -> tuple[float, float]:
+def _bezier(
+    p0: tuple[float, float],
+    p1: tuple[float, float],
+    p2: tuple[float, float],
+    p3: tuple[float, float],
+    t: float,
+) -> tuple[float, float]:
     u = 1.0 - t
     tt = t * t
     uu = u * u
@@ -87,7 +93,9 @@ def build_human_cursor_path(
 
     # Defaults based on distance (bounded)
     steps = int(policy.steps if policy.steps is not None else _clamp(10 + dist / 25.0, 12, 40))
-    duration_ms = int(policy.duration_ms if policy.duration_ms is not None else _clamp(120 + dist * 0.9, 120, 700))
+    duration_ms = int(
+        policy.duration_ms if policy.duration_ms is not None else _clamp(120 + dist * 0.9, 120, 700)
+    )
 
     # Control points: offset roughly perpendicular to travel direction
     if dist < 1e-6:
@@ -132,4 +140,3 @@ def build_human_cursor_path(
         # Keep path bounded for trace size
         "path": pts[:64],
     }
-