update models with modal detection

Author: SentienceDEV

sentience/models.py

@@ -144,6 +144,12 @@ class GridInfo(BaseModel):
     )
     is_dominant: bool = False  # Whether this grid is the dominant group (main content area)
 
+    # Z-index and modal detection fields (from gateway/sentience-core)
+    z_index: int = 0  # Z-index of this grid (max among elements in this grid)
+    z_index_max: int = 0  # Global max z-index across ALL grids (for comparison)
+    blocks_interaction: bool = False  # Whether this grid blocks interaction with content behind it
+    viewport_coverage: float = 0.0  # Ratio of grid area to viewport area (0.0-1.0)
+
 
 class Snapshot(BaseModel):
     """Snapshot response from extension"""
@@ -161,6 +167,9 @@ class Snapshot(BaseModel):
     dominant_group_key: str | None = None  # The most common group_key (main content group)
     # Phase 2: Runtime stability/debug info (confidence/reasons/metrics)
     diagnostics: SnapshotDiagnostics | None = None
+    # Modal detection fields (from gateway)
+    modal_detected: bool | None = None  # True if a modal/overlay grid was detected
+    modal_grids: list[GridInfo] | None = None  # Array of GridInfo for detected modal grids
 
     def save(self, filepath: str) -> None:
         """Save snapshot as JSON file"""

sentience/runtime_agent.py

@@ -92,25 +92,19 @@ async def run_step(
             snap = await self._snapshot_with_ramp(step=step)
 
             if await self._should_short_circuit_to_vision(step=step, snap=snap):
-                ok = await self._vision_executor_attempt(
-                    task_goal=task_goal, step=step, snap=snap
-                )
+                ok = await self._vision_executor_attempt(task_goal=task_goal, step=step, snap=snap)
                 return ok
 
             # 1) Structured executor attempt.
-            action = self._propose_structured_action(
-                task_goal=task_goal, step=step, snap=snap
-            )
+            action = self._propose_structured_action(task_goal=task_goal, step=step, snap=snap)
             await self._execute_action(action=action, snap=snap)
             ok = await self._apply_verifications(step=step)
             if ok:
                 return True
 
             # 2) Optional vision executor fallback (bounded).
             if step.vision_executor_enabled and step.max_vision_executor_attempts > 0:
-                ok = await self._vision_executor_attempt(
-                    task_goal=task_goal, step=step, snap=snap
-                )
+                ok = await self._vision_executor_attempt(task_goal=task_goal, step=step, snap=snap)
                 return ok
 
             return False

sentience/trace_event_builder.py

@@ -155,11 +155,7 @@ def build_step_end_event(
             "exec": exec_data,
             "post": {
                 "url": post_url,
-                **(
-                    {"snapshot_digest": post_snapshot_digest}
-                    if post_snapshot_digest
-                    else {}
-                ),
+                **({"snapshot_digest": post_snapshot_digest} if post_snapshot_digest else {}),
             },
             "verify": final_verify_data,
         }

tests/test_grid_bounds.py

@@ -348,3 +348,94 @@ def test_sorted_by_grid_id(self):
         assert result[0].grid_id == 0
         assert result[1].grid_id == 1
         assert result[2].grid_id == 2
+
+
+class TestGridInfoModalFields:
+    """Tests for GridInfo z-index and modal detection fields"""
+
+    def test_grid_info_default_values(self):
+        """Test that GridInfo has correct default values for new fields"""
+        grid_info = GridInfo(
+            grid_id=0,
+            bbox=BBox(x=0, y=0, width=100, height=100),
+            row_count=1,
+            col_count=1,
+            item_count=1,
+        )
+        # New optional fields should have defaults
+        assert grid_info.z_index == 0
+        assert grid_info.z_index_max == 0
+        assert grid_info.blocks_interaction is False
+        assert grid_info.viewport_coverage == 0.0
+
+    def test_grid_info_with_modal_fields(self):
+        """Test creating GridInfo with modal detection fields"""
+        grid_info = GridInfo(
+            grid_id=1,
+            bbox=BBox(x=100, y=100, width=500, height=400),
+            row_count=2,
+            col_count=3,
+            item_count=6,
+            confidence=0.95,
+            z_index=1000,
+            z_index_max=1000,
+            blocks_interaction=True,
+            viewport_coverage=0.25,
+        )
+        assert grid_info.z_index == 1000
+        assert grid_info.z_index_max == 1000
+        assert grid_info.blocks_interaction is True
+        assert grid_info.viewport_coverage == 0.25
+
+
+class TestSnapshotModalFields:
+    """Tests for Snapshot modal detection fields"""
+
+    def test_snapshot_without_modal(self):
+        """Test snapshot with no modal detected"""
+        snapshot = Snapshot(
+            status="success",
+            url="https://example.com",
+            elements=[],
+        )
+        # modal_detected and modal_grids should be None by default
+        assert snapshot.modal_detected is None
+        assert snapshot.modal_grids is None
+
+    def test_snapshot_with_modal_detected(self):
+        """Test snapshot with modal detected"""
+        modal_grid = GridInfo(
+            grid_id=1,
+            bbox=BBox(x=200, y=150, width=600, height=400),
+            row_count=1,
+            col_count=2,
+            item_count=5,
+            z_index=1000,
+            z_index_max=1000,
+            blocks_interaction=True,
+            viewport_coverage=0.20,
+        )
+        snapshot = Snapshot(
+            status="success",
+            url="https://example.com",
+            elements=[],
+            modal_detected=True,
+            modal_grids=[modal_grid],
+        )
+        assert snapshot.modal_detected is True
+        assert snapshot.modal_grids is not None
+        assert len(snapshot.modal_grids) == 1
+        assert snapshot.modal_grids[0].z_index == 1000
+        assert snapshot.modal_grids[0].blocks_interaction is True
+
+    def test_snapshot_modal_false(self):
+        """Test snapshot with modal_detected explicitly False"""
+        snapshot = Snapshot(
+            status="success",
+            url="https://example.com",
+            elements=[],
+            modal_detected=False,
+            modal_grids=None,
+        )
+        assert snapshot.modal_detected is False
+        assert snapshot.modal_grids is None