Skip to content

Commit f01079e

Browse files
rcholicrcholic
authored
Merge pull request #42 from SentienceAPI/click_xy
new features for click by rect box;semantic wait
2 parents 9839580 + 21057ef commit f01079e

File tree

7 files changed

+633
-34
lines changed

7 files changed

+633
-34
lines changed

‎README.md‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ from sentience import SentienceBrowser, snapshot, find, click
2020

2121
# Start browser with extension
2222
with SentienceBrowser(headless=False) as browser:
23-
browser.goto("https://example.com")
24-
browser.page.wait_for_load_state("networkidle")
23+
browser.goto("https://example.com", wait_until="domcontentloaded")
2524

2625
# Take snapshot - captures all interactive elements
2726
snap = snapshot(browser)
@@ -44,8 +43,7 @@ import time
4443

4544
with SentienceBrowser(headless=False) as browser:
4645
# Navigate to Amazon Best Sellers
47-
browser.goto("https://www.amazon.com/gp/bestsellers/")
48-
browser.page.wait_for_load_state("networkidle")
46+
browser.goto("https://www.amazon.com/gp/bestsellers/", wait_until="domcontentloaded")
4947
time.sleep(2) # Wait for dynamic content
5048

5149
# Take snapshot and find products
@@ -146,6 +144,7 @@ first_row = query(snap, "bbox.y<600")
146144

147145
### Actions - Interact with Elements
148146
- **`click(browser, element_id)`** - Click element by ID
147+
- **`click_rect(browser, rect)`** - Click at center of rectangle (coordinate-based)
149148
- **`type_text(browser, element_id, text)`** - Type into input fields
150149
- **`press(browser, key)`** - Press keyboard keys (Enter, Escape, Tab, etc.)
151150

@@ -160,17 +159,53 @@ print(f"Duration: {result.duration_ms}ms")
160159
print(f"URL changed: {result.url_changed}")
161160
```
162161

162+
**Coordinate-based clicking:**
163+
```python
164+
from sentience import click_rect
165+
166+
# Click at center of rectangle (x, y, width, height)
167+
click_rect(browser, {"x": 100, "y": 200, "w": 50, "h": 30})
168+
169+
# With visual highlight (default: red border for 2 seconds)
170+
click_rect(browser, {"x": 100, "y": 200, "w": 50, "h": 30}, highlight=True, highlight_duration=2.0)
171+
172+
# Using element's bounding box
173+
snap = snapshot(browser)
174+
element = find(snap, "role=button")
175+
if element:
176+
click_rect(browser, {
177+
"x": element.bbox.x,
178+
"y": element.bbox.y,
179+
"w": element.bbox.width,
180+
"h": element.bbox.height
181+
})
182+
```
183+
163184
### Wait & Assertions
164-
- **`wait_for(browser, selector, timeout=5.0)`** - Wait for element to appear
185+
- **`wait_for(browser, selector, timeout=5.0, interval=None, use_api=None)`** - Wait for element to appear
165186
- **`expect(browser, selector)`** - Assertion helper with fluent API
166187

167188
**Examples:**
168189
```python
169-
# Wait for element
190+
# Wait for element (auto-detects optimal interval based on API usage)
170191
result = wait_for(browser, "role=button text='Submit'", timeout=10.0)
171192
if result.found:
172193
print(f"Found after {result.duration_ms}ms")
173194

195+
# Use local extension with fast polling (0.25s interval)
196+
result = wait_for(browser, "role=button", timeout=5.0, use_api=False)
197+
198+
# Use remote API with network-friendly polling (1.5s interval)
199+
result = wait_for(browser, "role=button", timeout=5.0, use_api=True)
200+
201+
# Custom interval override
202+
result = wait_for(browser, "role=button", timeout=5.0, interval=0.5, use_api=False)
203+
204+
# Semantic wait conditions
205+
wait_for(browser, "clickable=true", timeout=5.0) # Wait for clickable element
206+
wait_for(browser, "importance>100", timeout=5.0) # Wait for important element
207+
wait_for(browser, "role=link visible=true", timeout=5.0) # Wait for visible link
208+
174209
# Assertions
175210
expect(browser, "role=button text='Submit'").to_exist(timeout=5.0)
176211
expect(browser, "role=heading").to_be_visible()
@@ -313,8 +348,7 @@ browser = SentienceBrowser() # headless=True if CI=true, else False
313348

314349
### 1. Wait for Dynamic Content
315350
```python
316-
browser.goto("https://example.com")
317-
browser.page.wait_for_load_state("networkidle")
351+
browser.goto("https://example.com", wait_until="domcontentloaded")
318352
time.sleep(1) # Extra buffer for AJAX/animations
319353
```
320354

‎examples/click_rect_demo.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
"""
2+
Example: Using click_rect for coordinate-based clicking with visual feedback
3+
"""
4+
5+
from sentience import SentienceBrowser, snapshot, find, click_rect
6+
import os
7+
8+
9+
def main():
10+
# Get API key from environment variable (optional - uses free tier if not set)
11+
api_key = os.environ.get("SENTIENCE_API_KEY")
12+
13+
with SentienceBrowser(api_key=api_key, headless=False) as browser:
14+
# Navigate to example.com
15+
browser.page.goto("https://example.com", wait_until="domcontentloaded")
16+
17+
print("=== click_rect Demo ===\n")
18+
19+
# Example 1: Click using rect dictionary
20+
print("1. Clicking at specific coordinates (100, 100) with size 50x30")
21+
print(" (You should see a red border highlight for 2 seconds)")
22+
result = click_rect(browser, {"x": 100, "y": 100, "w": 50, "h": 30})
23+
print(f" Result: success={result.success}, outcome={result.outcome}")
24+
print(f" Duration: {result.duration_ms}ms\n")
25+
26+
# Wait a bit
27+
browser.page.wait_for_timeout(1000)
28+
29+
# Example 2: Click using element's bbox
30+
print("2. Clicking using element's bounding box")
31+
snap = snapshot(browser)
32+
link = find(snap, "role=link")
33+
34+
if link:
35+
print(f" Found link: '{link.text}' at ({link.bbox.x}, {link.bbox.y})")
36+
print(" Clicking at center of element's bbox...")
37+
result = click_rect(browser, {
38+
"x": link.bbox.x,
39+
"y": link.bbox.y,
40+
"w": link.bbox.width,
41+
"h": link.bbox.height
42+
})
43+
print(f" Result: success={result.success}, outcome={result.outcome}")
44+
print(f" URL changed: {result.url_changed}\n")
45+
46+
# Navigate back if needed
47+
if result.url_changed:
48+
browser.page.goto("https://example.com", wait_until="domcontentloaded")
49+
browser.page.wait_for_load_state("networkidle")
50+
51+
# Example 3: Click without highlight (for headless/CI)
52+
print("3. Clicking without visual highlight")
53+
result = click_rect(browser, {"x": 200, "y": 200, "w": 40, "h": 20}, highlight=False)
54+
print(f" Result: success={result.success}\n")
55+
56+
# Example 4: Custom highlight duration
57+
print("4. Clicking with custom highlight duration (3 seconds)")
58+
result = click_rect(browser, {"x": 300, "y": 300, "w": 60, "h": 40}, highlight_duration=3.0)
59+
print(f" Result: success={result.success}")
60+
print(" (Red border should stay visible for 3 seconds)\n")
61+
62+
# Example 5: Click with snapshot capture
63+
print("5. Clicking and capturing snapshot after action")
64+
result = click_rect(
65+
browser,
66+
{"x": 150, "y": 150, "w": 50, "h": 30},
67+
take_snapshot=True
68+
)
69+
if result.snapshot_after:
70+
print(f" Snapshot captured: {len(result.snapshot_after.elements)} elements found")
71+
print(f" URL: {result.snapshot_after.url}\n")
72+
73+
print("✅ click_rect demo complete!")
74+
print("\nNote: click_rect uses Playwright's native mouse.click() for realistic")
75+
print("event simulation, triggering hover, focus, mousedown, mouseup sequences.")
76+
77+
78+
if __name__ == "__main__":
79+
main()
80+

‎examples/semantic_wait_demo.py‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
"""
2+
Example: Semantic wait_for using query DSL
3+
Demonstrates waiting for elements using semantic selectors
4+
"""
5+
6+
from sentience import SentienceBrowser, wait_for, click
7+
import os
8+
9+
10+
def main():
11+
# Get API key from environment variable (optional - uses free tier if not set)
12+
api_key = os.environ.get("SENTIENCE_API_KEY")
13+
14+
with SentienceBrowser(api_key=api_key, headless=False) as browser:
15+
# Navigate to example.com
16+
browser.page.goto("https://example.com", wait_until="domcontentloaded")
17+
18+
print("=== Semantic wait_for Demo ===\n")
19+
20+
# Example 1: Wait for element by role
21+
print("1. Waiting for link element (role=link)")
22+
wait_result = wait_for(browser, "role=link", timeout=5.0)
23+
if wait_result.found:
24+
print(f" ✅ Found after {wait_result.duration_ms}ms")
25+
print(f" Element: '{wait_result.element.text}' (id: {wait_result.element.id})")
26+
else:
27+
print(f" ❌ Not found (timeout: {wait_result.timeout})")
28+
print()
29+
30+
# Example 2: Wait for element by role and text
31+
print("2. Waiting for link with specific text")
32+
wait_result = wait_for(browser, "role=link text~'Example'", timeout=5.0)
33+
if wait_result.found:
34+
print(f" ✅ Found after {wait_result.duration_ms}ms")
35+
print(f" Element text: '{wait_result.element.text}'")
36+
else:
37+
print(" ❌ Not found")
38+
print()
39+
40+
# Example 3: Wait for clickable element
41+
print("3. Waiting for clickable element")
42+
wait_result = wait_for(browser, "clickable=true", timeout=5.0)
43+
if wait_result.found:
44+
print(f" ✅ Found clickable element after {wait_result.duration_ms}ms")
45+
print(f" Role: {wait_result.element.role}")
46+
print(f" Text: '{wait_result.element.text}'")
47+
print(f" Is clickable: {wait_result.element.visual_cues.is_clickable}")
48+
else:
49+
print(" ❌ Not found")
50+
print()
51+
52+
# Example 4: Wait for element with importance threshold
53+
print("4. Waiting for important element (importance > 100)")
54+
wait_result = wait_for(browser, "importance>100", timeout=5.0)
55+
if wait_result.found:
56+
print(f" ✅ Found important element after {wait_result.duration_ms}ms")
57+
print(f" Importance: {wait_result.element.importance}")
58+
print(f" Role: {wait_result.element.role}")
59+
else:
60+
print(" ❌ Not found")
61+
print()
62+
63+
# Example 5: Wait and then click
64+
print("5. Wait for element, then click it")
65+
wait_result = wait_for(browser, "role=link", timeout=5.0)
66+
if wait_result.found:
67+
print(" ✅ Found element, clicking...")
68+
click_result = click(browser, wait_result.element.id)
69+
print(f" Click result: success={click_result.success}, outcome={click_result.outcome}")
70+
if click_result.url_changed:
71+
print(f" ✅ Navigation occurred: {browser.page.url}")
72+
else:
73+
print(" ❌ Element not found, cannot click")
74+
print()
75+
76+
# Example 6: Using local extension (fast polling)
77+
print("6. Using local extension with auto-optimized interval")
78+
print(" When use_api=False, interval auto-adjusts to 0.25s (fast)")
79+
wait_result = wait_for(browser, "role=link", timeout=5.0, use_api=False)
80+
if wait_result.found:
81+
print(f" ✅ Found after {wait_result.duration_ms}ms")
82+
print(" (Used local extension, polled every 0.25 seconds)")
83+
print()
84+
85+
# Example 7: Using remote API (slower polling)
86+
print("7. Using remote API with auto-optimized interval")
87+
print(" When use_api=True, interval auto-adjusts to 1.5s (network-friendly)")
88+
if api_key:
89+
wait_result = wait_for(browser, "role=link", timeout=5.0, use_api=True)
90+
if wait_result.found:
91+
print(f" ✅ Found after {wait_result.duration_ms}ms")
92+
print(" (Used remote API, polled every 1.5 seconds)")
93+
else:
94+
print(" ⚠️ Skipped (no API key set)")
95+
print()
96+
97+
# Example 8: Custom interval override
98+
print("8. Custom interval override (manual control)")
99+
print(" You can still specify custom interval if needed")
100+
wait_result = wait_for(browser, "role=link", timeout=5.0, interval=0.5, use_api=False)
101+
if wait_result.found:
102+
print(f" ✅ Found after {wait_result.duration_ms}ms")
103+
print(" (Custom interval: 0.5 seconds)")
104+
print()
105+
106+
print("✅ Semantic wait_for demo complete!")
107+
print("\nNote: wait_for uses the semantic query DSL to find elements.")
108+
print("This is more robust than CSS selectors because it understands")
109+
print("the semantic meaning of elements (role, text, clickability, etc.).")
110+
111+
112+
if __name__ == "__main__":
113+
main()
114+

‎sentience/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from .models import Snapshot, Element, BBox, Viewport, ActionResult, WaitResult
77
from .snapshot import snapshot
88
from .query import query, find
9-
from .actions import click, type_text, press
9+
from .actions import click, type_text, press, click_rect
1010
from .wait import wait_for
1111
from .expect import expect
1212
from .inspector import Inspector, inspect
@@ -31,6 +31,7 @@
3131
"click",
3232
"type_text",
3333
"press",
34+
"click_rect",
3435
"wait_for",
3536
"expect",
3637
"Inspector",

0 commit comments

Comments
 (0)