resolve merge conflicts

Author: rcholic

.github/workflows/sync-extension.yml

@@ -25,6 +25,7 @@ jobs:
       uses: actions/checkout@v4
       with:
         token: ${{ secrets.GITHUB_TOKEN }}
+        fetch-depth: 0  # Fetch all history for proper branching
 
     - name: Set up Python
       uses: actions/setup-python@v5
@@ -62,7 +63,7 @@ jobs:
         mkdir -p extension-temp
         cd extension-temp
         
-        # Download individual files from release (reliable method)
+        # Download individual files from release (reliable method - no zip)
         echo "📁 Downloading individual files from release..."
         curl -L -H "Authorization: token ${{ secrets.SENTIENCE_CHROME_TOKEN }}" \
           "https://api.github.com/repos/$REPO/releases/tags/$TAG" | \
@@ -148,6 +149,11 @@ jobs:
           cp extension-temp/extension-package/pkg/*.d.ts sentience/extension/pkg/ 2>/dev/null || echo "⚠️ Type definitions not found"
         fi
         
+        # Verify copied files
+        echo "📋 Copied files:"
+        ls -la sentience/extension/
+        ls -la sentience/extension/pkg/ 2>/dev/null || echo "⚠️ pkg directory not created"
+        
     - name: Check for changes
       if: steps.release.outputs.skip != 'true'
       id: changes
@@ -167,7 +173,9 @@ jobs:
       if: steps.release.outputs.skip != 'true' && steps.changes.outputs.changed == 'true'
       uses: peter-evans/create-pull-request@v5
       with:
-        token: ${{ secrets.GITHUB_TOKEN }}
+        # Use PR_TOKEN if available (for repos with org restrictions), otherwise use GITHUB_TOKEN
+        # To use PAT: create secret named PR_TOKEN with a Personal Access Token that has 'repo' scope
+        token: ${{ secrets.PR_TOKEN }}
         commit-message: "chore: sync extension files from sentience-chrome ${{ steps.release.outputs.tag }}"
         title: "Sync Extension: ${{ steps.release.outputs.tag }}"
         body: |
@@ -177,7 +185,10 @@ jobs:
           - Extension manifest and scripts
           - WASM binary and bindings
           
-          **Source:** [sentience-chrome release ${{ steps.release.outputs.tag }}](${{ secrets.SENTIENCE_CHROME_REPO }}/releases/tag/${{ steps.release.outputs.tag }})
+          **Source:** [sentience-chrome release ${{ steps.release.outputs.tag }}](https://github.com/${{ secrets.SENTIENCE_CHROME_REPO }}/releases/tag/${{ steps.release.outputs.tag }})
         branch: sync-extension-${{ steps.release.outputs.tag }}
         delete-branch: true
+        labels: |
+          automated
+          extension-sync
 

README.md

@@ -46,6 +46,14 @@ with SentienceBrowser(headless=False) as browser:
 - Pydantic models for type safety
 - `snapshot.save(filepath)` - Save to JSON
 
+### Content Reading & Screenshots
+- `read(browser, format="text|markdown")` - Read page content as text or markdown
+  - Enhanced markdown conversion using `markdownify` (better than extension's lightweight conversion)
+  - Supports `enhance_markdown=True` to use improved conversion
+- `screenshot(browser, format="png|jpeg", quality=80)` - Capture standalone screenshot
+  - Returns base64-encoded data URL
+  - Supports PNG and JPEG formats with quality control
+
 ### Day 4: Query Engine
 - `query(snapshot, selector)` - Find elements matching selector
 - `find(snapshot, selector)` - Find single best match
@@ -96,6 +104,39 @@ See `examples/` directory:
 - `wait_and_click.py` - Wait and actions
 - `read_markdown.py` - Reading page content and converting to markdown
 
+### Content Reading Example
+
+```python
+from sentience import SentienceBrowser, read
+
+with SentienceBrowser() as browser:
+    browser.page.goto("https://example.com")
+    browser.page.wait_for_load_state("networkidle")
+    
+    # Read as enhanced markdown (better quality)
+    result = read(browser, format="markdown", enhance_markdown=True)
+    print(result["content"])  # High-quality markdown
+```
+
+### Screenshot Example
+
+```python
+from sentience import SentienceBrowser, screenshot
+import base64
+
+with SentienceBrowser() as browser:
+    browser.page.goto("https://example.com")
+    browser.page.wait_for_load_state("networkidle")
+    
+    # Capture PNG screenshot
+    data_url = screenshot(browser, format="png")
+    
+    # Save to file
+    image_data = base64.b64decode(data_url.split(",")[1])
+    with open("screenshot.png", "wb") as f:
+        f.write(image_data)
+```
+
 ## Testing
 
 ```bash

pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "jsonschema>=4.0.0",
     "requests>=2.31.0",  # For server-side API calls
     "playwright-stealth>=1.0.6",  # Bot evasion and stealth mode
+    "markdownify>=0.11.6",  # Enhanced HTML to Markdown conversion
 ]
 
 [project.urls]

sentience/__init__.py

@@ -13,6 +13,7 @@
 from .recorder import Recorder, Trace, TraceStep, record
 from .generator import ScriptGenerator, generate
 from .read import read
+from .screenshot import screenshot
 
 __version__ = "0.1.0"
 
@@ -41,5 +42,6 @@
     "ScriptGenerator",
     "generate",
     "read",
+    "screenshot",
 ]
 

sentience/read.py

@@ -8,15 +8,18 @@
 
 def read(
     browser: SentienceBrowser,
-    format: Literal["raw", "text", "markdown"] = "raw",  # noqa: A002
+    output_format: Literal["raw", "text", "markdown"] = "raw",
+    enhance_markdown: bool = True,
 ) -> dict:
     """
     Read page content as raw HTML, text, or markdown
     
     Args:
         browser: SentienceBrowser instance
-        format: Output format - "raw" (default, returns HTML for Turndown/markdownify),
-                "text" (plain text), or "markdown" (high-quality markdown via markdownify)
+        output_format: Output format - "raw" (default, returns HTML for external processing),
+                        "text" (plain text), or "markdown" (lightweight or enhanced markdown).
+        enhance_markdown: If True and output_format is "markdown", uses markdownify for better conversion.
+                          If False, uses the extension's lightweight markdown converter.
     
     Returns:
         dict with:
@@ -33,20 +36,19 @@ def read(
         html_content = result["content"]
         
         # Get high-quality markdown (uses markdownify internally)
-        result = read(browser, format="markdown")
+        result = read(browser, output_format="markdown")
         markdown = result["content"]
         
         # Get plain text
-        result = read(browser, format="text")
+        result = read(browser, output_format="text")
         text = result["content"]
     """
     if not browser.page:
         raise RuntimeError("Browser not started. Call browser.start() first.")
 
-    # For markdown format, get raw HTML first, then convert with markdownify
-    if format == "markdown":
-        # Get raw HTML from extension
-        raw_result = browser.page.evaluate(
+    if output_format == "markdown" and enhance_markdown:
+        # Get raw HTML from the extension first
+        raw_html_result = browser.page.evaluate(
             """
             (options) => {
                 return window.sentience.read(options);
@@ -55,57 +57,34 @@ def read(
             {"format": "raw"},
         )
 
-        if raw_result.get("status") != "success":
-            return raw_result
-        
-        # Convert to markdown using markdownify
-        try:
-            from markdownify import markdownify as md
-            html_content = raw_result["content"]
-            markdown_content = md(
-                html_content,
-                heading_style="ATX",  # Use # for headings
-                bullets="-",  # Use - for lists
-                strip=['script', 'style', 'nav', 'footer', 'header', 'noscript'],  # Strip unwanted tags
-            )
-            
-            # Return result with markdown content
-            return {
-                "status": "success",
-                "url": raw_result["url"],
-                "format": "markdown",
-                "content": markdown_content,
-                "length": len(markdown_content),
-            }
-        except ImportError:
-            # Fallback to extension's lightweight markdown if markdownify not installed
-            result = browser.page.evaluate(
-                """
-                (options) => {
-                    return window.sentience.read(options);
+        if raw_html_result.get("status") == "success":
+            html_content = raw_html_result["content"]
+            try:
+                # Use markdownify for enhanced markdown conversion
+                from markdownify import markdownify, MarkdownifyError
+                markdown_content = markdownify(html_content, heading_style="ATX", wrap=True)
+                return {
+                    "status": "success",
+                    "url": raw_html_result["url"],
+                    "format": "markdown",
+                    "content": markdown_content,
+                    "length": len(markdown_content),
                 }
-                """,
-                {"format": "markdown"},
-            )
-            return result
-        except (ValueError, TypeError, AttributeError) as e:
-            # If conversion fails, return error
-            return {
-                "status": "error",
-                "url": raw_result.get("url", ""),
-                "format": "markdown",
-                "content": "",
-                "length": 0,
-                "error": f"Markdown conversion failed: {e}",
-            }
-    else:
-        # For "raw" or "text", call extension directly
-        result = browser.page.evaluate(
-            """
-            (options) => {
-                return window.sentience.read(options);
-            }
-            """,
-            {"format": format},
-        )
-        return result
+            except ImportError:
+                print("Warning: 'markdownify' not installed. Install with 'pip install markdownify' for enhanced markdown. Falling back to extension's markdown.")
+            except MarkdownifyError as e:
+                print(f"Warning: markdownify failed ({e}), falling back to extension's markdown.")
+            except Exception as e:
+                print(f"Warning: An unexpected error occurred with markdownify ({e}), falling back to extension's markdown.")
+
+    # If not enhanced markdown, or fallback, call extension with requested format
+    result = browser.page.evaluate(
+        """
+        (options) => {
+            return window.sentience.read(options);
+        }
+        """,
+        {"format": output_format},
+    )
+    
+    return result

sentience/screenshot.py

@@ -0,0 +1,53 @@
+"""
+Screenshot functionality - standalone screenshot capture
+"""
+
+from typing import Optional, Literal, Dict, Any
+from .browser import SentienceBrowser
+
+
+def screenshot(
+    browser: SentienceBrowser,
+    format: Literal["png", "jpeg"] = "png",
+    quality: Optional[int] = None,
+) -> str:
+    """
+    Capture screenshot of current page
+    
+    Args:
+        browser: SentienceBrowser instance
+        format: Image format - "png" or "jpeg"
+        quality: JPEG quality (1-100), only used for JPEG format
+    
+    Returns:
+        Base64-encoded screenshot data URL (e.g., "data:image/png;base64,...")
+    
+    Raises:
+        RuntimeError: If browser not started
+        ValueError: If quality is invalid for JPEG
+    """
+    if not browser.page:
+        raise RuntimeError("Browser not started. Call browser.start() first.")
+    
+    if format == "jpeg" and quality is not None:
+        if not (1 <= quality <= 100):
+            raise ValueError("Quality must be between 1 and 100 for JPEG format")
+    
+    # Use Playwright's screenshot with base64 encoding
+    screenshot_options: Dict[str, Any] = {
+        "type": format,
+    }
+    
+    if format == "jpeg" and quality is not None:
+        screenshot_options["quality"] = quality
+    
+    # Capture screenshot as base64
+    # Playwright returns bytes when encoding is not specified, so we encode manually
+    import base64
+    image_bytes = browser.page.screenshot(**screenshot_options)
+    base64_data = base64.b64encode(image_bytes).decode('utf-8')
+    
+    # Return as data URL
+    mime_type = "image/png" if format == "png" else "image/jpeg"
+    return f"data:{mime_type};base64,{base64_data}"
+

tests/test_read.py

@@ -0,0 +1,64 @@
+"""
+Tests for read functionality
+"""
+
+from sentience import SentienceBrowser, read
+
+
+def test_read_text():
+    """Test reading page as text"""
+    with SentienceBrowser(headless=True) as browser:
+        browser.page.goto("https://example.com")
+        browser.page.wait_for_load_state("networkidle")
+        
+        result = read(browser, format="text")
+        
+        assert result["status"] == "success"
+        assert result["format"] == "text"
+        assert "content" in result
+        assert "length" in result
+        assert len(result["content"]) > 0
+        assert result["url"] == "https://example.com"
+
+
+def test_read_markdown():
+    """Test reading page as markdown"""
+    with SentienceBrowser(headless=True) as browser:
+        browser.page.goto("https://example.com")
+        browser.page.wait_for_load_state("networkidle")
+        
+        result = read(browser, format="markdown")
+        
+        assert result["status"] == "success"
+        assert result["format"] == "markdown"
+        assert "content" in result
+        assert "length" in result
+        assert len(result["content"]) > 0
+        assert result["url"] == "https://example.com"
+
+
+def test_read_markdown_enhanced():
+    """Test reading page as markdown with enhancement"""
+    with SentienceBrowser(headless=True) as browser:
+        browser.page.goto("https://example.com")
+        browser.page.wait_for_load_state("networkidle")
+        
+        # Test with enhancement (default)
+        result_enhanced = read(browser, format="markdown", enhance_markdown=True)
+        
+        assert result_enhanced["status"] == "success"
+        assert result_enhanced["format"] == "markdown"
+        assert len(result_enhanced["content"]) > 0
+        
+        # Test without enhancement
+        result_basic = read(browser, format="markdown", enhance_markdown=False)
+        
+        assert result_basic["status"] == "success"
+        assert result_basic["format"] == "markdown"
+        assert len(result_basic["content"]) > 0
+        
+        # Enhanced markdown should be different (and likely better formatted)
+        # Note: They might be similar for simple pages, but enhanced should handle more cases
+        assert isinstance(result_enhanced["content"], str)
+        assert isinstance(result_basic["content"], str)
+