From 9d2dba896310ee158a6c04d109092e5b22aba653 Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Fri, 23 Jan 2026 22:22:51 -0800 Subject: [PATCH 1/3] solution for handle Chrome permission popup --- src/agent-runtime.ts | 12 +++++ src/browser.ts | 34 ++++++++++++- src/index.ts | 2 +- src/tools/defaults.ts | 97 ++++++++++++++++++++++++++++++++++++- src/tools/registry.ts | 6 ++- src/types.ts | 1 + tests/tool-registry.test.ts | 72 +++++++++++++++++++++++++++ 7 files changed, 220 insertions(+), 4 deletions(-) diff --git a/src/agent-runtime.ts b/src/agent-runtime.ts index 0c8af02..7b40209 100644 --- a/src/agent-runtime.ts +++ b/src/agent-runtime.ts @@ -492,6 +492,17 @@ export class AgentRuntime { const hasEval = typeof (this as any).evaluateJs === 'function'; const hasKeyboard = Boolean((this.page as any)?.keyboard); const hasDownloads = this.downloads.length >= 0; + let hasPermissions = false; + try { + const context = + typeof (this.page as any)?.context === 'function' ? (this.page as any).context() : null; + hasPermissions = + Boolean(context) && + typeof context.clearPermissions === 'function' && + typeof context.grantPermissions === 'function'; + } catch { + hasPermissions = false; + } let hasFiles = false; if (this.toolRegistry) { hasFiles = Boolean(this.toolRegistry.get('read_file')); @@ -502,6 +513,7 @@ export class AgentRuntime { downloads: hasDownloads, filesystem_tools: hasFiles, keyboard: hasKeyboard, + permissions: hasPermissions, }; } diff --git a/src/browser.ts b/src/browser.ts index b554b8d..d26d484 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -12,6 +12,15 @@ import { SnapshotOptions } from './snapshot'; import { IBrowser } from './protocols/browser-protocol'; import { snapshot as snapshotFunction } from './snapshot'; +export type PermissionDefault = 'clear' | 'deny' | 'grant'; + +export type PermissionPolicy = { + default?: PermissionDefault; + autoGrant?: string[]; + geolocation?: { latitude: number; longitude: number; accuracy?: number }; + origin?: string; +}; + export function normalizeDomain(domain: string): string { const raw = domain.trim(); let host = raw; @@ -88,6 +97,7 @@ export class SentienceBrowser implements IBrowser { private _allowedDomains?: string[]; private _prohibitedDomains?: string[]; private _keepAlive: boolean; + private _permissionPolicy?: PermissionPolicy; /** * Create a new SentienceBrowser instance @@ -121,7 +131,8 @@ export class SentienceBrowser implements IBrowser { deviceScaleFactor?: number, allowedDomains?: string[], prohibitedDomains?: string[], - keepAlive: boolean = false + keepAlive: boolean = false, + permissionPolicy?: PermissionPolicy ) { this._apiKey = apiKey; @@ -162,6 +173,23 @@ export class SentienceBrowser implements IBrowser { this._allowedDomains = allowedDomains; this._prohibitedDomains = prohibitedDomains; this._keepAlive = keepAlive; + this._permissionPolicy = permissionPolicy; + } + + private async applyPermissionPolicy( + context: BrowserContext, + policy: PermissionPolicy + ): Promise { + const defaultPolicy = policy.default ?? 'clear'; + if (defaultPolicy === 'clear' || defaultPolicy === 'deny') { + await context.clearPermissions(); + } + if (policy.geolocation) { + await context.setGeolocation(policy.geolocation); + } + if (policy.autoGrant && policy.autoGrant.length > 0) { + await context.grantPermissions(policy.autoGrant, policy.origin); + } } async start(): Promise { @@ -276,6 +304,10 @@ export class SentienceBrowser implements IBrowser { this.context = await chromium.launchPersistentContext(this.userDataDir, launchOptions); + if (this._permissionPolicy) { + await this.applyPermissionPolicy(this.context, this._permissionPolicy); + } + this.page = this.context.pages()[0] || (await this.context.newPage()); // Inject storage state if provided (must be after context creation) diff --git a/src/index.ts b/src/index.ts index 86a62d9..c24f865 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,7 +2,7 @@ * Sentience TypeScript SDK - AI Agent Browser Automation */ -export { SentienceBrowser } from './browser'; +export { SentienceBrowser, PermissionPolicy } from './browser'; export { snapshot, SnapshotOptions } from './snapshot'; export { query, find, parseSelector } from './query'; export { diff --git a/src/tools/defaults.ts b/src/tools/defaults.ts index ab32340..013cba8 100644 --- a/src/tools/defaults.ts +++ b/src/tools/defaults.ts @@ -1,7 +1,7 @@ import { z } from 'zod'; import type { AgentRuntime } from '../agent-runtime'; import type { ActionResult, Snapshot, EvaluateJsResult } from '../types'; -import { ToolContext } from './context'; +import { ToolContext, UnsupportedCapabilityError } from './context'; import { defineTool, ToolRegistry } from './registry'; const snapshotSchema = z @@ -83,6 +83,19 @@ const evaluateJsInput = z.object({ truncate: z.boolean().default(true), }); +const grantPermissionsInput = z.object({ + permissions: z.array(z.string()).min(1), + origin: z.string().optional(), +}); + +const clearPermissionsInput = z.object({}); + +const setGeolocationInput = z.object({ + latitude: z.number(), + longitude: z.number(), + accuracy: z.number().optional(), +}); + function getRuntime(ctx: ToolContext | null, runtime?: ToolContext | AgentRuntime): AgentRuntime { if (ctx) return ctx.runtime; if (runtime instanceof ToolContext) return runtime.runtime; @@ -342,5 +355,87 @@ export function registerDefaultTools( }) ); + registry.register( + defineTool, ActionResult, ToolContext | null>({ + name: 'grant_permissions', + description: 'Grant browser permissions for the current context.', + input: grantPermissionsInput, + output: actionResultSchema, + handler: async (ctx, params): Promise => { + const runtimeRef = getRuntime(ctx, runtime); + if (ctx) { + ctx.require('permissions'); + } else if (!runtimeRef.can('permissions')) { + throw new UnsupportedCapabilityError('permissions'); + } + const context = + typeof (runtimeRef.page as any)?.context === 'function' + ? (runtimeRef.page as any).context() + : null; + if (!context) { + throw new Error('Permission context unavailable'); + } + await context.grantPermissions(params.permissions, params.origin); + return { success: true, duration_ms: 0, outcome: 'dom_updated' }; + }, + }) + ); + + registry.register( + defineTool, ActionResult, ToolContext | null>({ + name: 'clear_permissions', + description: 'Clear browser permissions for the current context.', + input: clearPermissionsInput, + output: actionResultSchema, + handler: async (ctx): Promise => { + const runtimeRef = getRuntime(ctx, runtime); + if (ctx) { + ctx.require('permissions'); + } else if (!runtimeRef.can('permissions')) { + throw new UnsupportedCapabilityError('permissions'); + } + const context = + typeof (runtimeRef.page as any)?.context === 'function' + ? (runtimeRef.page as any).context() + : null; + if (!context) { + throw new Error('Permission context unavailable'); + } + await context.clearPermissions(); + return { success: true, duration_ms: 0, outcome: 'dom_updated' }; + }, + }) + ); + + registry.register( + defineTool, ActionResult, ToolContext | null>({ + name: 'set_geolocation', + description: 'Set geolocation for the current browser context.', + input: setGeolocationInput, + output: actionResultSchema, + handler: async (ctx, params): Promise => { + const runtimeRef = getRuntime(ctx, runtime); + if (ctx) { + ctx.require('permissions'); + } else if (!runtimeRef.can('permissions')) { + throw new UnsupportedCapabilityError('permissions'); + } + const context = + typeof (runtimeRef.page as any)?.context === 'function' + ? (runtimeRef.page as any).context() + : null; + if (!context) { + throw new Error('Permission context unavailable'); + } + await context.setGeolocation({ + latitude: params.latitude, + longitude: params.longitude, + accuracy: params.accuracy, + }); + return { success: true, duration_ms: 0, outcome: 'dom_updated' }; + }, + }) + ); + return registry; } diff --git a/src/tools/registry.ts b/src/tools/registry.ts index 4e5e573..98260d0 100644 --- a/src/tools/registry.ts +++ b/src/tools/registry.ts @@ -84,7 +84,11 @@ export class ToolRegistry { name: string, payload: unknown, ctx: { - runtime?: { tracer?: { emit: (...args: any[]) => void }; stepId?: string; step_id?: string }; + runtime?: { + tracer?: { emit: (...args: any[]) => void }; + stepId?: string | null; + step_id?: string | null; + }; } | null = null ): Promise { const start = Date.now(); diff --git a/src/types.ts b/src/types.ts index 410cdd9..0d5b292 100644 --- a/src/types.ts +++ b/src/types.ts @@ -259,6 +259,7 @@ export interface BackendCapabilities { downloads: boolean; filesystem_tools: boolean; keyboard: boolean; + permissions: boolean; } export interface EvaluateJsRequest { diff --git a/tests/tool-registry.test.ts b/tests/tool-registry.test.ts index d2c34ff..da9c825 100644 --- a/tests/tool-registry.test.ts +++ b/tests/tool-registry.test.ts @@ -4,6 +4,8 @@ import path from 'path'; import { z } from 'zod'; import { defineTool, ToolRegistry } from '../src/tools/registry'; import { FileSandbox, registerFilesystemTools } from '../src/tools/filesystem'; +import { registerDefaultTools } from '../src/tools/defaults'; +import { ToolContext, UnsupportedCapabilityError } from '../src/tools/context'; describe('ToolRegistry', () => { it('validates and executes tools', async () => { @@ -35,3 +37,73 @@ describe('Filesystem tools', () => { expect(result.content).toBe('hi'); }); }); + +describe('Permission tools', () => { + it('grants permissions when supported', async () => { + const registry = new ToolRegistry(); + const calls: Array> = []; + const contextStub = { + grantPermissions: (permissions: string[], origin?: string) => { + calls.push({ kind: 'grant', permissions, origin }); + return Promise.resolve(); + }, + clearPermissions: () => Promise.resolve(), + setGeolocation: () => Promise.resolve(), + }; + + class RuntimeStub { + page = { context: () => contextStub }; + capabilities() { + return { + tabs: false, + evaluate_js: false, + downloads: false, + filesystem_tools: false, + keyboard: false, + permissions: true, + }; + } + can(name: keyof ReturnType) { + return Boolean(this.capabilities()[name]); + } + } + + const ctx = new ToolContext(new RuntimeStub() as any); + registerDefaultTools(registry, ctx); + await registry.execute( + 'grant_permissions', + { permissions: ['geolocation'], origin: 'https://x.com' }, + ctx + ); + expect(calls).toEqual([ + { kind: 'grant', permissions: ['geolocation'], origin: 'https://x.com' }, + ]); + }); + + it('rejects permissions when unsupported', async () => { + const registry = new ToolRegistry(); + + class RuntimeStub { + page = { context: () => null }; + capabilities() { + return { + tabs: false, + evaluate_js: false, + downloads: false, + filesystem_tools: false, + keyboard: false, + permissions: false, + }; + } + can(name: keyof ReturnType) { + return Boolean(this.capabilities()[name]); + } + } + + const ctx = new ToolContext(new RuntimeStub() as any); + registerDefaultTools(registry, ctx); + await expect( + registry.execute('grant_permissions', { permissions: ['geolocation'] }, ctx) + ).rejects.toBeInstanceOf(UnsupportedCapabilityError); + }); +}); From b8352432b7a1fd9ae8a511fd51250313b825d324 Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Fri, 23 Jan 2026 22:36:59 -0800 Subject: [PATCH 2/3] fix playwright grantPermissions call to pass {origin} --- src/browser.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/browser.ts b/src/browser.ts index d26d484..9894a2f 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -188,7 +188,8 @@ export class SentienceBrowser implements IBrowser { await context.setGeolocation(policy.geolocation); } if (policy.autoGrant && policy.autoGrant.length > 0) { - await context.grantPermissions(policy.autoGrant, policy.origin); + const options = policy.origin ? { origin: policy.origin } : undefined; + await context.grantPermissions(policy.autoGrant, options); } } From 4ac0d6ee77304b48952f7672c4983bcd0ec5cd0d Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Fri, 23 Jan 2026 23:15:37 -0800 Subject: [PATCH 3/3] intercept example.com to fix tests --- tests/actions.test.ts | 4 ++-- tests/browser.test.ts | 3 +++ tests/test-utils.ts | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/tests/actions.test.ts b/tests/actions.test.ts index 1456dd1..e9846f0 100644 --- a/tests/actions.test.ts +++ b/tests/actions.test.ts @@ -23,7 +23,7 @@ import { uncheck, uploadFile, } from '../src'; -import { createTestBrowser, getPageOrThrow } from './test-utils'; +import { createTestBrowser, getPageOrThrow, patchExampleDotCom } from './test-utils'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -220,8 +220,8 @@ describe('Actions', () => { try { await browser.start(); const page = getPageOrThrow(browser); + patchExampleDotCom(page); await page.goto('https://example.com'); - await page.waitForLoadState('networkidle', { timeout: 10000 }); await expect(search(browser, 'sentience sdk', 'duckduckgo')).rejects.toThrow( 'domain not allowed' diff --git a/tests/browser.test.ts b/tests/browser.test.ts index 393ecaa..0b448d2 100644 --- a/tests/browser.test.ts +++ b/tests/browser.test.ts @@ -7,6 +7,7 @@ import { chromium, BrowserContext, Page } from 'playwright'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; +import { patchExampleDotCom } from './test-utils'; describe('Browser Proxy Support', () => { describe('Proxy Parsing', () => { @@ -228,6 +229,7 @@ describe('Browser Proxy Support', () => { if (!page) { throw new Error('Browser page is not available'); } + patchExampleDotCom(page); await page.goto('https://example.com', { waitUntil: 'domcontentloaded', timeout: 20000 }); const viewportSize = await page.evaluate(() => ({ @@ -293,6 +295,7 @@ describe('Browser Proxy Support', () => { expect(sentienceBrowser.getContext()).toBe(context); // Test that we can use it + patchExampleDotCom(page); await page.goto('https://example.com'); await page.waitForLoadState('networkidle', { timeout: 10000 }); diff --git a/tests/test-utils.ts b/tests/test-utils.ts index 01a39cf..8c561d6 100644 --- a/tests/test-utils.ts +++ b/tests/test-utils.ts @@ -13,6 +13,10 @@ export async function createTestBrowser(headless?: boolean): Promise + + + + Example Link + + +
+ +`; + +export async function setTestPageContent(page: Page, html?: string): Promise { + await page.setContent(html ?? DEFAULT_TEST_HTML, { waitUntil: 'domcontentloaded' }); +} + +export function patchExampleDotCom(page: Page): void { + void page.route(/https?:\/\/example\.com\/?.*/, async route => { + await route.fulfill({ + status: 200, + contentType: 'text/html', + body: DEFAULT_TEST_HTML, + }); + }); +}