diff --git a/README.md b/README.md index 79c4273..de2804b 100644 --- a/README.md +++ b/README.md @@ -606,6 +606,22 @@ http.createServer(async (req, res) => { +- **browser_clear_device_motion** + - Title: Clear device motion override + - Description: Remove the device motion override and stop sending simulated motion events. + - Parameters: None + - Read-only: **false** + + + +- **browser_clear_device_orientation** + - Title: Clear device orientation override + - Description: Remove the device orientation override and return to default sensor behavior. + - Parameters: None + - Read-only: **false** + + + - **browser_clear_injections** - Title: Clear Injections - Description: Remove all custom code injections (keeps debug toolbar) @@ -1209,6 +1225,62 @@ Full API: See MODEL-COLLABORATION-API.md +- **browser_set_device_motion** + - Title: Set device motion sensors + - Description: Override accelerometer and gyroscope sensor values. Affects the DeviceMotionEvent API. + +**Acceleration** (m/s²): Linear acceleration excluding gravity +- x: left(-) to right(+) +- y: down(-) to up(+) +- z: backward(-) to forward(+) + +**Acceleration Including Gravity** (m/s²): Total acceleration including gravity +- At rest: { x: 0, y: -9.8, z: 0 } (Earth's gravity pulling down) + +**Rotation Rate** (deg/s): Angular velocity around each axis +- alpha: rotation around z-axis +- beta: rotation around x-axis +- gamma: rotation around y-axis + +**Common scenarios:** +- Device at rest: acceleration={x:0,y:0,z:0}, accelerationIncludingGravity={x:0,y:-9.8,z:0} +- Shaking horizontally: acceleration={x:5,y:0,z:0} +- Free fall: accelerationIncludingGravity={x:0,y:0,z:0} + +**Note:** Requires Chromium-based browser. + - Parameters: + - `acceleration` (object, optional): Linear acceleration excluding gravity + - `accelerationIncludingGravity` (object, optional): Total acceleration including gravity + - `rotationRate` (object, optional): Angular velocity around each axis + - `interval` (number, optional): Interval between samples in milliseconds (default: 16) + - Read-only: **false** + + + +- **browser_set_device_orientation** + - Title: Set device orientation + - Description: Override device orientation sensor values. Affects the DeviceOrientationEvent API. + +**Parameters:** +- **alpha** (0-360): Rotation around the z-axis (compass heading). 0 = North, 90 = East +- **beta** (-180 to 180): Front-to-back tilt. Positive = tilted backward +- **gamma** (-90 to 90): Left-to-right tilt. Positive = tilted right + +**Common orientations:** +- Flat on table: alpha=any, beta=0, gamma=0 +- Portrait upright: alpha=any, beta=90, gamma=0 +- Landscape left: alpha=any, beta=0, gamma=90 +- Tilted 45° forward: alpha=any, beta=-45, gamma=0 + +**Note:** Requires Chromium-based browser. This overrides the DeviceOrientationEvent. + - Parameters: + - `alpha` (number): Compass heading (0-360 degrees). 0=North, 90=East, 180=South, 270=West + - `beta` (number): Front-to-back tilt (-180 to 180 degrees). Positive=tilted backward + - `gamma` (number): Left-to-right tilt (-90 to 90 degrees). Positive=tilted right + - Read-only: **false** + + + - **browser_set_geolocation** - Title: Set geolocation at runtime - Description: Set the browser's geolocation at runtime without restarting. Automatically grants geolocation permission. diff --git a/src/tools.ts b/src/tools.ts index 8d0028d..1945809 100644 --- a/src/tools.ts +++ b/src/tools.ts @@ -28,6 +28,7 @@ import navigate from './tools/navigate.js'; import network from './tools/network.js'; import notifications from './tools/notifications.js'; import pdf from './tools/pdf.js'; +import sensors from './tools/sensors.js'; import requests from './tools/requests.js'; import snapshot from './tools/snapshot.js'; import tabs from './tools/tabs.js'; @@ -58,6 +59,7 @@ export const allTools: Tool[] = [ ...pdf, ...requests, ...screenshot, + ...sensors, ...snapshot, ...tabs, ...themeManagement, diff --git a/src/tools/sensors.ts b/src/tools/sensors.ts new file mode 100644 index 0000000..c0ae0f1 --- /dev/null +++ b/src/tools/sensors.ts @@ -0,0 +1,279 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from 'zod'; +import { defineTabTool } from './tool.js'; + +/** + * Set device orientation via CDP. + * This overrides the values returned by the DeviceOrientationEvent API. + */ +const setDeviceOrientation = defineTabTool({ + capability: 'core', + + schema: { + name: 'browser_set_device_orientation', + title: 'Set device orientation', + description: `Override device orientation sensor values. Affects the DeviceOrientationEvent API. + +**Parameters:** +- **alpha** (0-360): Rotation around the z-axis (compass heading). 0 = North, 90 = East +- **beta** (-180 to 180): Front-to-back tilt. Positive = tilted backward +- **gamma** (-90 to 90): Left-to-right tilt. Positive = tilted right + +**Common orientations:** +- Flat on table: alpha=any, beta=0, gamma=0 +- Portrait upright: alpha=any, beta=90, gamma=0 +- Landscape left: alpha=any, beta=0, gamma=90 +- Tilted 45° forward: alpha=any, beta=-45, gamma=0 + +**Note:** Requires Chromium-based browser. This overrides the DeviceOrientationEvent.`, + inputSchema: z.object({ + alpha: z.number().min(0).max(360).describe('Compass heading (0-360 degrees). 0=North, 90=East, 180=South, 270=West'), + beta: z.number().min(-180).max(180).describe('Front-to-back tilt (-180 to 180 degrees). Positive=tilted backward'), + gamma: z.number().min(-90).max(90).describe('Left-to-right tilt (-90 to 90 degrees). Positive=tilted right'), + }), + type: 'destructive', + }, + + handle: async (tab, params, response) => { + const page = tab.page; + const browserType = page.context().browser()?.browserType().name(); + + if (browserType !== 'chromium') + throw new Error(`Device orientation override requires Chromium browser. Current browser: ${browserType}`); + + const cdpSession = await page.context().newCDPSession(page); + + try { + // Cast to any because Playwright's types don't include all CDP commands + await (cdpSession as any).send('Emulation.setDeviceOrientationOverride', { + alpha: params.alpha, + beta: params.beta, + gamma: params.gamma, + }); + + response.addResult(`✅ Device orientation set: + • Alpha (compass): ${params.alpha}° (${getCompassDirection(params.alpha)}) + • Beta (front-back tilt): ${params.beta}° + • Gamma (left-right tilt): ${params.gamma}° + +Pages listening to DeviceOrientationEvent will now receive these values.`); + } finally { + await cdpSession.detach(); + } + }, +}); + +/** + * Clear device orientation override. + */ +const clearDeviceOrientation = defineTabTool({ + capability: 'core', + + schema: { + name: 'browser_clear_device_orientation', + title: 'Clear device orientation override', + description: 'Remove the device orientation override and return to default sensor behavior.', + inputSchema: z.object({}), + type: 'destructive', + }, + + handle: async (tab, _params, response) => { + const page = tab.page; + const browserType = page.context().browser()?.browserType().name(); + + if (browserType !== 'chromium') + throw new Error(`Device orientation override requires Chromium browser. Current browser: ${browserType}`); + + const cdpSession = await page.context().newCDPSession(page); + + try { + // Cast to any because Playwright's types don't include all CDP commands + await (cdpSession as any).send('Emulation.clearDeviceOrientationOverride'); + response.addResult('✅ Device orientation override cleared. Sensor will return to default behavior.'); + } finally { + await cdpSession.detach(); + } + }, +}); + +/** + * Set device motion (accelerometer + gyroscope) via CDP. + * This uses the generic sensor override API. + */ +const setDeviceMotion = defineTabTool({ + capability: 'core', + + schema: { + name: 'browser_set_device_motion', + title: 'Set device motion sensors', + description: `Override accelerometer and gyroscope sensor values. Affects the DeviceMotionEvent API. + +**Acceleration** (m/s²): Linear acceleration excluding gravity +- x: left(-) to right(+) +- y: down(-) to up(+) +- z: backward(-) to forward(+) + +**Acceleration Including Gravity** (m/s²): Total acceleration including gravity +- At rest: { x: 0, y: -9.8, z: 0 } (Earth's gravity pulling down) + +**Rotation Rate** (deg/s): Angular velocity around each axis +- alpha: rotation around z-axis +- beta: rotation around x-axis +- gamma: rotation around y-axis + +**Common scenarios:** +- Device at rest: acceleration={x:0,y:0,z:0}, accelerationIncludingGravity={x:0,y:-9.8,z:0} +- Shaking horizontally: acceleration={x:5,y:0,z:0} +- Free fall: accelerationIncludingGravity={x:0,y:0,z:0} + +**Note:** Requires Chromium-based browser.`, + inputSchema: z.object({ + acceleration: z.object({ + x: z.number().describe('Acceleration on x-axis (m/s²)'), + y: z.number().describe('Acceleration on y-axis (m/s²)'), + z: z.number().describe('Acceleration on z-axis (m/s²)'), + }).optional().describe('Linear acceleration excluding gravity'), + accelerationIncludingGravity: z.object({ + x: z.number().describe('Acceleration on x-axis including gravity (m/s²)'), + y: z.number().describe('Acceleration on y-axis including gravity (m/s²)'), + z: z.number().describe('Acceleration on z-axis including gravity (m/s²)'), + }).optional().describe('Total acceleration including gravity'), + rotationRate: z.object({ + alpha: z.number().describe('Rotation rate around z-axis (deg/s)'), + beta: z.number().describe('Rotation rate around x-axis (deg/s)'), + gamma: z.number().describe('Rotation rate around y-axis (deg/s)'), + }).optional().describe('Angular velocity around each axis'), + interval: z.number().optional().describe('Interval between samples in milliseconds (default: 16)'), + }), + type: 'destructive', + }, + + handle: async (tab, params, response) => { + const page = tab.page; + const browserType = page.context().browser()?.browserType().name(); + + if (browserType !== 'chromium') + throw new Error(`Device motion override requires Chromium browser. Current browser: ${browserType}`); + + // We need to inject a script to override the DeviceMotionEvent since CDP + // doesn't have a direct DeviceMotion override like it does for orientation. + // We'll create a mock that fires DeviceMotionEvent with our values. + + const acceleration = params.acceleration || { x: 0, y: 0, z: 0 }; + const accelerationIncludingGravity = params.accelerationIncludingGravity || { x: 0, y: -9.8, z: 0 }; + const rotationRate = params.rotationRate || { alpha: 0, beta: 0, gamma: 0 }; + const interval = params.interval || 16; + + await page.evaluate(({ acceleration, accelerationIncludingGravity, rotationRate, interval }) => { + // Store cleanup function globally + const win = window as Window & { __mcpDeviceMotionCleanup?: () => void }; + + // Clean up any existing override + if (win.__mcpDeviceMotionCleanup) + win.__mcpDeviceMotionCleanup(); + + // Create the mock DeviceMotionEvent data + const motionData = { + acceleration, + accelerationIncludingGravity, + rotationRate, + interval, + }; + + // Override the DeviceMotionEvent by dispatching custom events + const dispatchMotion = () => { + const event = new DeviceMotionEvent('devicemotion', motionData); + window.dispatchEvent(event); + }; + + // Start dispatching at the specified interval + const intervalId = setInterval(dispatchMotion, interval); + + // Dispatch immediately + dispatchMotion(); + + // Store cleanup + win.__mcpDeviceMotionCleanup = () => { + clearInterval(intervalId); + delete win.__mcpDeviceMotionCleanup; + }; + }, { acceleration, accelerationIncludingGravity, rotationRate, interval }); + + const output = ['✅ Device motion sensors set:']; + + if (params.acceleration) + output.push(` • Acceleration: x=${acceleration.x}, y=${acceleration.y}, z=${acceleration.z} m/s²`); + + if (params.accelerationIncludingGravity) + output.push(` • Acceleration (with gravity): x=${accelerationIncludingGravity.x}, y=${accelerationIncludingGravity.y}, z=${accelerationIncludingGravity.z} m/s²`); + + if (params.rotationRate) + output.push(` • Rotation rate: α=${rotationRate.alpha}, β=${rotationRate.beta}, γ=${rotationRate.gamma} deg/s`); + + output.push(` • Update interval: ${interval}ms`); + output.push('\nDeviceMotionEvent listeners will now receive these values.'); + + response.addResult(output.join('\n')); + }, +}); + +/** + * Clear device motion override. + */ +const clearDeviceMotion = defineTabTool({ + capability: 'core', + + schema: { + name: 'browser_clear_device_motion', + title: 'Clear device motion override', + description: 'Remove the device motion override and stop sending simulated motion events.', + inputSchema: z.object({}), + type: 'destructive', + }, + + handle: async (tab, _params, response) => { + const page = tab.page; + + await page.evaluate(() => { + const win = window as Window & { __mcpDeviceMotionCleanup?: () => void }; + if (win.__mcpDeviceMotionCleanup) { + win.__mcpDeviceMotionCleanup(); + return true; + } + return false; + }); + + response.addResult('✅ Device motion override cleared.'); + }, +}); + +/** + * Helper to convert compass heading to direction. + */ +function getCompassDirection(alpha: number): string { + const directions = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW']; + const index = Math.round(alpha / 45) % 8; + return directions[index]; +} + +export default [ + setDeviceOrientation, + clearDeviceOrientation, + setDeviceMotion, + clearDeviceMotion, +]; diff --git a/tests/capabilities.spec.ts b/tests/capabilities.spec.ts index 5f33035..7054dd3 100644 --- a/tests/capabilities.spec.ts +++ b/tests/capabilities.spec.ts @@ -19,30 +19,78 @@ import { test, expect } from './fixtures.js'; test('test snapshot tool list', async ({ client }) => { const { tools } = await client.listTools(); expect(new Set(tools.map(t => t.name))).toEqual(new Set([ + 'browser_clear_device_motion', + 'browser_clear_device_orientation', + 'browser_clear_injections', + 'browser_clear_notifications', + 'browser_clear_permissions', + 'browser_clear_requests', 'browser_click', - 'browser_console_messages', - 'browser_drag', - 'browser_evaluate', - 'browser_file_upload', - 'browser_handle_dialog', - 'browser_hover', - 'browser_select_option', - 'browser_type', 'browser_close', + 'browser_configure', + 'browser_configure_artifacts', + 'browser_configure_notifications', + 'browser_configure_snapshots', + 'browser_console_messages', + 'browser_disable_debug_toolbar', + 'browser_dismiss_all_file_choosers', + 'browser_dismiss_file_chooser', + 'browser_drag', + 'browser_enable_debug_toolbar', + 'browser_enable_voice_collaboration', + 'browser_evaluate', + 'browser_export_requests', + 'browser_file_upload', + 'browser_get_artifact_paths', + 'browser_get_requests', + 'browser_grant_permissions', + 'browser_handle_dialog', + 'browser_handle_notification', + 'browser_hover', + 'browser_inject_custom_code', 'browser_install', + 'browser_install_extension', + 'browser_install_popular_extension', + 'browser_list_devices', + 'browser_list_extensions', + 'browser_list_injections', + 'browser_list_notifications', + 'browser_mcp_theme_create', + 'browser_mcp_theme_get', + 'browser_mcp_theme_list', + 'browser_mcp_theme_reset', + 'browser_mcp_theme_set', + 'browser_navigate', 'browser_navigate_back', 'browser_navigate_forward', - 'browser_navigate', 'browser_network_requests', + 'browser_pause_recording', 'browser_press_key', + 'browser_recording_status', + 'browser_request_monitoring_status', 'browser_resize', + 'browser_resume_recording', + 'browser_reveal_artifact_paths', + 'browser_select_option', + 'browser_set_device_motion', + 'browser_set_device_orientation', + 'browser_set_geolocation', + 'browser_set_offline', + 'browser_set_recording_mode', 'browser_snapshot', + 'browser_start_recording', + 'browser_start_request_monitoring', + 'browser_status', + 'browser_stop_recording', 'browser_tab_close', 'browser_tab_list', 'browser_tab_new', 'browser_tab_select', 'browser_take_screenshot', + 'browser_type', + 'browser_uninstall_extension', 'browser_wait_for', + 'browser_wait_notification', ])); });