feat: add device orientation and motion sensor mocking tools

Add CDP-based sensor mocking tools for testing motion-sensitive web apps: - browser_set_device_orientation: Override DeviceOrientationEvent values (compass heading, front-back tilt, left-right tilt) - browser_clear_device_orientation: Clear orientation override - browser_set_device_motion: Override accelerometer and gyroscope values (acceleration, acceleration with gravity, rotation rate) - browser_clear_device_motion: Clear motion override These tools use Chrome DevTools Protocol (Chromium only) to mock sensor readings, enabling testing of AR/VR apps, compass apps, games, and other motion-sensitive web applications. Also updates test snapshots to include all new core tools added recently.
2026-01-13 01:18:06 -07:00 · 2026-01-13 01:18:06 -07:00 · a7d10f71bd
commit a7d10f71bd
parent b0ded548cf
4 changed files with 410 additions and 9 deletions
--- a/README.md
+++ b/README.md
@ -606,6 +606,22 @@ http.createServer(async (req, res) => {

 <!-- NOTE: This has been generated via update-readme.js -->

+- **browser_clear_device_motion**
+  - Title: Clear device motion override
+  - Description: Remove the device motion override and stop sending simulated motion events.
+  - Parameters: None
+  - Read-only: **false**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
+- **browser_clear_device_orientation**
+  - Title: Clear device orientation override
+  - Description: Remove the device orientation override and return to default sensor behavior.
+  - Parameters: None
+  - Read-only: **false**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
 - **browser_clear_injections**
  - Title: Clear Injections
  - Description: Remove all custom code injections (keeps debug toolbar)
@ -1209,6 +1225,62 @@ Full API: See MODEL-COLLABORATION-API.md

 <!-- NOTE: This has been generated via update-readme.js -->

+- **browser_set_device_motion**
+  - Title: Set device motion sensors
+  - Description: Override accelerometer and gyroscope sensor values. Affects the DeviceMotionEvent API.
+
+**Acceleration** (m/s²): Linear acceleration excluding gravity
+- x: left(-) to right(+)
+- y: down(-) to up(+)
+- z: backward(-) to forward(+)
+
+**Acceleration Including Gravity** (m/s²): Total acceleration including gravity
+- At rest: { x: 0, y: -9.8, z: 0 } (Earth's gravity pulling down)
+
+**Rotation Rate** (deg/s): Angular velocity around each axis
+- alpha: rotation around z-axis
+- beta: rotation around x-axis
+- gamma: rotation around y-axis
+
+**Common scenarios:**
+- Device at rest: acceleration={x:0,y:0,z:0}, accelerationIncludingGravity={x:0,y:-9.8,z:0}
+- Shaking horizontally: acceleration={x:5,y:0,z:0}
+- Free fall: accelerationIncludingGravity={x:0,y:0,z:0}
+
+**Note:** Requires Chromium-based browser.
+  - Parameters:
+    - `acceleration` (object, optional): Linear acceleration excluding gravity
+    - `accelerationIncludingGravity` (object, optional): Total acceleration including gravity
+    - `rotationRate` (object, optional): Angular velocity around each axis
+    - `interval` (number, optional): Interval between samples in milliseconds (default: 16)
+  - Read-only: **false**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
+- **browser_set_device_orientation**
+  - Title: Set device orientation
+  - Description: Override device orientation sensor values. Affects the DeviceOrientationEvent API.
+
+**Parameters:**
+- **alpha** (0-360): Rotation around the z-axis (compass heading). 0 = North, 90 = East
+- **beta** (-180 to 180): Front-to-back tilt. Positive = tilted backward
+- **gamma** (-90 to 90): Left-to-right tilt. Positive = tilted right
+
+**Common orientations:**
+- Flat on table: alpha=any, beta=0, gamma=0
+- Portrait upright: alpha=any, beta=90, gamma=0
+- Landscape left: alpha=any, beta=0, gamma=90
+- Tilted 45° forward: alpha=any, beta=-45, gamma=0
+
+**Note:** Requires Chromium-based browser. This overrides the DeviceOrientationEvent.
+  - Parameters:
+    - `alpha` (number): Compass heading (0-360 degrees). 0=North, 90=East, 180=South, 270=West
+    - `beta` (number): Front-to-back tilt (-180 to 180 degrees). Positive=tilted backward
+    - `gamma` (number): Left-to-right tilt (-90 to 90 degrees). Positive=tilted right
+  - Read-only: **false**
+
+<!-- NOTE: This has been generated via update-readme.js -->
+
 - **browser_set_geolocation**
  - Title: Set geolocation at runtime
  - Description: Set the browser's geolocation at runtime without restarting. Automatically grants geolocation permission.
--- a/src/tools.ts
+++ b/src/tools.ts
@ -28,6 +28,7 @@ import navigate from './tools/navigate.js';
 import network from './tools/network.js';
 import notifications from './tools/notifications.js';
 import pdf from './tools/pdf.js';
+import sensors from './tools/sensors.js';
 import requests from './tools/requests.js';
 import snapshot from './tools/snapshot.js';
 import tabs from './tools/tabs.js';
@ -58,6 +59,7 @@ export const allTools: Tool<any>[] = [
  ...pdf,
  ...requests,
  ...screenshot,
+  ...sensors,
  ...snapshot,
  ...tabs,
  ...themeManagement,
--- a/src/tools/sensors.ts
+++ b/src/tools/sensors.ts
@ -0,0 +1,279 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+import { defineTabTool } from './tool.js';
+
+/**
+ * Set device orientation via CDP.
+ * This overrides the values returned by the DeviceOrientationEvent API.
+ */
+const setDeviceOrientation = defineTabTool({
+  capability: 'core',
+
+  schema: {
+    name: 'browser_set_device_orientation',
+    title: 'Set device orientation',
+    description: `Override device orientation sensor values. Affects the DeviceOrientationEvent API.
+
+**Parameters:**
+- **alpha** (0-360): Rotation around the z-axis (compass heading). 0 = North, 90 = East
+- **beta** (-180 to 180): Front-to-back tilt. Positive = tilted backward
+- **gamma** (-90 to 90): Left-to-right tilt. Positive = tilted right
+
+**Common orientations:**
+- Flat on table: alpha=any, beta=0, gamma=0
+- Portrait upright: alpha=any, beta=90, gamma=0
+- Landscape left: alpha=any, beta=0, gamma=90
+- Tilted 45° forward: alpha=any, beta=-45, gamma=0
+
+**Note:** Requires Chromium-based browser. This overrides the DeviceOrientationEvent.`,
+    inputSchema: z.object({
+      alpha: z.number().min(0).max(360).describe('Compass heading (0-360 degrees). 0=North, 90=East, 180=South, 270=West'),
+      beta: z.number().min(-180).max(180).describe('Front-to-back tilt (-180 to 180 degrees). Positive=tilted backward'),
+      gamma: z.number().min(-90).max(90).describe('Left-to-right tilt (-90 to 90 degrees). Positive=tilted right'),
+    }),
+    type: 'destructive',
+  },
+
+  handle: async (tab, params, response) => {
+    const page = tab.page;
+    const browserType = page.context().browser()?.browserType().name();
+
+    if (browserType !== 'chromium')
+      throw new Error(`Device orientation override requires Chromium browser. Current browser: ${browserType}`);
+
+    const cdpSession = await page.context().newCDPSession(page);
+
+    try {
+      // Cast to any because Playwright's types don't include all CDP commands
+      await (cdpSession as any).send('Emulation.setDeviceOrientationOverride', {
+        alpha: params.alpha,
+        beta: params.beta,
+        gamma: params.gamma,
+      });
+
+      response.addResult(`✅ Device orientation set:
+  • Alpha (compass): ${params.alpha}° (${getCompassDirection(params.alpha)})
+  • Beta (front-back tilt): ${params.beta}°
+  • Gamma (left-right tilt): ${params.gamma}°
+
+Pages listening to DeviceOrientationEvent will now receive these values.`);
+    } finally {
+      await cdpSession.detach();
+    }
+  },
+});
+
+/**
+ * Clear device orientation override.
+ */
+const clearDeviceOrientation = defineTabTool({
+  capability: 'core',
+
+  schema: {
+    name: 'browser_clear_device_orientation',
+    title: 'Clear device orientation override',
+    description: 'Remove the device orientation override and return to default sensor behavior.',
+    inputSchema: z.object({}),
+    type: 'destructive',
+  },
+
+  handle: async (tab, _params, response) => {
+    const page = tab.page;
+    const browserType = page.context().browser()?.browserType().name();
+
+    if (browserType !== 'chromium')
+      throw new Error(`Device orientation override requires Chromium browser. Current browser: ${browserType}`);
+
+    const cdpSession = await page.context().newCDPSession(page);
+
+    try {
+      // Cast to any because Playwright's types don't include all CDP commands
+      await (cdpSession as any).send('Emulation.clearDeviceOrientationOverride');
+      response.addResult('✅ Device orientation override cleared. Sensor will return to default behavior.');
+    } finally {
+      await cdpSession.detach();
+    }
+  },
+});
+
+/**
+ * Set device motion (accelerometer + gyroscope) via CDP.
+ * This uses the generic sensor override API.
+ */
+const setDeviceMotion = defineTabTool({
+  capability: 'core',
+
+  schema: {
+    name: 'browser_set_device_motion',
+    title: 'Set device motion sensors',
+    description: `Override accelerometer and gyroscope sensor values. Affects the DeviceMotionEvent API.
+
+**Acceleration** (m/s²): Linear acceleration excluding gravity
+- x: left(-) to right(+)
+- y: down(-) to up(+)
+- z: backward(-) to forward(+)
+
+**Acceleration Including Gravity** (m/s²): Total acceleration including gravity
+- At rest: { x: 0, y: -9.8, z: 0 } (Earth's gravity pulling down)
+
+**Rotation Rate** (deg/s): Angular velocity around each axis
+- alpha: rotation around z-axis
+- beta: rotation around x-axis
+- gamma: rotation around y-axis
+
+**Common scenarios:**
+- Device at rest: acceleration={x:0,y:0,z:0}, accelerationIncludingGravity={x:0,y:-9.8,z:0}
+- Shaking horizontally: acceleration={x:5,y:0,z:0}
+- Free fall: accelerationIncludingGravity={x:0,y:0,z:0}
+
+**Note:** Requires Chromium-based browser.`,
+    inputSchema: z.object({
+      acceleration: z.object({
+        x: z.number().describe('Acceleration on x-axis (m/s²)'),
+        y: z.number().describe('Acceleration on y-axis (m/s²)'),
+        z: z.number().describe('Acceleration on z-axis (m/s²)'),
+      }).optional().describe('Linear acceleration excluding gravity'),
+      accelerationIncludingGravity: z.object({
+        x: z.number().describe('Acceleration on x-axis including gravity (m/s²)'),
+        y: z.number().describe('Acceleration on y-axis including gravity (m/s²)'),
+        z: z.number().describe('Acceleration on z-axis including gravity (m/s²)'),
+      }).optional().describe('Total acceleration including gravity'),
+      rotationRate: z.object({
+        alpha: z.number().describe('Rotation rate around z-axis (deg/s)'),
+        beta: z.number().describe('Rotation rate around x-axis (deg/s)'),
+        gamma: z.number().describe('Rotation rate around y-axis (deg/s)'),
+      }).optional().describe('Angular velocity around each axis'),
+      interval: z.number().optional().describe('Interval between samples in milliseconds (default: 16)'),
+    }),
+    type: 'destructive',
+  },
+
+  handle: async (tab, params, response) => {
+    const page = tab.page;
+    const browserType = page.context().browser()?.browserType().name();
+
+    if (browserType !== 'chromium')
+      throw new Error(`Device motion override requires Chromium browser. Current browser: ${browserType}`);
+
+    // We need to inject a script to override the DeviceMotionEvent since CDP
+    // doesn't have a direct DeviceMotion override like it does for orientation.
+    // We'll create a mock that fires DeviceMotionEvent with our values.
+
+    const acceleration = params.acceleration || { x: 0, y: 0, z: 0 };
+    const accelerationIncludingGravity = params.accelerationIncludingGravity || { x: 0, y: -9.8, z: 0 };
+    const rotationRate = params.rotationRate || { alpha: 0, beta: 0, gamma: 0 };
+    const interval = params.interval || 16;
+
+    await page.evaluate(({ acceleration, accelerationIncludingGravity, rotationRate, interval }) => {
+      // Store cleanup function globally
+      const win = window as Window & { __mcpDeviceMotionCleanup?: () => void };
+
+      // Clean up any existing override
+      if (win.__mcpDeviceMotionCleanup)
+        win.__mcpDeviceMotionCleanup();
+
+      // Create the mock DeviceMotionEvent data
+      const motionData = {
+        acceleration,
+        accelerationIncludingGravity,
+        rotationRate,
+        interval,
+      };
+
+      // Override the DeviceMotionEvent by dispatching custom events
+      const dispatchMotion = () => {
+        const event = new DeviceMotionEvent('devicemotion', motionData);
+        window.dispatchEvent(event);
+      };
+
+      // Start dispatching at the specified interval
+      const intervalId = setInterval(dispatchMotion, interval);
+
+      // Dispatch immediately
+      dispatchMotion();
+
+      // Store cleanup
+      win.__mcpDeviceMotionCleanup = () => {
+        clearInterval(intervalId);
+        delete win.__mcpDeviceMotionCleanup;
+      };
+    }, { acceleration, accelerationIncludingGravity, rotationRate, interval });
+
+    const output = ['✅ Device motion sensors set:'];
+
+    if (params.acceleration)
+      output.push(`  • Acceleration: x=${acceleration.x}, y=${acceleration.y}, z=${acceleration.z} m/s²`);
+
+    if (params.accelerationIncludingGravity)
+      output.push(`  • Acceleration (with gravity): x=${accelerationIncludingGravity.x}, y=${accelerationIncludingGravity.y}, z=${accelerationIncludingGravity.z} m/s²`);
+
+    if (params.rotationRate)
+      output.push(`  • Rotation rate: α=${rotationRate.alpha}, β=${rotationRate.beta}, γ=${rotationRate.gamma} deg/s`);
+
+    output.push(`  • Update interval: ${interval}ms`);
+    output.push('\nDeviceMotionEvent listeners will now receive these values.');
+
+    response.addResult(output.join('\n'));
+  },
+});
+
+/**
+ * Clear device motion override.
+ */
+const clearDeviceMotion = defineTabTool({
+  capability: 'core',
+
+  schema: {
+    name: 'browser_clear_device_motion',
+    title: 'Clear device motion override',
+    description: 'Remove the device motion override and stop sending simulated motion events.',
+    inputSchema: z.object({}),
+    type: 'destructive',
+  },
+
+  handle: async (tab, _params, response) => {
+    const page = tab.page;
+
+    await page.evaluate(() => {
+      const win = window as Window & { __mcpDeviceMotionCleanup?: () => void };
+      if (win.__mcpDeviceMotionCleanup) {
+        win.__mcpDeviceMotionCleanup();
+        return true;
+      }
+      return false;
+    });
+
+    response.addResult('✅ Device motion override cleared.');
+  },
+});
+
+/**
+ * Helper to convert compass heading to direction.
+ */
+function getCompassDirection(alpha: number): string {
+  const directions = ['N', 'NE', 'E', 'SE', 'S', 'SW', 'W', 'NW'];
+  const index = Math.round(alpha / 45) % 8;
+  return directions[index];
+}
+
+export default [
+  setDeviceOrientation,
+  clearDeviceOrientation,
+  setDeviceMotion,
+  clearDeviceMotion,
+];
--- a/tests/capabilities.spec.ts
+++ b/tests/capabilities.spec.ts
@ -19,30 +19,78 @@ import { test, expect } from './fixtures.js';
 test('test snapshot tool list', async ({ client }) => {
  const { tools } = await client.listTools();
  expect(new Set(tools.map(t => t.name))).toEqual(new Set([
+    'browser_clear_device_motion',
+    'browser_clear_device_orientation',
+    'browser_clear_injections',
+    'browser_clear_notifications',
+    'browser_clear_permissions',
+    'browser_clear_requests',
    'browser_click',
-    'browser_console_messages',
-    'browser_drag',
-    'browser_evaluate',
-    'browser_file_upload',
-    'browser_handle_dialog',
-    'browser_hover',
-    'browser_select_option',
-    'browser_type',
    'browser_close',
+    'browser_configure',
+    'browser_configure_artifacts',
+    'browser_configure_notifications',
+    'browser_configure_snapshots',
+    'browser_console_messages',
+    'browser_disable_debug_toolbar',
+    'browser_dismiss_all_file_choosers',
+    'browser_dismiss_file_chooser',
+    'browser_drag',
+    'browser_enable_debug_toolbar',
+    'browser_enable_voice_collaboration',
+    'browser_evaluate',
+    'browser_export_requests',
+    'browser_file_upload',
+    'browser_get_artifact_paths',
+    'browser_get_requests',
+    'browser_grant_permissions',
+    'browser_handle_dialog',
+    'browser_handle_notification',
+    'browser_hover',
+    'browser_inject_custom_code',
    'browser_install',
+    'browser_install_extension',
+    'browser_install_popular_extension',
+    'browser_list_devices',
+    'browser_list_extensions',
+    'browser_list_injections',
+    'browser_list_notifications',
+    'browser_mcp_theme_create',
+    'browser_mcp_theme_get',
+    'browser_mcp_theme_list',
+    'browser_mcp_theme_reset',
+    'browser_mcp_theme_set',
+    'browser_navigate',
    'browser_navigate_back',
    'browser_navigate_forward',
-    'browser_navigate',
    'browser_network_requests',
+    'browser_pause_recording',
    'browser_press_key',
+    'browser_recording_status',
+    'browser_request_monitoring_status',
    'browser_resize',
+    'browser_resume_recording',
+    'browser_reveal_artifact_paths',
+    'browser_select_option',
+    'browser_set_device_motion',
+    'browser_set_device_orientation',
+    'browser_set_geolocation',
+    'browser_set_offline',
+    'browser_set_recording_mode',
    'browser_snapshot',
+    'browser_start_recording',
+    'browser_start_request_monitoring',
+    'browser_status',
+    'browser_stop_recording',
    'browser_tab_close',
    'browser_tab_list',
    'browser_tab_new',
    'browser_tab_select',
    'browser_take_screenshot',
+    'browser_type',
+    'browser_uninstall_extension',
    'browser_wait_for',
+    'browser_wait_notification',
  ]));
 });