chore: brush up mcp servers (#35103)
This commit is contained in:
Pavel Feldman
GitHub
parent
07f54e7d8a
commit
a586a90e78
|
|
@ -127,22 +127,6 @@
|
|||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/sdk": {
|
||||
"version": "0.33.1",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.33.1.tgz",
|
||||
"integrity": "sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "^18.11.18",
|
||||
"@types/node-fetch": "^2.6.4",
|
||||
"abort-controller": "^3.0.0",
|
||||
"agentkeepalive": "^4.2.1",
|
||||
"form-data-encoder": "1.7.2",
|
||||
"formdata-node": "^4.3.2",
|
||||
"node-fetch": "^2.6.7"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/cli": {
|
||||
"version": "7.26.4",
|
||||
"resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.26.4.tgz",
|
||||
|
|
@ -1638,8 +1622,8 @@
|
|||
"resolved": "packages/playwright-ct-vue",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@playwright/experimental-tools": {
|
||||
"resolved": "packages/playwright-tools",
|
||||
"node_modules/@playwright/mcp": {
|
||||
"resolved": "packages/playwright-mcp",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@playwright/test": {
|
||||
|
|
@ -2078,17 +2062,6 @@
|
|||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node-fetch": {
|
||||
"version": "2.6.12",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz",
|
||||
"integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/prop-types": {
|
||||
"version": "15.7.14",
|
||||
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz",
|
||||
|
|
@ -2574,19 +2547,6 @@
|
|||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/abort-controller": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
||||
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"event-target-shim": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.5"
|
||||
}
|
||||
},
|
||||
"node_modules/accepts": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz",
|
||||
|
|
@ -2643,19 +2603,6 @@
|
|||
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agentkeepalive": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz",
|
||||
"integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"humanize-ms": "^1.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "6.12.6",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
|
||||
|
|
@ -2926,13 +2873,6 @@
|
|||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/available-typed-arrays": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz",
|
||||
|
|
@ -3335,19 +3275,6 @@
|
|||
"node": ">=0.1.90"
|
||||
}
|
||||
},
|
||||
"node_modules/combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/commander": {
|
||||
"version": "6.2.1",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz",
|
||||
|
|
@ -3689,16 +3616,6 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/depd": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
|
||||
|
|
@ -4492,16 +4409,6 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/event-target-shim": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
||||
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/eventsource": {
|
||||
"version": "3.0.5",
|
||||
"resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.5.tgz",
|
||||
|
|
@ -4823,43 +4730,6 @@
|
|||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/form-data": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz",
|
||||
"integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"es-set-tostringtag": "^2.1.0",
|
||||
"mime-types": "^2.1.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/form-data-encoder": {
|
||||
"version": "1.7.2",
|
||||
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
|
||||
"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/formdata-node": {
|
||||
"version": "4.4.1",
|
||||
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
|
||||
"integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-domexception": "1.0.0",
|
||||
"web-streams-polyfill": "4.0.0-beta.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 12.20"
|
||||
}
|
||||
},
|
||||
"node_modules/formidable": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
|
||||
|
|
@ -5369,16 +5239,6 @@
|
|||
"node": ">=10.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/humanize-ms": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
|
||||
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ms": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.5.2",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.2.tgz",
|
||||
|
|
@ -6529,47 +6389,6 @@
|
|||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.19",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
|
||||
|
|
@ -6801,37 +6620,6 @@
|
|||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"node_modules/openai": {
|
||||
"version": "4.85.1",
|
||||
"resolved": "https://registry.npmjs.org/openai/-/openai-4.85.1.tgz",
|
||||
"integrity": "sha512-jkX2fntHljUvSH3MkWh4jShl10oNkb+SsCj4auKlbu2oF4KWAnmHLNR5EpnUHK1ZNW05Rp0fjbJzYwQzMsH8ZA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@types/node": "^18.11.18",
|
||||
"@types/node-fetch": "^2.6.4",
|
||||
"abort-controller": "^3.0.0",
|
||||
"agentkeepalive": "^4.2.1",
|
||||
"form-data-encoder": "1.7.2",
|
||||
"formdata-node": "^4.3.2",
|
||||
"node-fetch": "^2.6.7"
|
||||
},
|
||||
"bin": {
|
||||
"openai": "bin/cli"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"ws": "^8.18.0",
|
||||
"zod": "^3.23.8"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"ws": {
|
||||
"optional": true
|
||||
},
|
||||
"zod": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/optionator": {
|
||||
"version": "0.9.4",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
|
||||
|
|
@ -8345,13 +8133,6 @@
|
|||
"node": ">=0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/trace-viewer": {
|
||||
"resolved": "packages/trace-viewer",
|
||||
"link": true
|
||||
|
|
@ -9264,34 +9045,6 @@
|
|||
"resolved": "packages/web",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "4.0.0-beta.3",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
|
||||
"integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
|
||||
"dev": true,
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
|
|
@ -10308,6 +10061,20 @@
|
|||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"packages/playwright-mcp": {
|
||||
"name": "@playwright/mcp",
|
||||
"version": "0.0.1",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright": "1.52.0-next"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.6.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"packages/playwright-test": {
|
||||
"name": "@playwright/test",
|
||||
"version": "1.52.0-next",
|
||||
|
|
@ -10325,6 +10092,7 @@
|
|||
"packages/playwright-tools": {
|
||||
"name": "@playwright/experimental-tools",
|
||||
"version": "0.0.0",
|
||||
"extraneous": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright": "1.52.0-next"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
{
|
||||
"name": "@playwright/mcp",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"description": "Playwright Tools for MCP",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/microsoft/playwright.git"
|
||||
},
|
||||
"homepage": "https://playwright.dev",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"author": {
|
||||
"name": "Microsoft Corporation"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"exports": {
|
||||
"./servers/server": "./lib/servers/server.js",
|
||||
"./servers/screenshot": "./lib/servers/screenshot.js",
|
||||
"./servers/snapshot": "./lib/servers/snapshot.js",
|
||||
"./tools/common": "./lib/tools/common.js",
|
||||
"./tools/screenshot": "./lib/tools/screenshot.js",
|
||||
"./tools/snapshot": "./lib/tools/snapshot.js",
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"playwright": "1.52.0-next"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@modelcontextprotocol/sdk": "^1.6.1"
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,22 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
const { schema, call, snapshot } = require('./lib/tools/browser');
|
||||
import { Server } from './server';
|
||||
import { navigate, wait, pressKey } from '../tools/common';
|
||||
import { screenshot, moveMouse, click, drag, type } from '../tools/screenshot';
|
||||
|
||||
module.exports = { schema, call, snapshot };
|
||||
const server = new Server({
|
||||
name: 'Playwright screenshot-based browser server',
|
||||
version: '0.0.1',
|
||||
tools: [
|
||||
navigate,
|
||||
screenshot,
|
||||
moveMouse,
|
||||
click,
|
||||
drag,
|
||||
type,
|
||||
pressKey,
|
||||
wait,
|
||||
]
|
||||
});
|
||||
server.start();
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Server as MCPServer } from '@modelcontextprotocol/sdk/server/index.js';
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
||||
import * as playwright from 'playwright';
|
||||
|
||||
import type { Tool } from '../tools/common';
|
||||
|
||||
export class Server {
|
||||
private _server: MCPServer;
|
||||
private _tools: Tool[];
|
||||
private _page: playwright.Page | undefined;
|
||||
|
||||
constructor(options: { name: string, version: string, tools: Tool[] }) {
|
||||
const { name, version, tools } = options;
|
||||
this._server = new MCPServer({ name, version }, { capabilities: { tools: {} } });
|
||||
this._tools = tools;
|
||||
|
||||
this._server.setRequestHandler(ListToolsRequestSchema, async () => {
|
||||
return { tools: tools.map(tool => tool.schema) };
|
||||
});
|
||||
|
||||
this._server.setRequestHandler(CallToolRequestSchema, async request => {
|
||||
const page = await this._openPage();
|
||||
|
||||
const tool = this._tools.find(tool => tool.schema.name === request.params.name);
|
||||
if (!tool) {
|
||||
return {
|
||||
content: [{ type: 'text', text: `Tool "${request.params.name}" not found` }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await tool.handle({ page }, request.params.arguments);
|
||||
return result;
|
||||
} catch (error) {
|
||||
return {
|
||||
content: [{ type: 'text', text: String(error) }],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
this._setupExitWatchdog();
|
||||
}
|
||||
|
||||
start() {
|
||||
const transport = new StdioServerTransport();
|
||||
void this._server.connect(transport);
|
||||
}
|
||||
|
||||
private async _createBrowser(): Promise<playwright.Browser> {
|
||||
if (process.env.PLAYWRIGHT_WS_ENDPOINT) {
|
||||
return await playwright.chromium.connect(
|
||||
process.env.PLAYWRIGHT_WS_ENDPOINT
|
||||
);
|
||||
}
|
||||
return await playwright.chromium.launch({ headless: false });
|
||||
}
|
||||
|
||||
private async _openPage(): Promise<playwright.Page> {
|
||||
if (!this._page) {
|
||||
const browser = await this._createBrowser();
|
||||
const context = await browser.newContext();
|
||||
this._page = await context.newPage();
|
||||
}
|
||||
return this._page;
|
||||
}
|
||||
|
||||
private _setupExitWatchdog() {
|
||||
process.stdin.on('close', async () => {
|
||||
this._server.close();
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
setTimeout(() => process.exit(0), 15000);
|
||||
await this._page?.context()?.browser()?.close();
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -14,14 +14,21 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import type playwright from 'playwright';
|
||||
import { Server } from './server';
|
||||
import { wait, pressKey } from '../tools/common';
|
||||
import { navigate, snapshot, click, hover, type } from '../tools/snapshot';
|
||||
|
||||
export type JSONSchemaType = string | number | boolean | JSONSchemaObject | JSONSchemaArray | null;
|
||||
interface JSONSchemaObject { [key: string]: JSONSchemaType; }
|
||||
interface JSONSchemaArray extends Array<JSONSchemaType> {}
|
||||
|
||||
export type ToolDeclaration = {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: any;
|
||||
};
|
||||
const server = new Server({
|
||||
name: 'Playwright snapshot-based browser server',
|
||||
version: '0.0.1',
|
||||
tools: [
|
||||
navigate,
|
||||
snapshot,
|
||||
click,
|
||||
hover,
|
||||
type,
|
||||
pressKey,
|
||||
wait,
|
||||
]
|
||||
});
|
||||
server.start();
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { waitForCompletion } from '../utils';
|
||||
|
||||
import type * as playwright from 'playwright';
|
||||
import type { ImageContent, TextContent } from '@modelcontextprotocol/sdk/types';
|
||||
|
||||
export type ToolContext = {
|
||||
page: playwright.Page;
|
||||
};
|
||||
|
||||
export type ToolSchema = {
|
||||
name: string;
|
||||
description: string;
|
||||
inputSchema: Record<string, any>;
|
||||
};
|
||||
|
||||
export type ToolResult = {
|
||||
content: (ImageContent | TextContent)[];
|
||||
isError?: boolean;
|
||||
};
|
||||
|
||||
export type Tool = {
|
||||
schema: ToolSchema;
|
||||
handle: (context: ToolContext, params?: Record<string, any>) => Promise<ToolResult>;
|
||||
};
|
||||
|
||||
export const navigate: Tool = {
|
||||
schema: {
|
||||
name: 'navigate',
|
||||
description: 'Navigate to a URL',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
url: {
|
||||
type: 'string',
|
||||
description: 'URL to navigate to',
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
await waitForCompletion(context.page, async () => {
|
||||
await context.page.goto(params!.url as string);
|
||||
});
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Navigated to ${params!.url}`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const wait: Tool = {
|
||||
schema: {
|
||||
name: 'wait',
|
||||
description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
time: {
|
||||
type: 'integer',
|
||||
description: 'Time to wait in seconds',
|
||||
},
|
||||
},
|
||||
required: ['time'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
await context.page.waitForTimeout(Math.min(10000, params!.time as number * 1000));
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Waited for ${params!.time} seconds`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const pressKey: Tool = {
|
||||
schema: {
|
||||
name: 'press_key',
|
||||
description: 'Press a key',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
key: {
|
||||
type: 'string',
|
||||
description: 'Name of the key to press or a character to generate, such as `ArrowLeft` or `a`',
|
||||
},
|
||||
},
|
||||
required: ['key'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
await waitForCompletion(context.page, async () => {
|
||||
await context.page.keyboard.press(params!.key as string);
|
||||
});
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: `Pressed key ${params!.key}`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Tool } from './common';
|
||||
import { waitForCompletion } from '../utils';
|
||||
|
||||
export const screenshot: Tool = {
|
||||
schema: {
|
||||
name: 'screenshot',
|
||||
description: 'Take a screenshot of the current page',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
}
|
||||
},
|
||||
|
||||
handle: async context => {
|
||||
const screenshot = await context.page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' });
|
||||
return {
|
||||
content: [{ type: 'image', data: screenshot.toString('base64'), mimeType: 'image/jpeg' }],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const moveMouse: Tool = {
|
||||
schema: {
|
||||
name: 'move_mouse',
|
||||
description: 'Move mouse to a given position',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
x: {
|
||||
type: 'number',
|
||||
description: 'X coordinate',
|
||||
},
|
||||
y: {
|
||||
type: 'number',
|
||||
description: 'Y coordinate',
|
||||
},
|
||||
},
|
||||
required: ['x', 'y'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
await context.page.mouse.move(params!.x as number, params!.y as number);
|
||||
return {
|
||||
content: [{ type: 'text', text: `Moved mouse to (${params!.x}, ${params!.y})` }],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const click: Tool = {
|
||||
schema: {
|
||||
name: 'click',
|
||||
description: 'Click left mouse button',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
}
|
||||
},
|
||||
|
||||
handle: async context => {
|
||||
await waitForCompletion(context.page, async () => {
|
||||
await context.page.mouse.down();
|
||||
await context.page.mouse.up();
|
||||
});
|
||||
return {
|
||||
content: [{ type: 'text', text: 'Clicked mouse' }],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const drag: Tool = {
|
||||
schema: {
|
||||
name: 'drag',
|
||||
description: 'Drag left mouse button',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
x: {
|
||||
type: 'number',
|
||||
description: 'X coordinate',
|
||||
},
|
||||
y: {
|
||||
type: 'number',
|
||||
description: 'Y coordinate',
|
||||
},
|
||||
},
|
||||
required: ['x', 'y'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
await waitForCompletion(context.page, async () => {
|
||||
await context.page.mouse.down();
|
||||
await context.page.mouse.move(params!.x as number, params!.y as number);
|
||||
await context.page.mouse.up();
|
||||
});
|
||||
return {
|
||||
content: [{ type: 'text', text: `Dragged mouse to (${params!.x}, ${params!.y})` }],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const type: Tool = {
|
||||
schema: {
|
||||
name: 'type',
|
||||
description: 'Type text',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
text: {
|
||||
type: 'string',
|
||||
description: 'Text to type',
|
||||
},
|
||||
},
|
||||
required: ['text'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
await waitForCompletion(context.page, async () => {
|
||||
await context.page.keyboard.type(params!.text as string);
|
||||
});
|
||||
return {
|
||||
content: [{ type: 'text', text: `Typed text "${params!.text}"` }],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
|
@ -0,0 +1,148 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { waitForCompletion } from '../utils';
|
||||
|
||||
import type * as playwright from 'playwright';
|
||||
import type { Tool, ToolContext, ToolResult } from './common';
|
||||
|
||||
const elementIdProperty = {
|
||||
elementId: {
|
||||
type: 'number',
|
||||
description: 'Target element',
|
||||
}
|
||||
};
|
||||
|
||||
export const snapshot: Tool = {
|
||||
schema: {
|
||||
name: 'snapshot',
|
||||
description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
}
|
||||
},
|
||||
|
||||
handle: async context => {
|
||||
return await captureAriaSnapshot(context.page);
|
||||
}
|
||||
};
|
||||
|
||||
export const navigate: Tool = {
|
||||
schema: {
|
||||
name: 'navigate',
|
||||
description: 'Navigate to a URL',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
url: {
|
||||
type: 'string',
|
||||
description: 'URL to navigate to',
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
return runAndCaptureSnapshot(context, () => context.page.goto(params!.url));
|
||||
}
|
||||
};
|
||||
|
||||
export const click: Tool = {
|
||||
schema: {
|
||||
name: 'click',
|
||||
description: 'Perform click on a web page',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...elementIdProperty,
|
||||
},
|
||||
required: ['elementId'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
const locator = elementIdLocator(context.page, params!);
|
||||
return runAndCaptureSnapshot(context, () => locator.click());
|
||||
}
|
||||
};
|
||||
|
||||
export const hover: Tool = {
|
||||
schema: {
|
||||
name: 'hover',
|
||||
description: 'Hover over element on page',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...elementIdProperty,
|
||||
},
|
||||
required: ['elementId'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
const locator = elementIdLocator(context.page, params!);
|
||||
return runAndCaptureSnapshot(context, () => locator.hover());
|
||||
}
|
||||
};
|
||||
|
||||
export const type: Tool = {
|
||||
schema: {
|
||||
name: 'type',
|
||||
description: 'Type text into editable element',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...elementIdProperty,
|
||||
text: {
|
||||
type: 'string',
|
||||
description: 'Text to enter',
|
||||
},
|
||||
submit: {
|
||||
type: 'boolean',
|
||||
description: 'Whether to submit entered text (press Enter after)'
|
||||
}
|
||||
},
|
||||
required: ['elementId', 'text'],
|
||||
}
|
||||
},
|
||||
|
||||
handle: async (context, params) => {
|
||||
const locator = elementIdLocator(context.page, params!);
|
||||
return await runAndCaptureSnapshot(context, async () => {
|
||||
locator.fill(params!.text as string);
|
||||
if (params!.submit)
|
||||
await locator.press('Enter');
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
function elementIdLocator(page: playwright.Page, params: Record<string, string>): playwright.Locator {
|
||||
return page.locator(`internal:aria-id=${params.elementId}`);
|
||||
}
|
||||
|
||||
async function runAndCaptureSnapshot(context: ToolContext, callback: () => Promise<any>): Promise<ToolResult> {
|
||||
const page = context.page;
|
||||
await waitForCompletion(page, () => callback());
|
||||
return captureAriaSnapshot(page);
|
||||
}
|
||||
|
||||
async function captureAriaSnapshot(page: playwright.Page): Promise<ToolResult> {
|
||||
const snapshot = await page.locator('html').ariaSnapshot({ _id: true } as any);
|
||||
return {
|
||||
content: [{ type: 'text', text: `# Current page snapshot\n${snapshot}` }],
|
||||
};
|
||||
}
|
||||
|
|
@ -1,12 +1,11 @@
|
|||
/**
|
||||
* Copyright 2017 Google Inc. All rights reserved.
|
||||
* Modifications copyright (c) Microsoft Corporation.
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
|
|
@ -15,20 +14,19 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ManualPromise } from 'playwright-core/lib/utils';
|
||||
import type * as playwright from 'playwright';
|
||||
|
||||
import type playwright from 'playwright';
|
||||
|
||||
export async function waitForNetwork<R>(page: playwright.Page, callback: () => Promise<R>): Promise<R> {
|
||||
export async function waitForCompletion<R>(page: playwright.Page, callback: () => Promise<R>): Promise<R> {
|
||||
const requests = new Set<playwright.Request>();
|
||||
let frameNavigated = false;
|
||||
const waitBarrier = new ManualPromise();
|
||||
let waitCallback: () => void = () => {};
|
||||
const waitBarrier = new Promise<void>(f => { waitCallback = f; });
|
||||
|
||||
const requestListener = (request: playwright.Request) => requests.add(request);
|
||||
const requestFinishedListener = (request: playwright.Request) => {
|
||||
requests.delete(request);
|
||||
if (!requests.size)
|
||||
waitBarrier.resolve();
|
||||
waitCallback();
|
||||
};
|
||||
|
||||
const frameNavigateListener = (frame: playwright.Frame) => {
|
||||
|
|
@ -38,13 +36,13 @@ export async function waitForNetwork<R>(page: playwright.Page, callback: () => P
|
|||
dispose();
|
||||
clearTimeout(timeout);
|
||||
void frame.waitForLoadState('load').then(() => {
|
||||
waitBarrier.resolve();
|
||||
waitCallback();
|
||||
});
|
||||
};
|
||||
|
||||
const onTimeout = () => {
|
||||
dispose();
|
||||
waitBarrier.resolve();
|
||||
waitCallback();
|
||||
};
|
||||
|
||||
page.on('request', requestListener);
|
||||
|
|
@ -62,7 +60,7 @@ export async function waitForNetwork<R>(page: playwright.Page, callback: () => P
|
|||
try {
|
||||
const result = await callback();
|
||||
if (!requests.size && !frameNavigated)
|
||||
waitBarrier.resolve();
|
||||
waitCallback();
|
||||
await waitBarrier;
|
||||
await page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
|
||||
return result;
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import type playwright from 'playwright';
|
||||
import { ToolDeclaration, JSONSchemaType } from './types';
|
||||
|
||||
export type ToolResult = {
|
||||
error?: string;
|
||||
code: Array<string>;
|
||||
snapshot: string;
|
||||
}
|
||||
|
||||
export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
|
||||
|
||||
export const schema: ToolDeclaration[];
|
||||
export const call: ToolCall;
|
||||
export const snapshot: (page) => Promise<string>;
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import type playwright from 'playwright';
|
||||
import { JSONSchemaType } from './types';
|
||||
|
||||
export type ToolResult = {
|
||||
output?: string;
|
||||
error?: string;
|
||||
base64_image?: string;
|
||||
};
|
||||
|
||||
export type ToolCall = (page: playwright.Page, tool: string, parameters: { [key: string]: JSONSchemaType; }) => Promise<ToolResult>;
|
||||
|
||||
export const call: ToolCall;
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
const { call } = require('./lib/tools/computer-20241022');
|
||||
|
||||
module.exports = { call };
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
{
|
||||
"name": "@playwright/experimental-tools",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"description": "Playwright Tools for AI",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/microsoft/playwright.git"
|
||||
},
|
||||
"homepage": "https://playwright.dev",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"author": {
|
||||
"name": "Microsoft Corporation"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"exports": {
|
||||
"./browser": {
|
||||
"types": "./browser.d.ts",
|
||||
"default": "./browser.js"
|
||||
},
|
||||
"./computer-20241022": {
|
||||
"types": "./computer-20241022.d.ts",
|
||||
"default": "./computer-20241022.js"
|
||||
},
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"dependencies": {
|
||||
"playwright": "1.52.0-next"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@anthropic-ai/sdk": "^0.33.1",
|
||||
"@modelcontextprotocol/sdk": "^1.6.1",
|
||||
"openai": "^4.79.1"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,143 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import browser from '@playwright/experimental-tools/browser';
|
||||
import dotenv from 'dotenv';
|
||||
import playwright from 'playwright';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const anthropic = new Anthropic();
|
||||
|
||||
export const system = `
|
||||
You are a web tester.
|
||||
|
||||
<Instructions>
|
||||
- Perform test according to the provided checklist
|
||||
- Use browser tools to perform actions on web page
|
||||
- Never ask questions, always perform a best guess action
|
||||
- Use one tool at a time, wait for its result before proceeding.
|
||||
- When ready use "reportResult" tool to report result
|
||||
</Instructions>`;
|
||||
|
||||
const reportTool: Anthropic.Tool = {
|
||||
name: 'reportResult',
|
||||
description: 'Submit test result',
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
'success': { type: 'boolean', description: 'Whether test passed' },
|
||||
'result': { type: 'string', description: 'Result of the test if some information has been requested' },
|
||||
'error': { type: 'string', description: 'Error message if test failed' }
|
||||
},
|
||||
required: ['success']
|
||||
}
|
||||
};
|
||||
|
||||
type Message = Anthropic.Beta.Messages.BetaMessageParam & {
|
||||
history: Anthropic.Beta.Messages.BetaMessageParam['content']
|
||||
};
|
||||
|
||||
async function anthropicAgentLoop(page: playwright.Page, task: string) {
|
||||
// Convert them into tools for Anthropic.
|
||||
const pageTools: Anthropic.Tool[] = browser.schema.map(tool => {
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
input_schema: tool.parameters as any,
|
||||
};
|
||||
});
|
||||
|
||||
// Add report tool.
|
||||
const tools = [reportTool, ...pageTools];
|
||||
|
||||
const history: Message[] = [{
|
||||
role: 'user',
|
||||
history: `Task: ${task}`,
|
||||
content: `Task: ${task}\n\n${await browser.snapshot(page)}`,
|
||||
}];
|
||||
|
||||
// Run agentic loop, cap steps.
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const response = await anthropic.messages.create({
|
||||
model: 'claude-3-5-sonnet-20241022',
|
||||
max_tokens: 1024,
|
||||
temperature: 0,
|
||||
tools,
|
||||
system,
|
||||
messages: toAnthropicMessages(history),
|
||||
});
|
||||
history.push({ role: 'assistant', content: response.content, history: response.content });
|
||||
|
||||
const toolUse = response.content.find(block => block.type === 'tool_use');
|
||||
if (!toolUse) {
|
||||
history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (toolUse.name === 'reportResult') {
|
||||
console.log(toolUse.input);
|
||||
return;
|
||||
}
|
||||
|
||||
// Run the Playwright tool.
|
||||
const { error, snapshot, code } = await browser.call(page, toolUse.name, toolUse.input as any);
|
||||
if (code.length)
|
||||
console.log(code.join('\n'));
|
||||
|
||||
// Report the result.
|
||||
const resultText = error ? `Error: ${error}\n` : 'Done\n';
|
||||
history.push({
|
||||
role: 'user',
|
||||
content: [{
|
||||
type: 'tool_result',
|
||||
tool_use_id: toolUse.id,
|
||||
content: [{ type: 'text', text: resultText + snapshot }],
|
||||
}],
|
||||
history: [{
|
||||
type: 'tool_result',
|
||||
tool_use_id: toolUse.id,
|
||||
content: [{ type: 'text', text: resultText }],
|
||||
}],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {
|
||||
return messages.map((message, i) => {
|
||||
if (i === messages.length - 1)
|
||||
return { ...message, history: undefined };
|
||||
return { ...message, content: message.history, history: undefined };
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await playwright.chromium.launch({ headless: false });
|
||||
const page = await browser.newPage();
|
||||
await anthropicAgentLoop(page, `
|
||||
- Go to http://github.com/microsoft
|
||||
- Search for "playwright" repository
|
||||
- Navigate to it
|
||||
- Switch into the Issues tab
|
||||
- Report 3 first issues
|
||||
`);
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
void main();
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
import browser from '@playwright/experimental-tools/browser';
|
||||
import dotenv from 'dotenv';
|
||||
import OpenAI from 'openai';
|
||||
import playwright from 'playwright';
|
||||
|
||||
import type { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const openai = new OpenAI();
|
||||
|
||||
export const system = `
|
||||
You are a web tester.
|
||||
|
||||
<Instructions>to
|
||||
- Perform test according to the provided checklist
|
||||
- Use browser tools to perform actions on web page
|
||||
- Never ask questions, always perform a best guess action
|
||||
- When ready use "reportResult" tool to report result
|
||||
- You can only make one tool call at a time.
|
||||
</Instructions>`;
|
||||
|
||||
type Message = ChatCompletionMessageParam & {
|
||||
history: any
|
||||
};
|
||||
|
||||
const reportTool: ChatCompletionTool = {
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'reportResult',
|
||||
description: 'Submit test result',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
success: { type: 'boolean', description: 'Whether test passed' },
|
||||
result: { type: 'string', description: 'Result of the test if requested' },
|
||||
error: { type: 'string', description: 'Error if test failed' },
|
||||
},
|
||||
required: ['success'],
|
||||
additionalProperties: false,
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
async function openAIAgentLoop(page: playwright.Page, task: string) {
|
||||
const pageTools: ChatCompletionTool[] = browser.schema.map(tool => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: {
|
||||
...tool.parameters,
|
||||
additionalProperties: false,
|
||||
},
|
||||
}
|
||||
}));
|
||||
|
||||
const tools = [reportTool, ...pageTools];
|
||||
|
||||
const history: Message[] = [
|
||||
{
|
||||
role: 'system', content: system, history: system
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
history: `Task: ${task}`,
|
||||
content: `Task: ${task}\n\n${await browser.snapshot(page)}`,
|
||||
}
|
||||
];
|
||||
|
||||
// Run agentic loop, cap steps.
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const completion = await openai.chat.completions.create({
|
||||
model: 'gpt-4o',
|
||||
messages: toOpenAIMessages(history),
|
||||
tools,
|
||||
tool_choice: 'required',
|
||||
store: true,
|
||||
});
|
||||
|
||||
const toolCalls = completion.choices[0]?.message?.tool_calls;
|
||||
if (!toolCalls || toolCalls.length !== 1 || toolCalls[0].type !== 'function') {
|
||||
history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
|
||||
continue;
|
||||
}
|
||||
|
||||
const toolCall = toolCalls[0];
|
||||
if (toolCall.function.name === 'reportResult') {
|
||||
console.log(JSON.parse(toolCall.function.arguments));
|
||||
return;
|
||||
}
|
||||
|
||||
history.push({ ...completion.choices[0].message, history: null });
|
||||
|
||||
// Run the Playwright tool.
|
||||
const params = JSON.parse(toolCall.function.arguments);
|
||||
const { error, snapshot, code } = await browser.call(page, toolCall.function.name, params);
|
||||
if (code.length)
|
||||
console.log(code.join('\n'));
|
||||
|
||||
if (toolCall.function.name === 'log')
|
||||
return;
|
||||
|
||||
// Report the result.
|
||||
const resultText = error ? `Error: ${error}\n` : 'Done\n';
|
||||
history.push({
|
||||
role: 'tool',
|
||||
tool_call_id: toolCall.id,
|
||||
content: resultText + snapshot,
|
||||
history: resultText,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toOpenAIMessages(messages: Message[]): ChatCompletionMessageParam[] {
|
||||
return messages.map((message, i) => {
|
||||
const copy: Message = { ...message };
|
||||
delete copy.history;
|
||||
if (i === messages.length - 1)
|
||||
return copy;
|
||||
copy.content = message.history;
|
||||
return copy;
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const browser = await playwright.chromium.launch({ headless: false });
|
||||
const page = await browser.newPage();
|
||||
await openAIAgentLoop(page, `
|
||||
- Go to http://github.com/microsoft
|
||||
- Search for "playwright" repository
|
||||
- Navigate to it
|
||||
- Switch into the Issues tab
|
||||
- Report 3 first issues
|
||||
`);
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
void main();
|
||||
|
|
@ -1,150 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/* eslint-disable no-console */
|
||||
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import computer from '@playwright/experimental-tools/computer-20241022';
|
||||
import dotenv from 'dotenv';
|
||||
import playwright from 'playwright';
|
||||
|
||||
import type { BetaImageBlockParam, BetaTextBlockParam } from '@anthropic-ai/sdk/resources/beta/messages/messages';
|
||||
import type { ToolResult } from '@playwright/experimental-tools/computer-20241022';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const anthropic = new Anthropic();
|
||||
|
||||
export const system = `
|
||||
You are a web tester.
|
||||
|
||||
<Instructions>
|
||||
- Perform test according to the provided checklist
|
||||
- Use browser tools to perform actions on web page
|
||||
- Never ask questions, always perform a best guess action
|
||||
- Use one tool at a time, wait for its result before proceeding.
|
||||
- When ready use "reportResult" tool to report result
|
||||
</Instructions>`;
|
||||
|
||||
const computerTool: Anthropic.Beta.BetaToolUnion = {
|
||||
type: 'computer_20241022',
|
||||
name: 'computer',
|
||||
display_width_px: 1920,
|
||||
display_height_px: 1080,
|
||||
display_number: 1,
|
||||
};
|
||||
|
||||
const reportTool: Anthropic.Tool = {
|
||||
name: 'reportResult',
|
||||
description: 'Submit test result',
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
'success': { type: 'boolean', description: 'Whether test passed' },
|
||||
'result': { type: 'string', description: 'Result of the test if some information has been requested' },
|
||||
'error': { type: 'string', description: 'Error message if test failed' }
|
||||
},
|
||||
required: ['success']
|
||||
}
|
||||
};
|
||||
|
||||
type Message = Anthropic.Beta.Messages.BetaMessageParam & {
|
||||
history: Anthropic.Beta.Messages.BetaMessageParam['content']
|
||||
};
|
||||
|
||||
async function anthropicAgentLoop(page: playwright.Page, task: string) {
|
||||
// Add report tool.
|
||||
const tools = [reportTool, computerTool];
|
||||
|
||||
const history: Message[] = [{
|
||||
role: 'user',
|
||||
history: `Task: ${task}`,
|
||||
content: `Task: ${task}`,
|
||||
}];
|
||||
|
||||
// Run agentic loop, cap steps.
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const response = await anthropic.beta.messages.create({
|
||||
model: 'claude-3-5-sonnet-20241022',
|
||||
max_tokens: 1024,
|
||||
temperature: 0,
|
||||
tools,
|
||||
system,
|
||||
messages: toAnthropicMessages(history),
|
||||
betas: ['computer-use-2024-10-22'],
|
||||
});
|
||||
|
||||
history.push({ role: 'assistant', content: response.content, history: response.content });
|
||||
|
||||
const toolUse = response.content.find(block => block.type === 'tool_use');
|
||||
if (!toolUse) {
|
||||
history.push({ role: 'user', content: 'expected exactly one tool call', history: 'expected exactly one tool call' });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (toolUse.name === 'reportResult') {
|
||||
console.log(toolUse.input);
|
||||
return;
|
||||
}
|
||||
|
||||
const result: ToolResult = await computer.call(page, toolUse.name, toolUse.input as any);
|
||||
const contentEntry: BetaTextBlockParam | BetaImageBlockParam = result.base64_image ? {
|
||||
type: 'image',
|
||||
source: { type: 'base64', media_type: 'image/jpeg', data: result.base64_image }
|
||||
} : {
|
||||
type: 'text',
|
||||
text: result.output || '',
|
||||
};
|
||||
history.push({
|
||||
role: 'user',
|
||||
content: [{
|
||||
type: 'tool_result',
|
||||
tool_use_id: toolUse.id,
|
||||
content: [contentEntry],
|
||||
}],
|
||||
history: [{
|
||||
type: 'tool_result',
|
||||
tool_use_id: toolUse.id,
|
||||
content: [{ type: 'text', text: '<redacted>' }],
|
||||
}],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function toAnthropicMessages(messages: Message[]): Anthropic.Beta.Messages.BetaMessageParam[] {
|
||||
return messages.map((message, i) => {
|
||||
if (i === messages.length - 1)
|
||||
return { ...message, history: undefined };
|
||||
return { ...message, content: message.history, history: undefined };
|
||||
});
|
||||
}
|
||||
|
||||
const githubTask = `
|
||||
- Search for "playwright" repository
|
||||
- Navigate to it
|
||||
- Switch into the Issues tab
|
||||
- Report 3 first issues
|
||||
`;
|
||||
|
||||
async function main() {
|
||||
const browser = await playwright.chromium.launch({ headless: false });
|
||||
const page = await browser.newPage();
|
||||
await page.goto('http://github.com/microsoft');
|
||||
await anthropicAgentLoop(page, githubTask);
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
void main();
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import {
|
||||
CallToolRequestSchema,
|
||||
ListToolsRequestSchema,
|
||||
} from '@modelcontextprotocol/sdk/types.js';
|
||||
import * as playwright from 'playwright';
|
||||
import browser from '@playwright/experimental-tools/browser';
|
||||
|
||||
const server = new Server(
|
||||
{
|
||||
name: 'MCP Server for Playwright',
|
||||
version: '0.0.1',
|
||||
},
|
||||
{
|
||||
capabilities: {
|
||||
tools: {},
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
||||
return {
|
||||
tools: browser.schema.map(tool => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: tool.parameters,
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
async function createBrowser(): Promise<playwright.Browser> {
|
||||
if (process.env.PLAYWRIGHT_WS_ENDPOINT) {
|
||||
return await playwright.chromium.connect(
|
||||
process.env.PLAYWRIGHT_WS_ENDPOINT
|
||||
);
|
||||
}
|
||||
return await playwright.chromium.launch({ headless: false });
|
||||
}
|
||||
|
||||
async function getPage(): Promise<playwright.Page> {
|
||||
if (!page) {
|
||||
const browser = await createBrowser();
|
||||
const context = await browser.newContext();
|
||||
page = await context.newPage();
|
||||
}
|
||||
return page;
|
||||
}
|
||||
|
||||
let page: playwright.Page | undefined;
|
||||
|
||||
async function main() {
|
||||
server.setRequestHandler(CallToolRequestSchema, async request => {
|
||||
const page = await getPage();
|
||||
const response = await browser.call(
|
||||
page,
|
||||
request.params.name,
|
||||
request.params.arguments as any
|
||||
);
|
||||
const content: { type: string; text: string }[] = [];
|
||||
if (response.error)
|
||||
content.push({ type: 'text', text: response.error });
|
||||
if (response.snapshot)
|
||||
content.push({ type: 'text', text: response.snapshot });
|
||||
return {
|
||||
content,
|
||||
isError: response.error ? true : false,
|
||||
};
|
||||
});
|
||||
|
||||
process.stdin.on('close', async () => {
|
||||
server.close();
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
setTimeout(() => process.exit(0), 15000);
|
||||
await page?.context()?.browser()?.close();
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
await server.connect(new StdioServerTransport());
|
||||
}
|
||||
|
||||
void main();
|
||||
|
|
@ -1,150 +0,0 @@
|
|||
/**
|
||||
* Copyright 2017 Google Inc. All rights reserved.
|
||||
* Modifications copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { waitForNetwork } from './utils';
|
||||
|
||||
import type { ToolResult } from '../../browser';
|
||||
import type { JSONSchemaType, ToolDeclaration } from '../../types';
|
||||
import type playwright from 'playwright';
|
||||
|
||||
|
||||
type LocatorEx = playwright.Locator & {
|
||||
_generateLocatorString: () => Promise<string>;
|
||||
};
|
||||
|
||||
const intentProperty = {
|
||||
intent: {
|
||||
type: 'string',
|
||||
description: 'Intent behind this particular action. Used as a comment.',
|
||||
}
|
||||
};
|
||||
|
||||
const elementIdProperty = {
|
||||
elementId: {
|
||||
type: 'number',
|
||||
description: 'Target element',
|
||||
}
|
||||
};
|
||||
|
||||
export const schema: ToolDeclaration[] = [
|
||||
{
|
||||
name: 'navigate',
|
||||
description: 'Navigate to a URL',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...intentProperty,
|
||||
url: {
|
||||
type: 'string',
|
||||
description: 'URL to navigate to',
|
||||
},
|
||||
},
|
||||
required: ['intent', 'elementId'],
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'click',
|
||||
description: 'Perform click on a web page',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...intentProperty,
|
||||
...elementIdProperty,
|
||||
},
|
||||
required: ['intent', 'elementId'],
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'enterText',
|
||||
description: 'Enter text into editable element',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...intentProperty,
|
||||
...elementIdProperty,
|
||||
text: {
|
||||
type: 'string',
|
||||
description: 'Text to enter',
|
||||
},
|
||||
submit: {
|
||||
type: 'boolean',
|
||||
description: 'Whether to submit entered text (press Enter after)'
|
||||
}
|
||||
},
|
||||
required: ['intent', 'elementId', 'text'],
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'wait',
|
||||
description: `Wait for given amount of time to see if the page updates. Use it after action if you think page is not ready yet`,
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...intentProperty,
|
||||
time: {
|
||||
type: 'integer',
|
||||
description: 'Time to wait in seconds',
|
||||
},
|
||||
},
|
||||
required: ['intent', 'time'],
|
||||
}
|
||||
},
|
||||
];
|
||||
|
||||
export async function call(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>): Promise<ToolResult> {
|
||||
const code: string[] = [];
|
||||
try {
|
||||
await waitForNetwork(page, async () => {
|
||||
await performAction(page, toolName, params, code);
|
||||
});
|
||||
} catch (e) {
|
||||
return { error: e.message, snapshot: await snapshot(page), code };
|
||||
}
|
||||
return { snapshot: await snapshot(page), code };
|
||||
}
|
||||
|
||||
export async function snapshot(page: playwright.Page) {
|
||||
const params = { _id: true } as any;
|
||||
return `<Page snapshot>\n${await page.locator('body').ariaSnapshot(params)}\n</Page snapshot>`;
|
||||
}
|
||||
|
||||
async function performAction(page: playwright.Page, toolName: string, params: Record<string, JSONSchemaType>, code: string[]) {
|
||||
const locator = elementLocator(page, params);
|
||||
code.push((params.intent as string).split('\n').map(line => `// ${line}`).join('\n'));
|
||||
if (toolName === 'navigate') {
|
||||
code.push(`await page.goto(${JSON.stringify(params.url)})`);
|
||||
await page.goto(params.url as string);
|
||||
} else if (toolName === 'wait') {
|
||||
await page.waitForTimeout(Math.min(10000, params.time as number * 1000));
|
||||
} else if (toolName === 'click') {
|
||||
code.push(`await page.${await locator._generateLocatorString()}.click()`);
|
||||
await locator.click();
|
||||
} else if (toolName === 'enterText') {
|
||||
code.push(`await page.${await locator._generateLocatorString()}.click()`);
|
||||
await locator.click();
|
||||
code.push(`await page.${await locator._generateLocatorString()}.fill(${JSON.stringify(params.text)})`);
|
||||
await locator.fill(params.text as string);
|
||||
if (params.submit) {
|
||||
code.push(`await page.${await locator._generateLocatorString()}.press("Enter")`);
|
||||
await locator.press('Enter');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function elementLocator(page: playwright.Page, params: any): LocatorEx {
|
||||
return page.locator(`internal:aria-id=${params.elementId}`) as LocatorEx;
|
||||
}
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
/**
|
||||
* Copyright 2017 Google Inc. All rights reserved.
|
||||
* Modifications copyright (c) Microsoft Corporation.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the 'License');
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an 'AS IS' BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { waitForNetwork } from './utils';
|
||||
|
||||
import type { ToolResult } from '../../computer-20241022';
|
||||
import type { JSONSchemaType } from '../../types';
|
||||
import type playwright from 'playwright';
|
||||
|
||||
|
||||
export async function call(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {
|
||||
if (toolName !== 'computer')
|
||||
throw new Error('Unsupported tool');
|
||||
return await waitForNetwork(page, async () => {
|
||||
return await performAction(page, toolName, input);
|
||||
});
|
||||
}
|
||||
|
||||
type PageState = {
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
|
||||
const pageStateSymbol = Symbol('pageState');
|
||||
|
||||
function pageState(page: playwright.Page): PageState {
|
||||
if (!(page as any)[pageStateSymbol])
|
||||
(page as any)[pageStateSymbol] = { x: 0, y: 0 };
|
||||
return (page as any)[pageStateSymbol];
|
||||
}
|
||||
|
||||
async function performAction(page: playwright.Page, toolName: string, input: Record<string, JSONSchemaType>): Promise<ToolResult> {
|
||||
const state = pageState(page);
|
||||
const { action } = input as { action: string };
|
||||
if (action === 'screenshot') {
|
||||
const screenshot = await page.screenshot({ type: 'jpeg', quality: 50, scale: 'css' });
|
||||
return {
|
||||
output: 'Screenshot',
|
||||
base64_image: screenshot.toString('base64'),
|
||||
};
|
||||
}
|
||||
if (action === 'mouse_move') {
|
||||
const { coordinate } = input as { coordinate: [number, number] };
|
||||
state.x = coordinate[0];
|
||||
state.y = coordinate[1];
|
||||
await page.mouse.move(state.x, state.y);
|
||||
return { output: 'Mouse moved' };
|
||||
}
|
||||
if (action === 'left_click') {
|
||||
await page.mouse.down();
|
||||
await page.mouse.up();
|
||||
return { output: 'Left clicked' };
|
||||
}
|
||||
if (action === 'left_click_drag') {
|
||||
await page.mouse.down();
|
||||
const { coordinate } = input as { coordinate: [number, number] };
|
||||
state.x = coordinate[0];
|
||||
state.y = coordinate[1];
|
||||
await page.mouse.move(state.x, state.y);
|
||||
await page.mouse.up();
|
||||
return { output: 'Left dragged' };
|
||||
}
|
||||
if (action === 'right_click') {
|
||||
await page.mouse.down({ button: 'right' });
|
||||
await page.mouse.up({ button: 'right' });
|
||||
return { output: 'Right clicked' };
|
||||
}
|
||||
if (action === 'double_click') {
|
||||
await page.mouse.down();
|
||||
await page.mouse.up();
|
||||
await page.mouse.down();
|
||||
await page.mouse.up();
|
||||
return { output: 'Double clicked' };
|
||||
}
|
||||
if (action === 'middle_click') {
|
||||
await page.mouse.down({ button: 'middle' });
|
||||
await page.mouse.up({ button: 'middle' });
|
||||
return { output: 'Middle clicked' };
|
||||
}
|
||||
if (action === 'key') {
|
||||
const { text } = input as { text: string };
|
||||
await page.keyboard.press(xToPlaywright(text));
|
||||
return { output: 'Text typed' };
|
||||
}
|
||||
if (action === 'cursor_position')
|
||||
return { output: `X=${state.x},Y=${state.y}` };
|
||||
throw new Error('Unimplemented tool: ' + toolName);
|
||||
}
|
||||
|
||||
const xToPlaywrightKeyMap = new Map([
|
||||
['BackSpace', 'Backspace'],
|
||||
['Tab', 'Tab'],
|
||||
['Return', 'Enter'],
|
||||
['Escape', 'Escape'],
|
||||
['space', ' '],
|
||||
['Delete', 'Delete'],
|
||||
['Home', 'Home'],
|
||||
['End', 'End'],
|
||||
['Left', 'ArrowLeft'],
|
||||
['Up', 'ArrowUp'],
|
||||
['Right', 'ArrowRight'],
|
||||
['Down', 'ArrowDown'],
|
||||
['Insert', 'Insert'],
|
||||
['Page_Up', 'PageUp'],
|
||||
['Page_Down', 'PageDown'],
|
||||
['F1', 'F1'],
|
||||
['F2', 'F2'],
|
||||
['F3', 'F3'],
|
||||
['F4', 'F4'],
|
||||
['F5', 'F5'],
|
||||
['F6', 'F6'],
|
||||
['F7', 'F7'],
|
||||
['F8', 'F8'],
|
||||
['F9', 'F9'],
|
||||
['F10', 'F10'],
|
||||
['F11', 'F11'],
|
||||
['F12', 'F12'],
|
||||
['Shift_L', 'Shift'],
|
||||
['Shift_R', 'Shift'],
|
||||
['Control_L', 'Control'],
|
||||
['Control_R', 'Control'],
|
||||
['Alt_L', 'Alt'],
|
||||
['Alt_R', 'Alt'],
|
||||
['Super_L', 'Meta'],
|
||||
['Super_R', 'Meta'],
|
||||
]);
|
||||
|
||||
const xToPlaywrightModifierMap = new Map([
|
||||
['alt', 'Alt'],
|
||||
['control', 'Control'],
|
||||
['meta', 'Meta'],
|
||||
['shift', 'Shift'],
|
||||
]);
|
||||
|
||||
|
||||
const xToPlaywright = (key: string) => {
|
||||
const tokens = key.split('+');
|
||||
if (tokens.length === 1)
|
||||
return xToPlaywrightKeyMap.get(key) || key;
|
||||
if (tokens.length === 2) {
|
||||
const modifier = xToPlaywrightModifierMap.get(tokens[0]);
|
||||
const key = xToPlaywrightKeyMap.get(tokens[1]) || tokens[1];
|
||||
return modifier + '+' + key;
|
||||
}
|
||||
throw new Error('Invalid key: ' + key);
|
||||
};
|
||||
|
|
@ -174,7 +174,7 @@ const workspace = new Workspace(ROOT_PATH, [
|
|||
}),
|
||||
new PWPackage({
|
||||
name: '@playwright/experimental-tools',
|
||||
path: path.join(ROOT_PATH, 'packages', 'playwright-tools'),
|
||||
path: path.join(ROOT_PATH, 'packages', 'playwright-mcp'),
|
||||
files: LICENCE_FILES,
|
||||
}),
|
||||
new PWPackage({
|
||||
|
|
|
|||
Loading…
Reference in New Issue