From 471a28e0d51abca7ca2086c51b3efb3e81f4d458 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Fri, 28 Mar 2025 12:03:54 +0000 Subject: [PATCH] feat: support proxy in connect/connectOverCDP (#35389) --- docs/src/api/class-browsertype.md | 20 +++++ docs/src/api/params.md | 6 +- packages/playwright-client/types/types.d.ts | 56 +++++++++++++ .../playwright-core/src/client/browserType.ts | 4 +- packages/playwright-core/src/client/types.ts | 6 ++ .../playwright-core/src/protocol/validator.ts | 12 +++ .../playwright-core/src/server/browserType.ts | 2 +- .../src/server/chromium/chromium.ts | 33 ++++++-- .../dispatchers/browserTypeDispatcher.ts | 2 +- .../dispatchers/localUtilsDispatcher.ts | 8 +- packages/playwright-core/src/server/fetch.ts | 44 +---------- .../socksClientCertificatesInterceptor.ts | 6 +- .../playwright-core/src/server/transport.ts | 39 +++++---- .../src/server/utils/network.ts | 79 +++++++++++++++---- packages/playwright-core/types/types.d.ts | 56 +++++++++++++ packages/protocol/src/channels.d.ts | 24 ++++++ packages/protocol/src/protocol.yml | 14 ++++ tests/library/browsertype-connect.spec.ts | 18 +++++ .../library/chromium/connect-over-cdp.spec.ts | 26 ++++++ tests/library/proxy.spec.ts | 4 +- 20 files changed, 365 insertions(+), 94 deletions(-) diff --git a/docs/src/api/class-browsertype.md b/docs/src/api/class-browsertype.md index f970d4aef3..1480d567c5 100644 --- a/docs/src/api/class-browsertype.md +++ b/docs/src/api/class-browsertype.md @@ -144,6 +144,16 @@ Some common examples: 1. `""` to expose localhost network. 1. `"*.test.internal-domain,*.staging.internal-domain,"` to expose test/staging deployments and localhost. +### option: BrowserType.connect.proxy +* since: v1.52 +- `proxy` <[Object]> + - `server` <[string]> Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy. + - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. + - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication. + - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication. + +Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used by the browser to load web pages. + ## async method: BrowserType.connectOverCDP * since: v1.9 - returns: <[Browser]> @@ -232,6 +242,16 @@ Logger sink for Playwright logging. Optional. Maximum time in milliseconds to wait for the connection to be established. Defaults to `30000` (30 seconds). Pass `0` to disable timeout. +### option: BrowserType.connectOverCDP.proxy +* since: v1.52 +- `proxy` <[Object]> + - `server` <[string]> Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy. + - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. + - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication. + - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication. + +Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used by the browser to load web pages. + ## method: BrowserType.executablePath * since: v1.8 - returns: <[string]> diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 4c49370044..aeb8f7fde3 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -226,10 +226,8 @@ Dangerous option; use with care. Defaults to `false`. ## browser-option-proxy - `proxy` <[Object]> - `server` <[string]> Proxy to be used for all requests. HTTP and SOCKS proxies are supported, for example - `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP - proxy. - - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, - .domain.com"`. + `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP proxy. + - `bypass` ?<[string]> Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. - `username` ?<[string]> Optional username to use if HTTP proxy requires authentication. - `password` ?<[string]> Optional password to use if HTTP proxy requires authentication. diff --git a/packages/playwright-client/types/types.d.ts b/packages/playwright-client/types/types.d.ts index aff26e8942..8ad9c6e348 100644 --- a/packages/playwright-client/types/types.d.ts +++ b/packages/playwright-client/types/types.d.ts @@ -21793,6 +21793,34 @@ export interface ConnectOverCDPOptions { */ logger?: Logger; + /** + * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used + * by the browser to load web pages. + */ + proxy?: { + /** + * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example + * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP + * proxy. + */ + server: string; + + /** + * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. + */ + bypass?: string; + + /** + * Optional username to use if HTTP proxy requires authentication. + */ + username?: string; + + /** + * Optional password to use if HTTP proxy requires authentication. + */ + password?: string; + }; + /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. @@ -21834,6 +21862,34 @@ export interface ConnectOptions { */ logger?: Logger; + /** + * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used + * by the browser to load web pages. + */ + proxy?: { + /** + * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example + * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP + * proxy. + */ + server: string; + + /** + * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. + */ + bypass?: string; + + /** + * Optional username to use if HTTP proxy requires authentication. + */ + username?: string; + + /** + * Optional password to use if HTTP proxy requires authentication. + */ + password?: string; + }; + /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. diff --git a/packages/playwright-core/src/client/browserType.ts b/packages/playwright-core/src/client/browserType.ts index f67c97ced8..47e5722335 100644 --- a/packages/playwright-core/src/client/browserType.ts +++ b/packages/playwright-core/src/client/browserType.ts @@ -129,6 +129,7 @@ export class BrowserType extends ChannelOwner imple exposeNetwork: params.exposeNetwork ?? params._exposeNetwork, slowMo: params.slowMo, timeout: params.timeout, + proxy: params.proxy, }; if ((params as any).__testHookRedirectPortForwarding) connectParams.socksProxyRedirectPortForTest = (params as any).__testHookRedirectPortForwarding; @@ -188,7 +189,8 @@ export class BrowserType extends ChannelOwner imple endpointURL, headers, slowMo: params.slowMo, - timeout: params.timeout + timeout: params.timeout, + proxy: params.proxy, }); const browser = Browser.from(result.browser); this._didLaunchBrowser(browser, {}, params.logger); diff --git a/packages/playwright-core/src/client/types.ts b/packages/playwright-core/src/client/types.ts index 5235fef8b4..0a7d93c8a4 100644 --- a/packages/playwright-core/src/client/types.ts +++ b/packages/playwright-core/src/client/types.ts @@ -103,6 +103,12 @@ export type ConnectOptions = { slowMo?: number, timeout?: number, logger?: Logger, + proxy?: { + server: string, + bypass?: string, + username?: string, + password?: string + }, }; export type LaunchServerOptions = { channel?: channels.BrowserTypeLaunchOptions['channel'], diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 0484fd1c11..96b31310b6 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -327,6 +327,12 @@ scheme.LocalUtilsConnectParams = tObject({ exposeNetwork: tOptional(tString), slowMo: tOptional(tNumber), timeout: tOptional(tNumber), + proxy: tOptional(tObject({ + server: tString, + bypass: tOptional(tString), + username: tOptional(tString), + password: tOptional(tString), + })), socksProxyRedirectPortForTest: tOptional(tNumber), }); scheme.LocalUtilsConnectResult = tObject({ @@ -650,6 +656,12 @@ scheme.BrowserTypeConnectOverCDPParams = tObject({ headers: tOptional(tArray(tType('NameValue'))), slowMo: tOptional(tNumber), timeout: tOptional(tNumber), + proxy: tOptional(tObject({ + server: tString, + bypass: tOptional(tString), + username: tOptional(tString), + password: tOptional(tString), + })), }); scheme.BrowserTypeConnectOverCDPResult = tObject({ browser: tChannel(['Browser']), diff --git a/packages/playwright-core/src/server/browserType.ts b/packages/playwright-core/src/server/browserType.ts index c4c44d83de..f0f3c53a58 100644 --- a/packages/playwright-core/src/server/browserType.ts +++ b/packages/playwright-core/src/server/browserType.ts @@ -292,7 +292,7 @@ export abstract class BrowserType extends SdkObject { await fs.promises.mkdir(options.tracesDir, { recursive: true }); } - async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number }, timeout?: number): Promise { + async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, proxy?: types.ProxySettings, timeout?: number, headers?: types.HeadersArray }): Promise { throw new Error('CDP connections are only supported by Chromium'); } diff --git a/packages/playwright-core/src/server/chromium/chromium.ts b/packages/playwright-core/src/server/chromium/chromium.ts index 35acc621e2..0f478c9d50 100644 --- a/packages/playwright-core/src/server/chromium/chromium.ts +++ b/packages/playwright-core/src/server/chromium/chromium.ts @@ -64,15 +64,15 @@ export class Chromium extends BrowserType { this._devtools = this._createDevTools(); } - override async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, headers?: types.HeadersArray }, timeout?: number) { + override async connectOverCDP(metadata: CallMetadata, endpointURL: string, options: { slowMo?: number, proxy?: types.ProxySettings, timeout?: number, headers?: types.HeadersArray }) { const controller = new ProgressController(metadata, this); controller.setLogName('browser'); return controller.run(async progress => { return await this._connectOverCDPInternal(progress, endpointURL, options); - }, TimeoutSettings.timeout({ timeout })); + }, TimeoutSettings.timeout(options)); } - async _connectOverCDPInternal(progress: Progress, endpointURL: string, options: types.LaunchOptions & { headers?: types.HeadersArray }, onClose?: () => Promise) { + async _connectOverCDPInternal(progress: Progress, endpointURL: string, options: types.LaunchOptions & { headers?: types.HeadersArray, proxy?: types.ProxySettings }, onClose?: () => Promise) { let headersMap: { [key: string]: string; } | undefined; if (options.headers) headersMap = headersArrayToObject(options.headers, false); @@ -84,10 +84,10 @@ export class Chromium extends BrowserType { const artifactsDir = await fs.promises.mkdtemp(ARTIFACTS_FOLDER); - const wsEndpoint = await urlToWSEndpoint(progress, endpointURL, headersMap); + const wsEndpoint = await urlToWSEndpoint(progress, endpointURL, headersMap, options.proxy); progress.throwIfAborted(); - const chromeTransport = await WebSocketTransport.connect(progress, wsEndpoint, headersMap); + const chromeTransport = await WebSocketTransport.connect(progress, wsEndpoint, { headers: headersMap, proxy: options.proxy }); const cleanedUp = new ManualPromise(); const doCleanup = async () => { await removeFolders([artifactsDir]); @@ -365,18 +365,35 @@ class ChromiumReadyState extends BrowserReadyState { } } -async function urlToWSEndpoint(progress: Progress, endpointURL: string, headers: { [key: string]: string; }) { +async function urlToWSEndpoint(progress: Progress, endpointURL: string, headers: { [key: string]: string; }, proxy?: types.ProxySettings) { if (endpointURL.startsWith('ws')) return endpointURL; progress.log(` retrieving websocket url from ${endpointURL}`); const httpURL = endpointURL.endsWith('/') ? `${endpointURL}json/version/` : `${endpointURL}/json/version/`; + // Chromium insists on localhost "Host" header for security reasons, and in the case of proxy + // we end up with the remote host instead of localhost. + const extraHeaders = proxy ? { Host: `localhost:9222` } : {}; const json = await fetchData({ url: httpURL, - headers, + headers: { ...headers, ...extraHeaders }, + proxy, }, async (_, resp) => new Error(`Unexpected status ${resp.statusCode} when connecting to ${httpURL}.\n` + `This does not look like a DevTools server, try connecting via ws://.`) ); - return JSON.parse(json).webSocketDebuggerUrl; + const wsUrl = JSON.parse(json).webSocketDebuggerUrl; + if (proxy) { + // webSocketDebuggerUrl will be a localhost URL, accessible from the browser's computer. + // When using a proxy, assume we need to connect through the original endpointURL. + // Example: + // connecting to http://example.com/ + // making request to http://example.com/json/version/ + // webSocketDebuggerUrl ends up as ws://localhost:9222/devtools/page/ + // we construct ws://example.com/devtools/page/ by appending the pathname to the original URL + const url = new URL(endpointURL); + url.pathname += (url.pathname.endsWith('/') ? '' : '/') + new URL(wsUrl).pathname.substring(1); + return url.toString(); + } + return wsUrl; } async function seleniumErrorHandler(params: HTTPRequestParams, response: http.IncomingMessage) { diff --git a/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts b/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts index bf21e726bc..40bf3eb9d8 100644 --- a/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/browserTypeDispatcher.ts @@ -43,7 +43,7 @@ export class BrowserTypeDispatcher extends Dispatcher { - const browser = await this._object.connectOverCDP(metadata, params.endpointURL, params, params.timeout); + const browser = await this._object.connectOverCDP(metadata, params.endpointURL, params); const browserDispatcher = new BrowserDispatcher(this, browser); return { browser: browserDispatcher, diff --git a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts index a681b349d3..2a66d0a788 100644 --- a/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts +++ b/packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts @@ -33,6 +33,7 @@ import type { RootDispatcher } from './dispatcher'; import type * as channels from '@protocol/channels'; import type * as http from 'http'; import type { HTTPRequestParams } from '../utils/network'; +import type { ProxySettings } from '../types'; export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.LocalUtilsChannel, RootDispatcher> implements channels.LocalUtilsChannel { _type_LocalUtils: boolean; @@ -90,9 +91,9 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels. 'x-playwright-proxy': params.exposeNetwork ?? '', ...params.headers, }; - const wsEndpoint = await urlToWSEndpoint(progress, params.wsEndpoint); + const wsEndpoint = await urlToWSEndpoint(progress, params.wsEndpoint, params.proxy); - const transport = await WebSocketTransport.connect(progress, wsEndpoint, wsHeaders, true, 'x-playwright-debug-log'); + const transport = await WebSocketTransport.connect(progress, wsEndpoint, { headers: wsHeaders, followRedirects: true, debugLogHeader: 'x-playwright-debug-log', proxy: params.proxy }); const socksInterceptor = new SocksInterceptor(transport, params.exposeNetwork, params.socksProxyRedirectPortForTest); const pipe = new JsonPipeDispatcher(this); transport.onmessage = json => { @@ -128,7 +129,7 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels. } } -async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: string): Promise { +async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: string, proxy?: ProxySettings): Promise { if (endpointURL.startsWith('ws')) return endpointURL; @@ -142,6 +143,7 @@ async function urlToWSEndpoint(progress: Progress | undefined, endpointURL: stri method: 'GET', timeout: progress?.timeUntilDeadline() ?? 30_000, headers: { 'User-Agent': getUserAgent() }, + proxy, }, async (params: HTTPRequestParams, response: http.IncomingMessage) => { return new Error(`Unexpected status ${response.statusCode} when connecting to ${fetchUrl.toString()}.\n` + `This does not look like a Playwright server, try connecting via ws://.`); diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts index 9760ea89a1..71f9de0833 100644 --- a/packages/playwright-core/src/server/fetch.ts +++ b/packages/playwright-core/src/server/fetch.ts @@ -18,14 +18,12 @@ import http from 'http'; import https from 'https'; import { Transform, pipeline } from 'stream'; import { TLSSocket } from 'tls'; -import url from 'url'; import * as zlib from 'zlib'; import { TimeoutSettings } from './timeoutSettings'; -import { assert, constructURLBasedOnBaseURL, eventsHelper, monotonicTime } from '../utils'; +import { assert, constructURLBasedOnBaseURL, createProxyAgent, eventsHelper, monotonicTime } from '../utils'; import { createGuid } from './utils/crypto'; import { getUserAgent } from './utils/userAgent'; -import { HttpsProxyAgent, SocksProxyAgent } from '../utilsBundle'; import { BrowserContext, verifyClientCertificates } from './browserContext'; import { CookieStore, domainMatches, parseRawCookie } from './cookieStore'; import { MultipartFormData } from './formData'; @@ -183,8 +181,8 @@ export abstract class APIRequestContext extends SdkObject { let agent; // We skip 'per-context' in order to not break existing users. 'per-context' was previously used to // workaround an upstream Chromium bug. Can be removed in the future. - if (proxy && proxy.server !== 'per-context' && !shouldBypassProxy(requestUrl, proxy.bypass)) - agent = createProxyAgent(proxy); + if (proxy?.server !== 'per-context') + agent = createProxyAgent(proxy, requestUrl); let maxRedirects = params.maxRedirects ?? (defaults.maxRedirects ?? 20); maxRedirects = maxRedirects === 0 ? -1 : maxRedirects; @@ -647,13 +645,6 @@ export class GlobalAPIRequestContext extends APIRequestContext { const timeoutSettings = new TimeoutSettings(); if (options.timeout !== undefined) timeoutSettings.setDefaultTimeout(options.timeout); - const proxy = options.proxy; - if (proxy?.server) { - let url = proxy?.server.trim(); - if (!/^\w+:\/\//.test(url)) - url = 'http://' + url; - proxy.server = url; - } if (options.storageState) { this._origins = options.storageState.origins?.map(origin => ({ indexedDB: [], ...origin })); this._cookieStore.addCookies(options.storageState.cookies || []); @@ -668,7 +659,7 @@ export class GlobalAPIRequestContext extends APIRequestContext { maxRedirects: options.maxRedirects, httpCredentials: options.httpCredentials, clientCertificates: options.clientCertificates, - proxy, + proxy: options.proxy, timeoutSettings, }; this._tracing = new Tracing(this, options.tracesDir); @@ -705,20 +696,6 @@ export class GlobalAPIRequestContext extends APIRequestContext { } } -export function createProxyAgent(proxy: types.ProxySettings) { - const proxyOpts = url.parse(proxy.server); - if (proxyOpts.protocol?.startsWith('socks')) { - return new SocksProxyAgent({ - host: proxyOpts.hostname, - port: proxyOpts.port || undefined, - }); - } - if (proxy.username) - proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`; - // TODO: We should use HttpProxyAgent conditional on proxyOpts.protocol instead of always using CONNECT method. - return new HttpsProxyAgent(proxyOpts); -} - function toHeadersArray(rawHeaders: string[]): types.HeadersArray { const result: types.HeadersArray = []; for (let i = 0; i < rawHeaders.length; i += 2) @@ -791,19 +768,6 @@ function removeHeader(headers: { [name: string]: string }, name: string) { delete headers[name]; } -function shouldBypassProxy(url: URL, bypass?: string): boolean { - if (!bypass) - return false; - const domains = bypass.split(',').map(s => { - s = s.trim(); - if (!s.startsWith('.')) - s = '.' + s; - return s; - }); - const domain = '.' + url.hostname; - return domains.some(d => domain.endsWith(d)); -} - function setBasicAuthorizationHeader(headers: { [name: string]: string }, credentials: HTTPCredentials) { const { username, password } = credentials; const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64'); diff --git a/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts b/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts index a68cb88abb..2cb5883782 100644 --- a/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts +++ b/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts @@ -23,7 +23,7 @@ import tls from 'tls'; import { SocksProxy } from './utils/socksProxy'; import { ManualPromise, escapeHTML, generateSelfSignedCertificate, rewriteErrorMessage } from '../utils'; import { verifyClientCertificates } from './browserContext'; -import { createProxyAgent } from './fetch'; +import { createProxyAgent } from './utils/network'; import { debugLogger } from './utils/debugLogger'; import { createSocket, createTLSSocket } from './utils/happyEyeballs'; @@ -242,7 +242,7 @@ export class ClientCertificatesProxy { ignoreHTTPSErrors: boolean | undefined; secureContextMap: Map = new Map(); alpnCache: ALPNCache; - proxyAgentFromOptions: ReturnType | undefined; + proxyAgentFromOptions: ReturnType; constructor( contextOptions: Pick @@ -250,7 +250,7 @@ export class ClientCertificatesProxy { verifyClientCertificates(contextOptions.clientCertificates); this.alpnCache = new ALPNCache(); this.ignoreHTTPSErrors = contextOptions.ignoreHTTPSErrors; - this.proxyAgentFromOptions = contextOptions.proxy ? createProxyAgent(contextOptions.proxy) : undefined; + this.proxyAgentFromOptions = createProxyAgent(contextOptions.proxy); this._initSecureContexts(contextOptions.clientCertificates); this._socksProxy = new SocksProxy(); this._socksProxy.setPattern('*'); diff --git a/packages/playwright-core/src/server/transport.ts b/packages/playwright-core/src/server/transport.ts index ec2caec0c5..a4e092ef6e 100644 --- a/packages/playwright-core/src/server/transport.ts +++ b/packages/playwright-core/src/server/transport.ts @@ -15,13 +15,13 @@ * limitations under the License. */ -import { makeWaitForNextTask } from '../utils'; +import { createProxyAgent, makeWaitForNextTask } from '../utils'; import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './utils/happyEyeballs'; import { ws } from '../utilsBundle'; import type { WebSocket } from '../utilsBundle'; import type { Progress } from './progress'; -import type { HeadersArray } from './types'; +import type { HeadersArray, ProxySettings } from './types'; import type { ClientRequest, IncomingMessage } from 'http'; export const perMessageDeflate = { @@ -60,6 +60,13 @@ export interface ConnectionTransport { onclose?: (reason?: string) => void, } +type WebSocketTransportOptions = { + headers?: { [key: string]: string; }; + followRedirects?: boolean; + debugLogHeader?: string; + proxy?: ProxySettings; +}; + export class WebSocketTransport implements ConnectionTransport { private _ws: WebSocket; private _progress?: Progress; @@ -70,14 +77,14 @@ export class WebSocketTransport implements ConnectionTransport { readonly wsEndpoint: string; readonly headers: HeadersArray = []; - static async connect(progress: (Progress|undefined), url: string, headers?: { [key: string]: string; }, followRedirects?: boolean, debugLogHeader?: string): Promise { - return await WebSocketTransport._connect(progress, url, headers || {}, { follow: !!followRedirects, hadRedirects: false }, debugLogHeader); + static async connect(progress: (Progress|undefined), url: string, options: WebSocketTransportOptions = {}): Promise { + return await WebSocketTransport._connect(progress, url, options, false /* hadRedirects */); } - static async _connect(progress: (Progress|undefined), url: string, headers: { [key: string]: string; }, redirect: { follow: boolean, hadRedirects: boolean }, debugLogHeader?: string): Promise { + static async _connect(progress: (Progress|undefined), url: string, options: WebSocketTransportOptions, hadRedirects: boolean): Promise { const logUrl = stripQueryParams(url); progress?.log(` ${logUrl}`); - const transport = new WebSocketTransport(progress, url, logUrl, headers, redirect.follow && redirect.hadRedirects, debugLogHeader); + const transport = new WebSocketTransport(progress, url, logUrl, { ...options, followRedirects: !!options.followRedirects && hadRedirects }); let success = false; progress?.cleanupWhenAborted(async () => { if (!success) @@ -94,13 +101,13 @@ export class WebSocketTransport implements ConnectionTransport { transport._ws.close(); }); transport._ws.on('unexpected-response', (request: ClientRequest, response: IncomingMessage) => { - if (redirect.follow && !redirect.hadRedirects && (response.statusCode === 301 || response.statusCode === 302 || response.statusCode === 307 || response.statusCode === 308)) { + if (options.followRedirects && !hadRedirects && (response.statusCode === 301 || response.statusCode === 302 || response.statusCode === 307 || response.statusCode === 308)) { fulfill({ redirect: response }); transport._ws.close(); return; } for (let i = 0; i < response.rawHeaders.length; i += 2) { - if (debugLogHeader && response.rawHeaders[i] === debugLogHeader) + if (options.debugLogHeader && response.rawHeaders[i] === options.debugLogHeader) progress?.log(response.rawHeaders[i + 1]); } const chunks: Buffer[] = []; @@ -117,32 +124,34 @@ export class WebSocketTransport implements ConnectionTransport { if (result.redirect) { // Strip authorization headers from the redirected request. - const newHeaders = Object.fromEntries(Object.entries(headers || {}).filter(([name]) => { + const newHeaders = Object.fromEntries(Object.entries(options.headers || {}).filter(([name]) => { return !name.includes('access-key') && name.toLowerCase() !== 'authorization'; })); - return WebSocketTransport._connect(progress, result.redirect.headers.location!, newHeaders, { follow: true, hadRedirects: true }, debugLogHeader); + return WebSocketTransport._connect(progress, result.redirect.headers.location!, { ...options, headers: newHeaders }, true /* hadRedirects */); } success = true; return transport; } - constructor(progress: Progress|undefined, url: string, logUrl: string, headers?: { [key: string]: string; }, followRedirects?: boolean, debugLogHeader?: string) { + constructor(progress: Progress|undefined, url: string, logUrl: string, options: WebSocketTransportOptions) { this.wsEndpoint = url; this._logUrl = logUrl; + const proxyAgent = createProxyAgent(options.proxy, new URL(url)); + const happyEyeballsAgent = (/^(https|wss):\/\//.test(url)) ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent; this._ws = new ws(url, [], { maxPayload: 256 * 1024 * 1024, // 256Mb, // Prevent internal http client error when passing negative timeout. handshakeTimeout: Math.max(progress?.timeUntilDeadline() ?? 30_000, 1), - headers, - followRedirects, - agent: (/^(https|wss):\/\//.test(url)) ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent, + headers: options.headers, + followRedirects: options.followRedirects, + agent: proxyAgent || happyEyeballsAgent, perMessageDeflate, }); this._ws.on('upgrade', response => { for (let i = 0; i < response.rawHeaders.length; i += 2) { this.headers.push({ name: response.rawHeaders[i], value: response.rawHeaders[i + 1] }); - if (debugLogHeader && response.rawHeaders[i] === debugLogHeader) + if (options.debugLogHeader && response.rawHeaders[i] === options.debugLogHeader) progress?.log(response.rawHeaders[i + 1]); } }); diff --git a/packages/playwright-core/src/server/utils/network.ts b/packages/playwright-core/src/server/utils/network.ts index 8e0357f958..d954a69bdf 100644 --- a/packages/playwright-core/src/server/utils/network.ts +++ b/packages/playwright-core/src/server/utils/network.ts @@ -20,10 +20,11 @@ import http2 from 'http2'; import https from 'https'; import url from 'url'; -import { HttpsProxyAgent, getProxyForUrl } from '../../utilsBundle'; +import { HttpsProxyAgent, SocksProxyAgent, getProxyForUrl } from '../../utilsBundle'; import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happyEyeballs'; import type net from 'net'; +import type { ProxySettings } from '../types'; export type HTTPRequestParams = { url: string, @@ -32,6 +33,7 @@ export type HTTPRequestParams = { data?: string | Buffer, timeout?: number, rejectUnauthorized?: boolean, + proxy?: ProxySettings, }; export const NET_DEFAULT_TIMEOUT = 30_000; @@ -49,22 +51,26 @@ export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.Inco const timeout = params.timeout ?? NET_DEFAULT_TIMEOUT; - const proxyURL = getProxyForUrl(params.url); - if (proxyURL) { - const parsedProxyURL = url.parse(proxyURL); - if (params.url.startsWith('http:')) { - options = { - path: parsedUrl.href, - host: parsedProxyURL.hostname, - port: parsedProxyURL.port, - headers: options.headers, - method: options.method - }; - } else { - (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:'; + if (params.proxy) { + options.agent = createProxyAgent(params.proxy, new URL(params.url)); + } else { + const proxyURL = getProxyForUrl(params.url); + if (proxyURL) { + const parsedProxyURL = url.parse(proxyURL); + if (params.url.startsWith('http:')) { + options = { + path: parsedUrl.href, + host: parsedProxyURL.hostname, + port: parsedProxyURL.port, + headers: options.headers, + method: options.method + }; + } else { + (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:'; - options.agent = new HttpsProxyAgent(parsedProxyURL); - options.rejectUnauthorized = false; + options.agent = new HttpsProxyAgent(parsedProxyURL); + options.rejectUnauthorized = false; + } } } @@ -109,6 +115,47 @@ export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequ }); } +function shouldBypassProxy(url: URL, bypass?: string): boolean { + if (!bypass) + return false; + const domains = bypass.split(',').map(s => { + s = s.trim(); + if (!s.startsWith('.')) + s = '.' + s; + return s; + }); + const domain = '.' + url.hostname; + return domains.some(d => domain.endsWith(d)); +} + +export function createProxyAgent(proxy?: ProxySettings, forUrl?: URL) { + if (!proxy) + return; + if (forUrl && proxy.bypass && shouldBypassProxy(forUrl, proxy.bypass)) + return; + + // Browsers allow to specify proxy without a protocol, defaulting to http. + let proxyServer = proxy.server.trim(); + if (!/^\w+:\/\//.test(proxyServer)) + proxyServer = 'http://' + proxyServer; + + const proxyOpts = url.parse(proxyServer); + if (proxyOpts.protocol?.startsWith('socks')) { + return new SocksProxyAgent({ + host: proxyOpts.hostname, + port: proxyOpts.port || undefined, + }); + } + if (proxy.username) + proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`; + if (forUrl && ['ws:', 'wss:'].includes(forUrl.protocol)) { + // Force CONNECT method for WebSockets. + return new HttpsProxyAgent(proxyOpts); + } + // TODO: We should use HttpProxyAgent conditional on proxyOpts.protocol instead of always using CONNECT method. + return new HttpsProxyAgent(proxyOpts); +} + export function createHttpServer(requestListener?: (req: http.IncomingMessage, res: http.ServerResponse) => void): http.Server; export function createHttpServer(options: http.ServerOptions, requestListener?: (req: http.IncomingMessage, res: http.ServerResponse) => void): http.Server; export function createHttpServer(...args: any[]): http.Server { diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index aff26e8942..8ad9c6e348 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -21793,6 +21793,34 @@ export interface ConnectOverCDPOptions { */ logger?: Logger; + /** + * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used + * by the browser to load web pages. + */ + proxy?: { + /** + * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example + * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP + * proxy. + */ + server: string; + + /** + * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. + */ + bypass?: string; + + /** + * Optional username to use if HTTP proxy requires authentication. + */ + username?: string; + + /** + * Optional password to use if HTTP proxy requires authentication. + */ + password?: string; + }; + /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. @@ -21834,6 +21862,34 @@ export interface ConnectOptions { */ logger?: Logger; + /** + * Proxy settings to use for the connection between the client and the remote browser. Note this proxy **is not** used + * by the browser to load web pages. + */ + proxy?: { + /** + * Proxy to be used for the remote connection. HTTP and SOCKS proxies are supported, for example + * `http://myproxy.com:3128` or `socks5://myproxy.com:3128`. Short form `myproxy.com:3128` is considered an HTTP + * proxy. + */ + server: string; + + /** + * Optional comma-separated domains to bypass proxy, for example `".com, chromium.org, .domain.com"`. + */ + bypass?: string; + + /** + * Optional username to use if HTTP proxy requires authentication. + */ + username?: string; + + /** + * Optional password to use if HTTP proxy requires authentication. + */ + password?: string; + }; + /** * Slows down Playwright operations by the specified amount of milliseconds. Useful so that you can see what is going * on. Defaults to 0. diff --git a/packages/protocol/src/channels.d.ts b/packages/protocol/src/channels.d.ts index 2e1c447b88..5a9c5687b6 100644 --- a/packages/protocol/src/channels.d.ts +++ b/packages/protocol/src/channels.d.ts @@ -540,6 +540,12 @@ export type LocalUtilsConnectParams = { exposeNetwork?: string, slowMo?: number, timeout?: number, + proxy?: { + server: string, + bypass?: string, + username?: string, + password?: string, + }, socksProxyRedirectPortForTest?: number, }; export type LocalUtilsConnectOptions = { @@ -547,6 +553,12 @@ export type LocalUtilsConnectOptions = { exposeNetwork?: string, slowMo?: number, timeout?: number, + proxy?: { + server: string, + bypass?: string, + username?: string, + password?: string, + }, socksProxyRedirectPortForTest?: number, }; export type LocalUtilsConnectResult = { @@ -1165,11 +1177,23 @@ export type BrowserTypeConnectOverCDPParams = { headers?: NameValue[], slowMo?: number, timeout?: number, + proxy?: { + server: string, + bypass?: string, + username?: string, + password?: string, + }, }; export type BrowserTypeConnectOverCDPOptions = { headers?: NameValue[], slowMo?: number, timeout?: number, + proxy?: { + server: string, + bypass?: string, + username?: string, + password?: string, + }, }; export type BrowserTypeConnectOverCDPResult = { browser: BrowserChannel, diff --git a/packages/protocol/src/protocol.yml b/packages/protocol/src/protocol.yml index c146e995da..a169f1a54c 100644 --- a/packages/protocol/src/protocol.yml +++ b/packages/protocol/src/protocol.yml @@ -702,6 +702,13 @@ LocalUtils: exposeNetwork: string? slowMo: number? timeout: number? + proxy: + type: object? + properties: + server: string + bypass: string? + username: string? + password: string? socksProxyRedirectPortForTest: number? returns: pipe: JsonPipe @@ -1010,6 +1017,13 @@ BrowserType: items: NameValue slowMo: number? timeout: number? + proxy: + type: object? + properties: + server: string + bypass: string? + username: string? + password: string? returns: browser: Browser defaultContext: BrowserContext? diff --git a/tests/library/browsertype-connect.spec.ts b/tests/library/browsertype-connect.spec.ts index 1298e38b5e..df182c32ae 100644 --- a/tests/library/browsertype-connect.spec.ts +++ b/tests/library/browsertype-connect.spec.ts @@ -765,6 +765,24 @@ for (const kind of ['launchServer', 'run-server'] as const) { await browser.close(); }); + test('should connect over http proxy', { + annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/33894' }, + }, async ({ connect, startRemoteServer, proxyServer }) => { + const remoteServer = await startRemoteServer(kind); + const url = new URL(remoteServer.wsEndpoint()); + proxyServer.forwardTo(+url.port, { allowConnectRequests: true }); + const browser = await connect(`http://some.random.host.does.not.exist:1337`, { + proxy: { server: `localhost:${proxyServer.PORT}` }, + }); + const page = await browser.newPage(); + expect(await page.evaluate('11 * 11')).toBe(121); + await browser.close(); + // We should "CONNECT" twice: + // - to convert http url into ws url + // - actually connect to the ws endpoint + expect(proxyServer.connectHosts).toEqual(['some.random.host.does.not.exist:1337', 'some.random.host.does.not.exist:1337']); + }); + test.describe('socks proxy', () => { test.skip(({ mode }) => mode !== 'default'); test.skip(kind === 'launchServer', 'not supported yet'); diff --git a/tests/library/chromium/connect-over-cdp.spec.ts b/tests/library/chromium/connect-over-cdp.spec.ts index 3191403be8..71ceefbe45 100644 --- a/tests/library/chromium/connect-over-cdp.spec.ts +++ b/tests/library/chromium/connect-over-cdp.spec.ts @@ -446,6 +446,32 @@ test('should be able to connect via localhost', async ({ browserType }, testInfo } }); +test('should be able to connect over http proxy', { + annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/35206' }, +}, async ({ browserType, proxyServer }, testInfo) => { + const port = 9339 + testInfo.workerIndex; + const browserServer = await browserType.launch({ + args: ['--remote-debugging-port=' + port] + }); + proxyServer.forwardTo(port, { allowConnectRequests: true }); + try { + const cdpBrowser = await browserType.connectOverCDP(`http://some.random.host.does.not.exist:1337`, { + proxy: { server: `localhost:${proxyServer.PORT}` }, + }); + const contexts = cdpBrowser.contexts(); + expect(contexts.length).toBe(1); + const page = await contexts[0].newPage(); + expect(await page.evaluate('11 * 11')).toBe(121); + await cdpBrowser.close(); + // We should "CONNECT" twice: + // - to convert http url into ws url + // - actually connect to the ws endpoint + expect(proxyServer.connectHosts).toEqual(['some.random.host.does.not.exist:1337', 'some.random.host.does.not.exist:1337']); + } finally { + await browserServer.close(); + } +}); + test('emulate media should not be affected by second connectOverCDP', async ({ browserType }, testInfo) => { test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/24109' }); test.fixme(); diff --git a/tests/library/proxy.spec.ts b/tests/library/proxy.spec.ts index b7df2b66f7..71233106b5 100644 --- a/tests/library/proxy.spec.ts +++ b/tests/library/proxy.spec.ts @@ -323,7 +323,7 @@ it('should use SOCKS proxy for websocket requests', async ({ browserType, server await closeProxyServer(); }); -it('should use http proxy for websocket requests', async ({ browserName, browserType, server, proxyServer }) => { +it('should use http proxy for websocket requests', async ({ isWindows, browserName, browserType, server, proxyServer }) => { proxyServer.forwardTo(server.PORT, { allowConnectRequests: true }); const browser = await browserType.launch({ proxy: { server: `localhost:${proxyServer.PORT}` } @@ -350,7 +350,7 @@ it('should use http proxy for websocket requests', async ({ browserName, browser // WebKit does not use CONNECT for websockets, but other browsers do. if (browserName === 'webkit') - expect(proxyServer.wsUrls).toContain('ws://fake-localhost-127-0-0-1.nip.io:1337/ws'); + expect(proxyServer.wsUrls).toContain(isWindows ? '/ws' : 'ws://fake-localhost-127-0-0-1.nip.io:1337/ws'); else expect(proxyServer.connectHosts).toContain('fake-localhost-127-0-0-1.nip.io:1337');