root
/
playwright
mirror of https://github.com/microsoft/playwright.git
1
0
Fork 0
Code Issues Actions 3 Packages Projects Releases Wiki Activity

fix(test runner): make sure options, trace and screenshot apply to all contexts (#8083) 

- Uses some auto fixtures to set default options and instrumentation on BrowserType.
- Moves screenshot, trace and video to worker-scoped fixtures.
- Throws in page/context when used from beforeAll/afterAll.
- Plumbs around BrowserType to be accessible from Browser and BrowserContext.
This commit is contained in:
Dmitry Gozman 2021-08-09 18:09:11 -07:00 committed by GitHub
parent 8eac1e96d3
commit 3bf3318350
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 711 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/client/browser.ts
View File

@ -23,12 +23,14 @@ import { BrowserContextOptions } from './types';
import { isSafeCloseError } from '../utils/errors';
import * as api from '../../types/types';
import { CDPSession } from './cdpSession';
import type { BrowserType } from './browserType';
export class Browser extends ChannelOwner<channels.BrowserChannel, channels.BrowserInitializer> implements api.Browser {
readonly _contexts = new Set<BrowserContext>();
private _isConnected = true;
private _closedPromise: Promise<void>;
_remoteType: 'owns-connection' | 'uses-connection' | null = null;
private _browserType!: BrowserType;
readonly _name: string;
static from(browser: channels.BrowserChannel): Browser {
@ -46,13 +48,22 @@ export class Browser extends ChannelOwner<channels.BrowserChannel, channels.Brow
this._closedPromise = new Promise(f => this.once(Events.Browser.Disconnected, f));
}
_setBrowserType(browserType: BrowserType) {
this._browserType = browserType;
for (const context of this._contexts)
context._setBrowserType(browserType);
}
async newContext(options: BrowserContextOptions = {}): Promise<BrowserContext> {
return this._wrapApiCall(async (channel: channels.BrowserChannel) => {
options = { ...this._browserType._defaultContextOptions, ...options };
const contextOptions = await prepareBrowserContextParams(options);
const context = BrowserContext.from((await channel.newContext(contextOptions)).context);
context._options = contextOptions;
this._contexts.add(context);
context._logger = options.logger || this._logger;
context._setBrowserType(this._browserType);
await this._browserType._onDidCreateContext?.(context);
return context;
});
}

9
src/client/browserContext.ts
View File

@ -33,11 +33,13 @@ import * as api from '../../types/types';
import * as structs from '../../types/structs';
import { CDPSession } from './cdpSession';
import { Tracing } from './tracing';
import type { BrowserType } from './browserType';
export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel, channels.BrowserContextInitializer> implements api.BrowserContext {
_pages = new Set<Page>();
private _routes: { url: URLMatch, handler: network.RouteHandler }[] = [];
readonly _browser: Browser | null = null;
private _browserType: BrowserType | undefined;
readonly _bindings = new Map<string, (source: structs.BindingSource, ...args: any[]) => any>();
_timeoutSettings = new TimeoutSettings();
_ownerPage: Page | undefined;
@ -89,6 +91,11 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
this._closedPromise = new Promise(f => this.once(Events.BrowserContext.Close, f));
}
_setBrowserType(browserType: BrowserType) {
this._browserType = browserType;
browserType._contexts.add(this);
}
private _onPage(page: Page): void {
this._pages.add(page);
this.emit(Events.BrowserContext.Page, page);
@ -311,12 +318,14 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
_onClose() {
if (this._browser)
this._browser._contexts.delete(this);
this._browserType?._contexts?.delete(this);
this.emit(Events.BrowserContext.Close, this);
}
async close(): Promise<void> {
try {
await this._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
await this._browserType?._onWillCloseContext?.(this);
await channel.close();
await this._closedPromise;
});

16
src/client/browserType.ts
View File

@ -18,7 +18,7 @@ import * as channels from '../protocol/channels';
import { Browser } from './browser';
import { BrowserContext, prepareBrowserContextParams } from './browserContext';
import { ChannelOwner } from './channelOwner';
import { LaunchOptions, LaunchServerOptions, ConnectOptions, LaunchPersistentContextOptions } from './types';
import { LaunchOptions, LaunchServerOptions, ConnectOptions, LaunchPersistentContextOptions, BrowserContextOptions } from './types';
import WebSocket from 'ws';
import { Connection } from './connection';
import { Events } from './events';
@ -45,6 +45,13 @@ export interface BrowserServer extends api.BrowserServer {
export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, channels.BrowserTypeInitializer> implements api.BrowserType {
private _timeoutSettings = new TimeoutSettings();
_serverLauncher?: BrowserServerLauncher;
_contexts = new Set<BrowserContext>();
// Instrumentation.
_defaultContextOptions: BrowserContextOptions = {};
_defaultLaunchOptions: LaunchOptions = {};
_onDidCreateContext?: (context: BrowserContext) => Promise<void>;
_onWillCloseContext?: (context: BrowserContext) => Promise<void>;
static from(browserType: channels.BrowserTypeChannel): BrowserType {
return (browserType as any)._object;
@ -69,6 +76,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
return this._wrapApiCall(async (channel: channels.BrowserTypeChannel) => {
assert(!(options as any).userDataDir, 'userDataDir option is not supported in `browserType.launch`. Use `browserType.launchPersistentContext` instead');
assert(!(options as any).port, 'Cannot specify a port without launching as a server.');
options = { ...this._defaultLaunchOptions, ...options };
const launchOptions: channels.BrowserTypeLaunchParams = {
...options,
ignoreDefaultArgs: Array.isArray(options.ignoreDefaultArgs) ? options.ignoreDefaultArgs : undefined,
@ -77,6 +85,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
};
const browser = Browser.from((await channel.launch(launchOptions)).browser);
browser._logger = logger;
browser._setBrowserType(this);
return browser;
}, logger);
}
@ -90,6 +99,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
async launchPersistentContext(userDataDir: string, options: LaunchPersistentContextOptions = {}): Promise<BrowserContext> {
return this._wrapApiCall(async (channel: channels.BrowserTypeChannel) => {
assert(!(options as any).port, 'Cannot specify a port without launching as a server.');
options = { ...this._defaultLaunchOptions, ...this._defaultContextOptions, ...options };
const contextParams = await prepareBrowserContextParams(options);
const persistentParams: channels.BrowserTypeLaunchPersistentContextParams = {
...contextParams,
@ -103,6 +113,8 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
const context = BrowserContext.from(result.context);
context._options = contextParams;
context._logger = options.logger;
context._setBrowserType(this);
await this._onDidCreateContext?.(context);
return context;
}, options.logger);
}
@ -181,6 +193,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
const browser = Browser.from(playwright._initializer.preLaunchedBrowser!);
browser._logger = logger;
browser._remoteType = 'owns-connection';
browser._setBrowserType((playwright as any)[browser._name]);
const closeListener = () => {
// Emulate all pages, contexts and the browser closing upon disconnect.
for (const context of browser.contexts()) {
@ -252,6 +265,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
browser._contexts.add(BrowserContext.from(result.defaultContext));
browser._remoteType = 'uses-connection';
browser._logger = logger;
browser._setBrowserType(this);
return browser;
}, logger);
}

8
src/client/channelOwner.ts
View File

@ -47,7 +47,6 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
if (this._parent) {
this._parent._objects.set(guid, this);
this._logger = this._parent._logger;
this._csi = this._parent._csi;
}
this._channel = this._createChannel(new EventEmitter(), null);
@ -95,10 +94,15 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
const stackTrace = captureStackTrace();
const { apiName, frameTexts } = stackTrace;
const channel = this._createChannel({}, stackTrace);
let ancestorWithCSI: ChannelOwner<any> = this;
while (!ancestorWithCSI._csi && ancestorWithCSI._parent)
ancestorWithCSI = ancestorWithCSI._parent;
let csiCallback: ((e?: Error) => void) | undefined;
try {
logApiCall(logger, `=> ${apiName} started`);
csiCallback = this._csi?.onApiCall(apiName);
csiCallback = ancestorWithCSI._csi?.onApiCall(apiName);
const result = await func(channel as any, stackTrace);
csiCallback?.();
logApiCall(logger, `<= ${apiName} succeeded`);

3
src/client/page.ts
View File

@ -487,9 +487,10 @@ export class Page extends ChannelOwner<channels.PageChannel, channels.PageInitia
async close(options: { runBeforeUnload?: boolean } = {runBeforeUnload: undefined}) {
try {
await this._wrapApiCall(async (channel: channels.PageChannel) => {
await channel.close(options);
if (this._ownedContext)
await this._ownedContext.close();
else
await channel.close(options);
});
} catch (e) {
if (isSafeCloseError(e))

244
src/test/index.ts
View File

@ -16,27 +16,51 @@
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext } from '../../types/types';
import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext, BrowserType } from '../../types/types';
import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions } from '../../types/test';
import { rootTestType } from './testType';
import { createGuid, removeFolders } from '../utils/utils';
export { expect } from './expect';
export const _baseTest: TestType<{}, {}> = rootTestType.test;
const artifactsFolder = path.join(os.tmpdir(), 'pwt-' + createGuid());
type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {
_combinedContextOptions: BrowserContextOptions,
_setupContextOptionsAndArtifacts: void;
};
type WorkerAndFileFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {
_browserType: BrowserType;
_artifactsDir: () => string,
};
export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions, PlaywrightWorkerArgs & PlaywrightWorkerOptions>({
export const test = _baseTest.extend<TestFixtures, WorkerAndFileFixtures>({
defaultBrowserType: [ 'chromium', { scope: 'worker' } ],
browserName: [ ({ defaultBrowserType }, use) => use(defaultBrowserType), { scope: 'worker' } ],
playwright: [ require('../inprocess'), { scope: 'worker' } ],
headless: [ undefined, { scope: 'worker' } ],
channel: [ undefined, { scope: 'worker' } ],
launchOptions: [ {}, { scope: 'worker' } ],
screenshot: [ 'off', { scope: 'worker' } ],
video: [ 'off', { scope: 'worker' } ],
trace: [ 'off', { scope: 'worker' } ],
browser: [ async ({ playwright, browserName, headless, channel, launchOptions }, use) => {
_artifactsDir: [async ({}, use, workerInfo) => {
let dir: string | undefined;
await use(() => {
if (!dir) {
dir = path.join(workerInfo.project.outputDir, '.playwright-artifacts-' + workerInfo.workerIndex);
fs.mkdirSync(dir, { recursive: true });
}
return dir;
});
if (dir)
await removeFolders([dir]);
}, { scope: 'worker' }],
_browserType: [async ({ playwright, browserName, headless, channel, launchOptions }, use) => {
if (!['chromium', 'firefox', 'webkit'].includes(browserName))
throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`);
const browserType = playwright[browserName];
const options: LaunchOptions = {
handleSIGINT: false,
timeout: 0,
@ -46,15 +70,18 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
options.headless = headless;
if (channel !== undefined)
options.channel = channel;
const browser = await playwright[browserName].launch(options);
await use(browser);
await browser.close();
await removeFolders([artifactsFolder]);
(browserType as any)._defaultLaunchOptions = options;
await use(browserType);
(browserType as any)._defaultLaunchOptions = undefined;
}, { scope: 'worker' }],
browser: [ async ({ _browserType }, use) => {
const browser = await _browserType.launch();
await use(browser);
await browser.close();
}, { scope: 'worker' } ],
screenshot: 'off',
video: 'off',
trace: 'off',
acceptDownloads: undefined,
bypassCSP: undefined,
colorScheme: undefined,
@ -81,11 +108,7 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
},
contextOptions: {},
createContext: async ({
browser,
screenshot,
trace,
video,
_combinedContextOptions: async ({
acceptDownloads,
bypassCSP,
colorScheme,
@ -107,22 +130,7 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
userAgent,
baseURL,
contextOptions,
actionTimeout,
navigationTimeout
}, use, testInfo) => {
testInfo.snapshotSuffix = process.platform;
if (process.env.PWDEBUG)
testInfo.setTimeout(0);
let videoMode = typeof video === 'string' ? video : video.mode;
if (videoMode === 'retry-with-video')
videoMode = 'on-first-retry';
if (trace === 'retry-with-trace')
trace = 'on-first-retry';
const captureVideo = (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1));
const captureTrace = (trace === 'on' || trace === 'retain-on-failure' || (trace === 'on-first-retry' && testInfo.retry === 1));
}, use) => {
const options: BrowserContextOptions = {};
if (acceptDownloads !== undefined)
options.acceptDownloads = acceptDownloads;
@ -164,6 +172,124 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
options.viewport = viewport;
if (baseURL !== undefined)
options.baseURL = baseURL;
await use({
...contextOptions,
...options,
});
},
_setupContextOptionsAndArtifacts: [async ({ _browserType, _combinedContextOptions, _artifactsDir, trace, screenshot, actionTimeout, navigationTimeout }, use, testInfo) => {
testInfo.snapshotSuffix = process.platform;
if (process.env.PWDEBUG)
testInfo.setTimeout(0);
if (trace === 'retry-with-trace')
trace = 'on-first-retry';
const captureTrace = (trace === 'on' || trace === 'retain-on-failure' || (trace === 'on-first-retry' && testInfo.retry === 1));
const temporaryTraceFiles: string[] = [];
const temporaryScreenshots: string[] = [];
const onDidCreateContext = async (context: BrowserContext) => {
context.setDefaultTimeout(actionTimeout || 0);
context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0);
if (captureTrace)
await context.tracing.start({ screenshots: true, snapshots: true });
(context as any)._csi = {
onApiCall: (name: string) => {
return (testInfo as any)._addStep('pw:api', name);
},
};
};
const onWillCloseContext = async (context: BrowserContext) => {
if (captureTrace) {
// Export trace for now. We'll know whether we have to preserve it
// after the test finishes.
const tracePath = path.join(_artifactsDir(), createGuid() + '.zip');
temporaryTraceFiles.push(tracePath);
await (context.tracing as any)._export({ path: tracePath });
}
if (screenshot === 'on' || screenshot === 'only-on-failure') {
// Capture screenshot for now. We'll know whether we have to preserve them
// after the test finishes.
await Promise.all(context.pages().map(async page => {
const screenshotPath = path.join(_artifactsDir(), createGuid() + '.png');
temporaryScreenshots.push(screenshotPath);
await page.screenshot({ timeout: 5000, path: screenshotPath }).catch(() => {});
}));
}
};
// 1. Setup instrumentation and process existing contexts.
const oldOnDidCreateContext = (_browserType as any)._onDidCreateContext;
(_browserType as any)._onDidCreateContext = onDidCreateContext;
(_browserType as any)._onWillCloseContext = onWillCloseContext;
(_browserType as any)._defaultContextOptions = _combinedContextOptions;
const existingContexts = Array.from((_browserType as any)._contexts) as BrowserContext[];
await Promise.all(existingContexts.map(onDidCreateContext));
// 2. Run the test.
await use();
// 3. Determine whether we need the artifacts.
const testFailed = testInfo.status !== testInfo.expectedStatus;
const isHook = testInfo.title === 'beforeAll' || testInfo.title === 'afterAll';
const preserveTrace = captureTrace && !isHook && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1));
const captureScreenshots = !isHook && (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed));
const traceAttachments: string[] = [];
const addTraceAttachment = () => {
const tracePath = testInfo.outputPath(`trace${traceAttachments.length ? '-' + traceAttachments.length : ''}.zip`);
traceAttachments.push(tracePath);
testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
return tracePath;
};
const screenshotAttachments: string[] = [];
const addScreenshotAttachment = () => {
const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${screenshotAttachments.length + 1}.png`);
screenshotAttachments.push(screenshotPath);
testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' });
return screenshotPath;
};
// 4. Cleanup instrumentation.
const leftoverContexts = Array.from((_browserType as any)._contexts) as BrowserContext[];
(_browserType as any)._onDidCreateContext = oldOnDidCreateContext;
(_browserType as any)._onWillCloseContext = undefined;
(_browserType as any)._defaultContextOptions = undefined;
leftoverContexts.forEach(context => (context as any)._csi = undefined);
// 5. Collect artifacts from any non-closed contexts.
await Promise.all(leftoverContexts.map(async context => {
if (preserveTrace)
await (context.tracing as any)._export({ path: addTraceAttachment() });
if (captureScreenshots)
await Promise.all(context.pages().map(page => page.screenshot({ timeout: 5000, path: addScreenshotAttachment() }).catch(() => {})));
}));
// 6. Either remove or attach temporary traces and screenshots for contexts closed
// before the test has finished.
await Promise.all(temporaryTraceFiles.map(async file => {
if (preserveTrace)
await fs.promises.rename(file, addTraceAttachment()).catch(() => {});
else
await fs.promises.unlink(file).catch(() => {});
}));
await Promise.all(temporaryScreenshots.map(async file => {
if (captureScreenshots)
await fs.promises.rename(file, addScreenshotAttachment()).catch(() => {});
else
await fs.promises.unlink(file).catch(() => {});
}));
}, { auto: true }],
createContext: async ({ browser, video, _artifactsDir }, use, testInfo) => {
let videoMode = typeof video === 'string' ? video : video.mode;
if (videoMode === 'retry-with-video')
videoMode = 'on-first-retry';
const captureVideo = (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1));
const allContexts: BrowserContext[] = [];
const allPages: Page[] = [];
@ -171,63 +297,20 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
await use(async (additionalOptions = {}) => {
let recordVideoDir: string | null = null;
const recordVideoSize = typeof video === 'string' ? undefined : video.size;
if (captureVideo) {
await fs.promises.mkdir(artifactsFolder, { recursive: true });
recordVideoDir = artifactsFolder;
}
if (captureVideo)
recordVideoDir = _artifactsDir();
const combinedOptions: BrowserContextOptions = {
recordVideo: recordVideoDir ? { dir: recordVideoDir, size: recordVideoSize } : undefined,
...contextOptions,
...options,
...additionalOptions,
};
const context = await browser.newContext(combinedOptions);
(context as any)._csi = {
onApiCall: (name: string) => {
return (testInfo as any)._addStep('pw:api', name);
},
};
context.setDefaultTimeout(actionTimeout || 0);
context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0);
context.on('page', page => allPages.push(page));
if (captureTrace) {
const name = path.relative(testInfo.project.outputDir, testInfo.outputDir).replace(/[\/\\]/g, '-');
const suffix = allContexts.length ? '-' + allContexts.length : '';
await context.tracing.start({ name: name + suffix, screenshots: true, snapshots: true });
}
allContexts.push(context);
return context;
});
const testFailed = testInfo.status !== testInfo.expectedStatus;
await Promise.all(allContexts.map(async (context, contextIndex) => {
const preserveTrace = captureTrace && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1));
if (preserveTrace) {
const suffix = contextIndex ? '-' + contextIndex : '';
const tracePath = testInfo.outputPath(`trace${suffix}.zip`);
await context.tracing.stop({ path: tracePath });
testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
} else if (captureTrace) {
await context.tracing.stop();
}
}));
const captureScreenshots = (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed));
if (captureScreenshots) {
await Promise.all(allPages.map(async (page, index) => {
const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index + 1}.png`);
try {
await page.screenshot({ timeout: 5000, path: screenshotPath });
testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' });
} catch {
}
}));
}
const prependToError = (testInfo.status === 'timedOut' && allContexts.length) ?
formatPendingCalls((allContexts[0] as any)._connection.pendingProtocolCalls()) : '';
await Promise.all(allContexts.map(context => context.close()));
@ -241,6 +324,7 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
}
}
const testFailed = testInfo.status !== testInfo.expectedStatus;
const preserveVideo = captureVideo && (videoMode === 'on' || (testFailed && videoMode === 'retain-on-failure') || (videoMode === 'on-first-retry' && testInfo.retry === 1));
if (preserveVideo) {
await Promise.all(allPages.map(async page => {
@ -259,7 +343,9 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
}
},
context: async ({ createContext }, use) => {
context: async ({ createContext }, use, testInfo) => {
if (testInfo.title === 'beforeAll' || testInfo.title === 'afterAll')
throw new Error(`"context" and "page" fixtures are not suppoted in ${testInfo.title}. Use browser.newContext() instead.`);
await use(await createContext());
},

85
tests/playwright-test/golden.spec.ts
View File

@ -19,11 +19,23 @@ import * as fs from 'fs';
import * as path from 'path';
import { test, expect, stripAscii } from './playwright-test-fixtures';
const files = {
'helper.ts': `
export const test = pwt.test.extend({
auto: [ async ({}, run, testInfo) => {
testInfo.snapshotSuffix = '';
await run();
}, { auto: true } ]
});
`
};
test('should support golden', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot('snapshot.txt');
});
@ -34,6 +46,7 @@ test('should support golden', async ({runInlineTest}) => {
test('should fail on wrong golden', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': `Line1
Line2
Line3
@ -42,7 +55,7 @@ Line5
Line6
Line7`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
const data = [];
data.push('Line1');
@ -67,9 +80,10 @@ Line7`,
test('should write detailed failure result to an output folder', async ({runInlineTest}, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world updated').toMatchSnapshot('snapshot.txt');
});
@ -89,9 +103,10 @@ test('should write detailed failure result to an output folder', async ({runInli
test("doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher", async ({runInlineTest}, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world updated').not.toMatchSnapshot('snapshot.txt');
});
@ -110,9 +125,10 @@ test("doesn\'t create comparison artifacts in an output folder for passed negate
test('should pass on different snapshots with negate matcher', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world updated').not.toMatchSnapshot('snapshot.txt');
});
@ -124,9 +140,10 @@ test('should pass on different snapshots with negate matcher', async ({runInline
test('should fail on same snapshots with negate matcher', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').not.toMatchSnapshot('snapshot.txt');
});
@ -140,8 +157,9 @@ test('should fail on same snapshots with negate matcher', async ({runInlineTest}
test('should write missing expectations locally', async ({runInlineTest}, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot('snapshot.txt');
});
@ -157,8 +175,9 @@ test('should write missing expectations locally', async ({runInlineTest}, testIn
test('shouldn\'t write missing expectations locally for negated matcher', async ({runInlineTest}, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').not.toMatchSnapshot('snapshot.txt');
});
@ -175,9 +194,10 @@ test('should update snapshot with the update-snapshots flag', async ({runInlineT
const EXPECTED_SNAPSHOT = 'Hello world';
const ACTUAL_SNAPSHOT = 'Hello world updated';
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt');
});
@ -195,9 +215,10 @@ test('shouldn\'t update snapshot with the update-snapshots flag for negated matc
const EXPECTED_SNAPSHOT = 'Hello world';
const ACTUAL_SNAPSHOT = 'Hello world updated';
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('${ACTUAL_SNAPSHOT}').not.toMatchSnapshot('snapshot.txt');
});
@ -213,8 +234,9 @@ test('shouldn\'t update snapshot with the update-snapshots flag for negated matc
test('should silently write missing expectations locally with the update-snapshots flag', async ({runInlineTest}, testInfo) => {
const ACTUAL_SNAPSHOT = 'Hello world new';
const result = await runInlineTest({
...files,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt');
});
@ -230,8 +252,9 @@ test('should silently write missing expectations locally with the update-snapsho
test('should silently write missing expectations locally with the update-snapshots flag for negated matcher', async ({runInlineTest}, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').not.toMatchSnapshot('snapshot.txt');
});
@ -246,11 +269,12 @@ test('should silently write missing expectations locally with the update-snapsho
test('should match multiple snapshots', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot1.txt': `Snapshot1`,
'a.spec.js-snapshots/snapshot2.txt': `Snapshot2`,
'a.spec.js-snapshots/snapshot3.txt': `Snapshot3`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Snapshot1').toMatchSnapshot('snapshot1.txt');
expect('Snapshot2').toMatchSnapshot('snapshot2.txt');
@ -263,6 +287,7 @@ test('should match multiple snapshots', async ({runInlineTest}) => {
test('should match snapshots from multiple projects', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'playwright.config.ts': `
import * as path from 'path';
module.exports = { projects: [
@ -271,14 +296,14 @@ test('should match snapshots from multiple projects', async ({runInlineTest}) =>
]};
`,
'p1/a.spec.js': `
const { test } = pwt;
const { test } = require('../helper');
test('is a test', ({}) => {
expect('Snapshot1').toMatchSnapshot('snapshot.txt');
});
`,
'p1/a.spec.js-snapshots/snapshot.txt': `Snapshot1`,
'p2/a.spec.js': `
const { test } = pwt;
const { test } = require('../helper');
test('is a test', ({}) => {
expect('Snapshot2').toMatchSnapshot('snapshot.txt');
});
@ -290,9 +315,10 @@ test('should match snapshots from multiple projects', async ({runInlineTest}) =>
test('should use provided name', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/provided.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot('provided.txt');
});
@ -303,8 +329,9 @@ test('should use provided name', async ({runInlineTest}) => {
test('should throw without a name', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot();
});
@ -316,9 +343,10 @@ test('should throw without a name', async ({runInlineTest}) => {
test('should use provided name via options', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/provided.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot({ name: 'provided.txt' });
});
@ -329,9 +357,10 @@ test('should use provided name via options', async ({runInlineTest}) => {
test('should compare binary', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.dat': Buffer.from([1,2,3,4]),
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect(Buffer.from([1,2,3,4])).toMatchSnapshot('snapshot.dat');
});
@ -342,10 +371,11 @@ test('should compare binary', async ({runInlineTest}) => {
test('should compare PNG images', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.png':
Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'),
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64')).toMatchSnapshot('snapshot.png');
});
@ -356,10 +386,11 @@ test('should compare PNG images', async ({runInlineTest}) => {
test('should compare different PNG images', async ({runInlineTest}, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.png':
Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'),
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png');
});
@ -384,10 +415,11 @@ test('should respect threshold', async ({runInlineTest}) => {
const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png'));
const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png'));
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/snapshot.png': expected,
'a.spec.js-snapshots/snapshot2.png': expected,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 });
expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png', { threshold: 0.2 });
@ -403,6 +435,7 @@ test('should respect project threshold', async ({runInlineTest}) => {
const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png'));
const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png'));
const result = await runInlineTest({
...files,
'playwright.config.ts': `
module.exports = { projects: [
{ expect: { toMatchSnapshot: { threshold: 0.2 } } },
@ -411,7 +444,7 @@ test('should respect project threshold', async ({runInlineTest}) => {
'a.spec.js-snapshots/snapshot.png': expected,
'a.spec.js-snapshots/snapshot2.png': expected,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 });
expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png');
@ -425,9 +458,10 @@ test('should respect project threshold', async ({runInlineTest}) => {
test('should sanitize snapshot name', async ({runInlineTest}) => {
const result = await runInlineTest({
...files,
'a.spec.js-snapshots/-snapshot-.txt': `Hello world`,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot('../../snapshot!.txt');
});
@ -438,8 +472,9 @@ test('should sanitize snapshot name', async ({runInlineTest}) => {
test('should write missing expectations with sanitized snapshot name', async ({runInlineTest}, testInfo) => {
const result = await runInlineTest({
...files,
'a.spec.js': `
const { test } = pwt;
const { test } = require('./helper');
test('is a test', ({}) => {
expect('Hello world').toMatchSnapshot('../../snapshot!.txt');
});

280
tests/playwright-test/playwright.artifacts.spec.ts Normal file
View File

@ -0,0 +1,280 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { test, expect } from './playwright-test-fixtures';
import fs from 'fs';
import path from 'path';
function listFiles(dir: string): string[] {
const result: string[] = [];
const entries = fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
for (const entry of entries) {
result.push(entry.name);
if (entry.isDirectory())
result.push(...listFiles(path.join(dir, entry.name)).map(x => ' ' + x));
}
return result;
}
const testFiles = {
'artifacts.spec.ts': `
import fs from 'fs';
import os from 'os';
import path from 'path';
import rimraf from 'rimraf';
const { test } = pwt;
test.describe('shared', () => {
let page;
test.beforeAll(async ({ browser }) => {
page = await browser.newPage({});
await page.setContent('<button>Click me</button><button>And me</button>');
});
test.afterAll(async () => {
await page.close();
});
test('shared passing', async ({ }) => {
await page.click('text=Click me');
});
test('shared failing', async ({ }) => {
await page.click('text=And me');
expect(1).toBe(2);
});
});
test('passing', async ({ page }) => {
await page.setContent('I am the page');
});
test('two contexts', async ({ page, createContext }) => {
await page.setContent('I am the page');
const context2 = await createContext();
const page2 = await context2.newPage();
await page2.setContent('I am the page');
});
test('failing', async ({ page }) => {
await page.setContent('I am the page');
expect(1).toBe(2);
});
test('two contexts failing', async ({ page, createContext }) => {
await page.setContent('I am the page');
const context2 = await createContext();
const page2 = await context2.newPage();
await page2.setContent('I am the page');
expect(1).toBe(2);
});
test('own context passing', async ({ browser }) => {
const page = await browser.newPage();
await page.setContent('<button>Click me</button><button>And me</button>');
await page.click('text=Click me');
await page.close();
});
test('own context failing', async ({ browser }) => {
const page = await browser.newPage();
await page.setContent('<button>Click me</button><button>And me</button>');
await page.click('text=Click me');
await page.close();
expect(1).toBe(2);
});
const testPersistent = test.extend({
page: async ({ playwright, browserName }, use) => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'user-data-dir-'));
const context = await playwright[browserName].launchPersistentContext(dir);
await use(context.pages()[0]);
await context.close();
rimraf.sync(dir);
},
});
testPersistent('persistent passing', async ({ page }) => {
await page.setContent('<button>Click me</button><button>And me</button>');
});
testPersistent('persistent failing', async ({ page }) => {
await page.setContent('<button>Click me</button><button>And me</button>');
expect(1).toBe(2);
});
`,
};
test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = { use: { screenshot: 'on' } };
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
expect(result.failed).toBe(5);
expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
'artifacts-failing',
' test-failed-1.png',
'artifacts-own-context-failing',
' test-failed-1.png',
'artifacts-own-context-passing',
' test-finished-1.png',
'artifacts-passing',
' test-finished-1.png',
'artifacts-persistent-failing',
' test-failed-1.png',
'artifacts-persistent-passing',
' test-finished-1.png',
'artifacts-shared-failing',
' test-failed-1.png',
'artifacts-shared-passing',
' test-finished-1.png',
'artifacts-two-contexts',
' test-finished-1.png',
' test-finished-2.png',
'artifacts-two-contexts-failing',
' test-failed-1.png',
' test-failed-2.png',
'report.json',
]);
});
test('should work with screenshot: only-on-failure', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = { use: { screenshot: 'only-on-failure' } };
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
expect(result.failed).toBe(5);
expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
'artifacts-failing',
' test-failed-1.png',
'artifacts-own-context-failing',
' test-failed-1.png',
'artifacts-persistent-failing',
' test-failed-1.png',
'artifacts-shared-failing',
' test-failed-1.png',
'artifacts-two-contexts-failing',
' test-failed-1.png',
' test-failed-2.png',
'report.json',
]);
});
test('should work with trace: on', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = { use: { trace: 'on' } };
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
expect(result.failed).toBe(5);
expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
'artifacts-failing',
' trace.zip',
'artifacts-own-context-failing',
' trace.zip',
'artifacts-own-context-passing',
' trace.zip',
'artifacts-passing',
' trace.zip',
'artifacts-persistent-failing',
' trace.zip',
'artifacts-persistent-passing',
' trace.zip',
'artifacts-shared-failing',
' trace.zip',
'artifacts-shared-passing',
' trace.zip',
'artifacts-two-contexts',
' trace-1.zip',
' trace.zip',
'artifacts-two-contexts-failing',
' trace-1.zip',
' trace.zip',
'report.json',
]);
});
test('should work with trace: retain-on-failure', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = { use: { trace: 'retain-on-failure' } };
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
expect(result.failed).toBe(5);
expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
'artifacts-failing',
' trace.zip',
'artifacts-own-context-failing',
' trace.zip',
'artifacts-persistent-failing',
' trace.zip',
'artifacts-shared-failing',
' trace.zip',
'artifacts-two-contexts-failing',
' trace-1.zip',
' trace.zip',
'report.json',
]);
});
test('should work with trace: on-first-retry', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = { use: { trace: 'on-first-retry' } };
`,
}, { workers: 1, retries: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
expect(result.failed).toBe(5);
expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
'artifacts-failing-retry1',
' trace.zip',
'artifacts-own-context-failing-retry1',
' trace.zip',
'artifacts-persistent-failing-retry1',
' trace.zip',
'artifacts-shared-failing-retry1',
' trace.zip',
'artifacts-two-contexts-failing-retry1',
' trace-1.zip',
' trace.zip',
'report.json',
]);
});

139
tests/playwright-test/playwright.spec.ts
View File

@ -161,6 +161,65 @@ test('should override use:browserName with --browser', async ({ runInlineTest })
]);
});
test('should respect context options in various contexts', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { use: { viewport: { width: 500, height: 500 } } };
`,
'a.test.ts': `
import fs from 'fs';
import os from 'os';
import path from 'path';
import rimraf from 'rimraf';
const { test } = pwt;
test.use({ locale: 'fr-CH' });
let context;
test.beforeAll(async ({ browser }) => {
context = await browser.newContext();
});
test.afterAll(async () => {
await context.close();
});
test('shared context', async ({}) => {
const page = await context.newPage();
expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
});
test('own context', async ({ browser }) => {
const page = await browser.newPage();
expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
await page.close();
});
test('default context', async ({ page }) => {
expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
});
test('persistent context', async ({ playwright, browserName }) => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'user-data-dir-'));
const context = await playwright[browserName].launchPersistentContext(dir);
const page = context.pages()[0];
expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
await context.close();
rimraf.sync(dir);
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(4);
});
test('should report error and pending operations on timeout', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'a.test.ts': `
@ -185,6 +244,22 @@ test('should report error and pending operations on timeout', async ({ runInline
expect(stripAscii(result.output)).toContain(`10 | page.textContent('text=More missing'),`);
});
test('should throw when using page in beforeAll', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test.beforeAll(async ({ page }) => {
});
test('ok', async ({ page }) => {
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(0);
expect(result.output).toContain(`Error: "context" and "page" fixtures are not suppoted in beforeAll. Use browser.newContext() instead.`);
});
test('should report click error on sigint', async ({ runInlineTest }) => {
test.skip(process.platform === 'win32', 'No sending SIGINT on Windows');
@ -208,37 +283,6 @@ test('should report click error on sigint', async ({ runInlineTest }) => {
expect(stripAscii(result.output)).toContain(`8 | const promise = page.click('text=Missing');`);
});
test('should work with screenshot: only-on-failure', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { use: { screenshot: 'only-on-failure' }, name: 'chromium' };
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async ({ page }) => {
await page.setContent('<div>PASS</div>');
test.expect(1 + 1).toBe(2);
});
test('fail', async ({ page }) => {
await page.setContent('<div>FAIL</div>');
const page2 = await page.context().newPage();
await page2.setContent('<div>FAIL</div>');
test.expect(1 + 1).toBe(1);
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(1);
const screenshotPass = testInfo.outputPath('test-results', 'a-pass-chromium', 'test-failed-1.png');
const screenshotFail1 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-1.png');
const screenshotFail2 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-2.png');
expect(fs.existsSync(screenshotPass)).toBe(false);
expect(fs.existsSync(screenshotFail1)).toBe(true);
expect(fs.existsSync(screenshotFail2)).toBe(true);
});
test('should work with video: retain-on-failure', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'playwright.config.ts': `
@ -339,36 +383,3 @@ test('should work with video size', async ({ runInlineTest }, testInfo) => {
expect(videoPlayer.videoWidth).toBe(220);
expect(videoPlayer.videoHeight).toBe(110);
});
test('should work with multiple contexts and trace: on', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'playwright.config.ts': `
module.exports = { use: { trace: 'on' } };
`,
'a.test.ts': `
const { test } = pwt;
test('pass', async ({ page, createContext }) => {
await page.setContent('<div>PASS</div>');
const context1 = await createContext();
const page1 = await context1.newPage();
await page1.setContent('<div>PASS</div>');
const context2 = await createContext({ locale: 'en-US' });
const page2 = await context2.newPage();
await page2.setContent('<div>PASS</div>');
test.expect(1 + 1).toBe(2);
});
`,
}, { workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(1);
const traceDefault = testInfo.outputPath('test-results', 'a-pass', 'trace.zip');
const trace1 = testInfo.outputPath('test-results', 'a-pass', 'trace-1.zip');
const trace2 = testInfo.outputPath('test-results', 'a-pass', 'trace-2.zip');
expect(fs.existsSync(traceDefault)).toBe(true);
expect(fs.existsSync(trace1)).toBe(true);
expect(fs.existsSync(trace2)).toBe(true);
});

56
tests/playwright-test/reporter.spec.ts
View File

@ -114,7 +114,6 @@ test('should work with custom reporter', async ({ runInlineTest }) => {
]);
});
test('should work without a file extension', async ({ runInlineTest }) => {
const result = await runInlineTest({
'reporter.ts': smallReporterJS,
@ -162,6 +161,9 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => {
test('should report expect steps', async ({ runInlineTest }) => {
const expectReporterJS = `
class Reporter {
onStdOut(chunk) {
process.stdout.write(chunk);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
console.log('%%%% begin', JSON.stringify(copy));
@ -232,6 +234,15 @@ test('should report expect steps', async ({ runInlineTest }) => {
test('should report api steps', async ({ runInlineTest }) => {
const expectReporterJS = `
class Reporter {
onStdOut(chunk) {
process.stdout.write(chunk);
}
onTestBegin(test) {
console.log('%%%% test begin ' + test.title);
}
onTestEnd(test) {
console.log('%%%% test end ' + test.title);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
console.log('%%%% begin', JSON.stringify(copy));
@ -259,11 +270,31 @@ test('should report api steps', async ({ runInlineTest }) => {
await page.setContent('<button></button>');
await page.click('button');
});
test.describe('suite', () => {
let myPage;
test.beforeAll(async ({ browser }) => {
myPage = await browser.newPage();
await myPage.setContent('<button></button>');
});
test('pass1', async () => {
await myPage.click('button');
});
test('pass2', async () => {
await myPage.click('button');
});
test.afterAll(async () => {
await myPage.close();
});
});
`
}, { reporter: '', workers: 1 });
expect(result.exitCode).toBe(0);
expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
`%%%% test begin pass`,
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
@ -276,6 +307,23 @@ test('should report api steps', async ({ runInlineTest }) => {
`%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%%%% test end pass`,
`%%%% test begin pass1`,
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%%%% test end pass1`,
`%%%% test begin pass2`,
`%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
`%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
`%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
`%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
`%%%% test end pass2`,
]);
});
@ -283,6 +331,9 @@ test('should report api steps', async ({ runInlineTest }) => {
test('should report api step failure', async ({ runInlineTest }) => {
const expectReporterJS = `
class Reporter {
onStdOut(chunk) {
process.stdout.write(chunk);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
console.log('%%%% begin', JSON.stringify(copy));
@ -333,6 +384,9 @@ test('should report api step failure', async ({ runInlineTest }) => {
test('should report test.step', async ({ runInlineTest }) => {
const expectReporterJS = `
class Reporter {
onStdOut(chunk) {
process.stdout.write(chunk);
}
onStepBegin(test, result, step) {
const copy = { ...step, startTime: undefined, duration: undefined };
console.log('%%%% begin', JSON.stringify(copy));

2
tests/playwright-test/retry.spec.ts
View File

@ -68,7 +68,7 @@ test('should retry timeout', async ({ runInlineTest }) => {
await new Promise(f => setTimeout(f, 10000));
});
`
}, { timeout: 100, retries: 2 });
}, { timeout: 1000, retries: 2 });
expect(exitCode).toBe(1);
expect(passed).toBe(0);
expect(failed).toBe(1);

1
tests/playwright-test/test-output-dir.spec.ts
View File

@ -88,6 +88,7 @@ test('should include the project name', async ({ runInlineTest }) => {
'helper.ts': `
export const test = pwt.test.extend({
auto: [ async ({}, run, testInfo) => {
testInfo.snapshotSuffix = '';
await run();
}, { auto: true } ]
});

58
types/test.d.ts vendored
View File

@ -2297,6 +2297,35 @@ export interface PlaywrightWorkerOptions {
* [fixtures.channel](https://playwright.dev/docs/api/class-fixtures#fixtures-channel) take priority over this.
*/
launchOptions: LaunchOptions;
/**
* Whether to automatically capture a screenshot after each test. Defaults to `'off'`.
* - `'off'`: Do not capture screenshots.
* - `'on'`: Capture screenshot after each test.
* - `'only-on-failure'`: Capture screenshot after each test failure.
*
* Learn more about [automatic screenshots](https://playwright.dev/docs/test-configuration#automatic-screenshots).
*/
screenshot: 'off' | 'on' | 'only-on-failure';
/**
* Whether to record a trace for each test. Defaults to `'off'`.
* - `'off'`: Do not record a trace.
* - `'on'`: Record a trace for each test.
* - `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs.
* - `'on-first-retry'`: Record a trace only when retrying a test for the first time.
*
* Learn more about [recording trace](https://playwright.dev/docs/test-configuration#record-test-trace).
*/
trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
/**
* Whether to record video for each test. Defaults to `'off'`.
* - `'off'`: Do not record video.
* - `'on'`: Record video for each test.
* - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
* - `'on-first-retry'`: Record video only when retrying a test for the first time.
*
* Learn more about [recording video](https://playwright.dev/docs/test-configuration#record-video).
*/
video: VideoMode | { mode: VideoMode, size: ViewportSize };
}
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video';
@ -2390,35 +2419,6 @@ export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' |
*
*/
export interface PlaywrightTestOptions {
/**
* Whether to automatically capture a screenshot after each test. Defaults to `'off'`.
* - `'off'`: Do not capture screenshots.
* - `'on'`: Capture screenshot after each test.
* - `'only-on-failure'`: Capture screenshot after each test failure.
*
* Learn more about [automatic screenshots](https://playwright.dev/docs/test-configuration#automatic-screenshots).
*/
screenshot: 'off' | 'on' | 'only-on-failure';
/**
* Whether to record a trace for each test. Defaults to `'off'`.
* - `'off'`: Do not record a trace.
* - `'on'`: Record a trace for each test.
* - `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs.
* - `'on-first-retry'`: Record a trace only when retrying a test for the first time.
*
* Learn more about [recording trace](https://playwright.dev/docs/test-configuration#record-test-trace).
*/
trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
/**
* Whether to record video for each test. Defaults to `'off'`.
* - `'off'`: Do not record video.
* - `'on'`: Record video for each test.
* - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
* - `'on-first-retry'`: Record video only when retrying a test for the first time.
*
* Learn more about [recording video](https://playwright.dev/docs/test-configuration#record-video).
*/
video: VideoMode | { mode: VideoMode, size: ViewportSize };
/**
* Whether to automatically download all the attachments. Defaults to `false` where all the downloads are canceled.
*/

6
utils/generate_types/overrides-test.d.ts vendored
View File

@ -284,14 +284,14 @@ export interface PlaywrightWorkerOptions {
headless: boolean | undefined;
channel: BrowserChannel | undefined;
launchOptions: LaunchOptions;
screenshot: 'off' | 'on' | 'only-on-failure';
trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
video: VideoMode | { mode: VideoMode, size: ViewportSize };
}
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video';
export interface PlaywrightTestOptions {
screenshot: 'off' | 'on' | 'only-on-failure';
trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
video: VideoMode | { mode: VideoMode, size: ViewportSize };
acceptDownloads: boolean | undefined;
bypassCSP: boolean | undefined;
colorScheme: ColorScheme | undefined;