root
/
playwright
mirror of https://github.com/microsoft/playwright.git
1
0
Fork 0
Code Issues Actions 1 Packages Projects Releases Wiki Activity

feat(trace mode): add on-first-failure mode for traces (#29647) 

Implements the changes suggested in #29531
This commit is contained in:
Stevan Freeborn 2024-02-28 16:39:18 -06:00 committed by GitHub
parent d48aadac7e
commit 52b803ecf5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 138 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/src/test-api/class-testoptions.md
View File

@ -546,8 +546,8 @@ export default defineConfig({
## property: TestOptions.trace
* since: v1.10
- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry">>
- `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries">> Trace recording mode.
- type: <[Object]|[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"retain-on-first-failure">>
- `mode` <[TraceMode]<"off"|"on"|"retain-on-failure"|"on-first-retry"|"on-all-retries"|"retain-on-first-failure">> Trace recording mode.
- `attachments` ?<[boolean]> Whether to include test attachments. Defaults to true. Optional.
- `screenshots` ?<[boolean]> Whether to capture screenshots during tracing. Screenshots are used to build a timeline preview. Defaults to true. Optional.
- `snapshots` ?<[boolean]> Whether to capture DOM snapshot on every action. Defaults to true. Optional.
@ -559,6 +559,7 @@ Whether to record trace for each test. Defaults to `'off'`.
* `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs.
* `'on-first-retry'`: Record trace only when retrying a test for the first time.
* `'on-all-retries'`: Record traces only when retrying for all retries.
* `'retain-on-first-failure'`: Record traces only when the test fails for the first time.
For more control, pass an object that specifies `mode` and trace features to enable.

2
packages/playwright/src/program.ts
View File

@ -290,7 +290,7 @@ function resolveReporter(id: string) {
return require.resolve(id, { paths: [process.cwd()] });
}
const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure'];
const kTraceModes: TraceMode[] = ['on', 'off', 'on-first-retry', 'on-all-retries', 'retain-on-failure', 'retain-on-first-failure'];
const testOptions: [string, string][] = [
['--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`],

30
packages/playwright/src/worker/testTracing.ts
View File

@ -48,8 +48,31 @@ export class TestTracing {
this._tracesDir = path.join(this._artifactsDir, 'traces');
}
private _shouldCaptureTrace() {
if (process.env.PW_TEST_DISABLE_TRACING)
return false;
if (this._options?.mode === 'on')
return true;
if (this._options?.mode === 'retain-on-failure')
return true;
if (this._options?.mode === 'on-first-retry' && this._testInfo.retry === 1)
return true;
if (this._options?.mode === 'on-all-retries' && this._testInfo.retry > 0)
return true;
if (this._options?.mode === 'retain-on-first-failure' && this._testInfo.retry === 0)
return true;
return false;
}
async startIfNeeded(value: TraceFixtureValue) {
const defaultTraceOptions: TraceOptions = { screenshots: true, snapshots: true, sources: true, attachments: true, _live: false, mode: 'off' };
if (!value) {
this._options = defaultTraceOptions;
} else if (typeof value === 'string') {
@ -59,9 +82,7 @@ export class TestTracing {
this._options = { ...defaultTraceOptions, ...value, mode: (mode as string) === 'retry-with-trace' ? 'on-first-retry' : mode };
}
let shouldCaptureTrace = this._options.mode === 'on' || this._options.mode === 'retain-on-failure' || (this._options.mode === 'on-first-retry' && this._testInfo.retry === 1) || (this._options.mode === 'on-all-retries' && this._testInfo.retry > 0);
shouldCaptureTrace = shouldCaptureTrace && !process.env.PW_TEST_DISABLE_TRACING;
if (!shouldCaptureTrace) {
if (!this._shouldCaptureTrace()) {
this._options = undefined;
return;
}
@ -110,7 +131,8 @@ export class TestTracing {
return;
const testFailed = this._testInfo.status !== this._testInfo.expectedStatus;
const shouldAbandonTrace = !testFailed && this._options.mode === 'retain-on-failure';
const shouldAbandonTrace = !testFailed && (this._options.mode === 'retain-on-failure' || this._options.mode === 'retain-on-first-failure');
if (shouldAbandonTrace) {
for (const file of this._temporaryTraceFiles)
await fs.promises.unlink(file).catch(() => {});

6
packages/playwright/types/test.d.ts vendored
View File

@ -5586,6 +5586,7 @@ export interface PlaywrightWorkerOptions {
* - `'retain-on-failure'`: Record trace for each test, but remove all traces from successful test runs.
* - `'on-first-retry'`: Record trace only when retrying a test for the first time.
* - `'on-all-retries'`: Record traces only when retrying for all retries.
* - `'retain-on-first-failure'`: Record traces only when the test fails for the first time.
*
* For more control, pass an object that specifies `mode` and trace features to enable.
*
@ -5636,7 +5637,7 @@ export interface PlaywrightWorkerOptions {
}
export type ScreenshotMode = 'off' | 'on' | 'only-on-failure';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
/**
@ -7099,7 +7100,8 @@ type MergedExpect<List> = Expect<MergedExpectMatchers<List>>;
export function mergeExpects<List extends any[]>(...expects: List): MergedExpect<List>;
// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459
export {};
export { };
/**

2
tests/page/pageTestApi.ts
View File

@ -27,7 +27,7 @@ export type PageWorkerFixtures = {
headless: boolean;
channel: string;
screenshot: ScreenshotMode | { mode: ScreenshotMode } & Pick<PageScreenshotOptions, 'fullPage' | 'omitBackground'>;
trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | /** deprecated */ 'retry-with-trace';
trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'retain-on-first-failure' | 'on-all-retries' | /** deprecated */ 'retry-with-trace';
video: VideoMode | { mode: VideoMode, size: ViewportSize };
browserName: 'chromium' | 'firefox' | 'webkit';
browserVersion: string;

25
tests/playwright-test/playwright.artifacts.spec.ts
View File

@ -338,6 +338,31 @@ test('should work with trace: on-all-retries', async ({ runInlineTest }, testInf
]);
});
test('should work with trace: retain-on-first-failure', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
...testFiles,
'playwright.config.ts': `
module.exports = { use: { trace: 'retain-on-first-failure' } };
`,
}, { workers: 1, retries: 2 });
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(5);
expect(result.failed).toBe(5);
expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
'artifacts-failing',
' trace.zip',
'artifacts-own-context-failing',
' trace.zip',
'artifacts-persistent-failing',
' trace.zip',
'artifacts-shared-shared-failing',
' trace.zip',
'artifacts-two-contexts-failing',
' trace.zip',
]);
});
test('should take screenshot when page is closed in afterEach', async ({ runInlineTest }, testInfo) => {
const result = await runInlineTest({
'playwright.config.ts': `

77
tests/playwright-test/playwright.trace.spec.ts
View File

@ -133,7 +133,6 @@ test('should record api trace', async ({ runInlineTest, server }, testInfo) => {
]);
});
test('should not throw with trace: on-first-retry and two retries in the same worker', async ({ runInlineTest }, testInfo) => {
const files = {};
for (let i = 0; i < 6; i++) {
@ -402,7 +401,7 @@ test('should respect PW_TEST_DISABLE_TRACING', async ({ runInlineTest }, testInf
expect(fs.existsSync(testInfo.outputPath('test-results', 'a-test-1', 'trace.zip'))).toBe(false);
});
for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries']) {
for (const mode of ['off', 'retain-on-failure', 'on-first-retry', 'on-all-retries', 'retain-on-first-failure']) {
test(`trace:${mode} should not create trace zip artifact if page test passed`, async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
@ -1034,3 +1033,77 @@ test('should attribute worker fixture teardown to the right test', async ({ runI
' step in foo teardown',
]);
});
test('trace:retain-on-first-failure should create trace but only on first failure', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('fail', async ({ page }) => {
await page.goto('about:blank');
expect(true).toBe(false);
});
`,
}, { trace: 'retain-on-first-failure', retries: 1 });
const retryTracePath = test.info().outputPath('test-results', 'a-fail-retry1', 'trace.zip');
const retryTraceExists = fs.existsSync(retryTracePath);
expect(retryTraceExists).toBe(false);
const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip');
const trace = await parseTrace(tracePath);
expect(trace.apiNames).toContain('page.goto');
expect(result.failed).toBe(1);
});
test('trace:retain-on-first-failure should create trace if context is closed before failure in the test', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('fail', async ({ page, context }) => {
await page.goto('about:blank');
await context.close();
expect(1).toBe(2);
});
`,
}, { trace: 'retain-on-first-failure' });
const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip');
const trace = await parseTrace(tracePath);
expect(trace.apiNames).toContain('page.goto');
expect(result.failed).toBe(1);
});
test('trace:retain-on-first-failure should create trace if context is closed before failure in afterEach', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('fail', async ({ page, context }) => {
});
test.afterEach(async ({ page, context }) => {
await page.goto('about:blank');
await context.close();
expect(1).toBe(2);
});
`,
}, { trace: 'retain-on-first-failure' });
const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip');
const trace = await parseTrace(tracePath);
expect(trace.apiNames).toContain('page.goto');
expect(result.failed).toBe(1);
});
test('trace:retain-on-first-failure should create trace if request context is disposed before failure', async ({ runInlineTest, server }) => {
const result = await runInlineTest({
'a.spec.ts': `
import { test, expect } from '@playwright/test';
test('fail', async ({ request }) => {
expect(await request.get('${server.EMPTY_PAGE}')).toBeOK();
await request.dispose();
expect(1).toBe(2);
});
`,
}, { trace: 'retain-on-first-failure' });
const tracePath = test.info().outputPath('test-results', 'a-fail', 'trace.zip');
const trace = await parseTrace(tracePath);
expect(trace.apiNames).toContain('apiRequestContext.get');
expect(result.failed).toBe(1);
});

5
utils/generate_types/overrides-test.d.ts vendored
View File

@ -248,7 +248,7 @@ export interface PlaywrightWorkerOptions {
}
export type ScreenshotMode = 'off' | 'on' | 'only-on-failure';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries';
export type TraceMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | 'on-all-retries' | 'retain-on-first-failure';
export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry';
export interface PlaywrightTestOptions {
@ -484,4 +484,5 @@ type MergedExpect<List> = Expect<MergedExpectMatchers<List>>;
export function mergeExpects<List extends any[]>(...expects: List): MergedExpect<List>;
// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459
export {};
export { };