root
/
playwright
mirror of https://github.com/microsoft/playwright.git
1
0
Fork 0
Code Issues Actions 2 Packages Projects Releases Wiki Activity

feat(blob): zip .jsonl report files (#23720) 

For linux tests without tracing blob-report-1.zip takes 19M, while
unpacked size is 228 MB. That size is counted for GitHub artifact
billing:

<img width="434" alt="image"
src="https://github.com/microsoft/playwright/assets/9798949/5bc32511-6686-4581-a348-acb6a54cd99b">

We zip individual .jsonl reports so that they still have unique names
and can be easily uploaded into the same artifacts directory without
name collisions.
This commit is contained in:
Yury Semikhatsky 2023-06-14 17:10:39 -07:00 committed by GitHub
parent 60de8308a8
commit 77d322028c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 49 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
packages/playwright-test/src/reporters/blob.ts
View File

@ -16,14 +16,15 @@
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { calculateSha1, createGuid } from 'playwright-core/lib/utils'; import { ManualPromise, calculateSha1, createGuid } from 'playwright-core/lib/utils';
import { mime } from 'playwright-core/lib/utilsBundle'; import { mime } from 'playwright-core/lib/utilsBundle';
import { Readable } from 'stream'; import { Readable } from 'stream';
import type { EventEmitter } from 'events';
import type { FullConfig, FullResult, TestResult } from '../../types/testReporter'; import type { FullConfig, FullResult, TestResult } from '../../types/testReporter';
import type { Suite } from '../common/test'; import type { Suite } from '../common/test';
import type { JsonAttachment, JsonEvent } from '../isomorphic/teleReceiver'; import type { JsonAttachment, JsonEvent } from '../isomorphic/teleReceiver';
import { TeleReporterEmitter } from './teleEmitter'; import { TeleReporterEmitter } from './teleEmitter';
import { yazl } from 'playwright-core/lib/zipBundle';
type BlobReporterOptions = { type BlobReporterOptions = {
configDir: string; configDir: string;
@ -41,7 +42,7 @@ export class BlobReporter extends TeleReporterEmitter {
private _copyFilePromises = new Set<Promise<void>>(); private _copyFilePromises = new Set<Promise<void>>();
private _outputDir!: string; private _outputDir!: string;
private _reportFile!: string; private _reportName!: string;
constructor(options: BlobReporterOptions) { constructor(options: BlobReporterOptions) {
super(message => this._messages.push(message), false); super(message => this._messages.push(message), false);
@ -63,7 +64,7 @@ export class BlobReporter extends TeleReporterEmitter {
override onBegin(config: FullConfig<{}, {}>, suite: Suite): void { override onBegin(config: FullConfig<{}, {}>, suite: Suite): void {
this._outputDir = path.resolve(this._options.configDir, this._options.outputDir || 'blob-report'); this._outputDir = path.resolve(this._options.configDir, this._options.outputDir || 'blob-report');
fs.mkdirSync(path.join(this._outputDir, 'resources'), { recursive: true }); fs.mkdirSync(path.join(this._outputDir, 'resources'), { recursive: true });
this._reportFile = this._computeOutputFileName(config); this._reportName = this._computeReportName(config);
super.onBegin(config, suite); super.onBegin(config, suite);
} }
@ -71,10 +72,21 @@ export class BlobReporter extends TeleReporterEmitter {
await super.onEnd(result); await super.onEnd(result);
const lines = this._messages.map(m => JSON.stringify(m) + '\n'); const lines = this._messages.map(m => JSON.stringify(m) + '\n');
const content = Readable.from(lines); const content = Readable.from(lines);
const zipFile = new yazl.ZipFile();
const zipFinishPromise = new ManualPromise<undefined>();
(zipFile as any as EventEmitter).on('error', error => zipFinishPromise.reject(error));
const zipFileName = path.join(this._outputDir, this._reportName + '.zip');
zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => {
zipFinishPromise.resolve(undefined);
});
zipFile.addReadStream(content, this._reportName + '.jsonl');
zipFile.end();
await Promise.all([ await Promise.all([
...this._copyFilePromises, ...this._copyFilePromises,
// Requires Node v14.18.0+ // Requires Node v14.18.0+
fs.promises.writeFile(this._reportFile, content as any).catch(e => console.error(`Failed to write report ${this._reportFile}: ${e}`)) zipFinishPromise.catch(e => console.error(`Failed to write report ${zipFileName}: ${e}`))
]); ]);
} }
@ -94,13 +106,13 @@ export class BlobReporter extends TeleReporterEmitter {
}); });
} }
private _computeOutputFileName(config: FullConfig) { private _computeReportName(config: FullConfig) {
let shardSuffix = ''; let shardSuffix = '';
if (config.shard) { if (config.shard) {
const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0'); const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0');
shardSuffix = `${paddedNumber}-of-${config.shard.total}-`; shardSuffix = `${paddedNumber}-of-${config.shard.total}-`;
} }
return path.join(this._outputDir, `report-${shardSuffix}${createGuid()}.jsonl`); return `report-${shardSuffix}${createGuid()}`;
} }
private _startCopyingFile(from: string, to: string) { private _startCopyingFile(from: string, to: string) {

21
packages/playwright-test/src/reporters/merge.ts
View File

@ -23,6 +23,7 @@ import type { JsonConfig, JsonEvent, JsonProject, JsonSuite, JsonTestResultEnd }
import { TeleReporterReceiver } from '../isomorphic/teleReceiver'; import { TeleReporterReceiver } from '../isomorphic/teleReceiver';
import { createReporters } from '../runner/reporters'; import { createReporters } from '../runner/reporters';
import { Multiplexer } from './multiplexer'; import { Multiplexer } from './multiplexer';
import { ZipFile } from 'playwright-core/lib/utils';
export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], resolvePaths: boolean) { export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[], resolvePaths: boolean) {
const shardFiles = await sortedShardFiles(dir); const shardFiles = await sortedShardFiles(dir);
@ -52,16 +53,30 @@ function patchAttachmentPaths(events: JsonEvent[], resourceDir: string) {
} }
} }
function parseEvents(reportJsonl: string): JsonEvent[] { function parseEvents(reportJsonl: Buffer): JsonEvent[] {
return reportJsonl.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[]; return reportJsonl.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[];
} }
async function extractReportFromZip(file: string): Promise<Buffer> {
const zipFile = new ZipFile(file);
const entryNames = await zipFile.entries();
try {
for (const entryName of entryNames) {
if (entryName.endsWith('.jsonl'))
return await zipFile.read(entryName);
}
} finally {
zipFile.close();
}
throw new Error(`Cannot find *.jsonl file in ${file}`);
}
async function mergeEvents(dir: string, shardReportFiles: string[]) { async function mergeEvents(dir: string, shardReportFiles: string[]) {
const events: JsonEvent[] = []; const events: JsonEvent[] = [];
const beginEvents: JsonEvent[] = []; const beginEvents: JsonEvent[] = [];
const endEvents: JsonEvent[] = []; const endEvents: JsonEvent[] = [];
for (const reportFile of shardReportFiles) { for (const reportFile of shardReportFiles) {
const reportJsonl = await fs.promises.readFile(path.join(dir, reportFile), 'utf8'); const reportJsonl = await extractReportFromZip(path.join(dir, reportFile));
const parsedEvents = parseEvents(reportJsonl); const parsedEvents = parseEvents(reportJsonl);
for (const event of parsedEvents) { for (const event of parsedEvents) {
if (event.method === 'onBegin') if (event.method === 'onBegin')
@ -159,7 +174,7 @@ function mergeEndEvents(endEvents: JsonEvent[]): JsonEvent {
async function sortedShardFiles(dir: string) { async function sortedShardFiles(dir: string) {
const files = await fs.promises.readdir(dir); const files = await fs.promises.readdir(dir);
return files.filter(file => file.endsWith('.jsonl')).sort(); return files.filter(file => file.startsWith('report-') && file.endsWith('.zip')).sort();
} }
class ProjectNamePatcher { class ProjectNamePatcher {

24
tests/playwright-test/reporter-blob.spec.ts
View File

@ -148,7 +148,7 @@ test('should call methods in right order', async ({ runInlineTest, mergeReports
await runInlineTest(files, { shard: `3/3` }); await runInlineTest(files, { shard: `3/3` });
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.jsonl/), expect.stringMatching(/report-3-of-3.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js')] }); const { exitCode, output } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js')] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
const lines = output.split('\n').filter(l => l.trim().length); const lines = output.split('\n').filter(l => l.trim().length);
@ -212,7 +212,7 @@ test('should merge into html', async ({ runInlineTest, mergeReports, showReport,
await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }); await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.jsonl/), expect.stringMatching(/report-2-of-3.*.jsonl/), expect.stringMatching(/report-3-of-3.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -272,7 +272,7 @@ test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports,
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.jsonl/), expect.stringMatching(/report-3-of-3.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -378,7 +378,7 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
await runInlineTest(files, { shard: `${i + 1}/${totalShards}` }); await runInlineTest(files, { shard: `${i + 1}/${totalShards}` });
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.jsonl/), expect.stringMatching(/report-2-of-3.*.jsonl/), expect.stringMatching(/report-3-of-3.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-2-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] }); const { exitCode, output } = await mergeReports(reportDir, { PW_TEST_DEBUG_REPORTERS: '1', PW_TEST_DEBUG_REPORTERS_PRINT_STEPS: '1', PWTEST_TTY_WIDTH: '80' }, { additionalArgs: ['--reporter', 'list'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -449,7 +449,7 @@ test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, p
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.zip/), 'resources']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] }); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -512,7 +512,7 @@ test('generate html with attachment urls', async ({ runInlineTest, mergeReports,
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.zip/), 'resources']);
const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html', '--attachments', 'missing'] }); const { exitCode } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never' }, { additionalArgs: ['--reporter', 'html', '--attachments', 'missing'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -586,7 +586,7 @@ test('resource names should not clash between runs', async ({ runInlineTest, sho
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.jsonl/), expect.stringMatching(/report-2-of-2.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.zip/), expect.stringMatching(/report-2-of-2.*.zip/), 'resources']);
const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -661,7 +661,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html,line'] }); const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html,line'] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -722,7 +722,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.jsonl/), expect.stringMatching(/report-2-of-2.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-2.*.zip/), expect.stringMatching(/report-2-of-2.*.zip/), 'resources']);
const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] }); const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
@ -737,7 +737,7 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep
// Check report presence. // Check report presence.
const mergedBlobReportFiles = await fs.promises.readdir(test.info().outputPath('merged/merged-blob')); const mergedBlobReportFiles = await fs.promises.readdir(test.info().outputPath('merged/merged-blob'));
expect(mergedBlobReportFiles).toEqual([expect.stringMatching(/report.*.jsonl/), 'resources']); expect(mergedBlobReportFiles).toEqual([expect.stringMatching(/report.*.zip/), 'resources']);
}); });
test('onError in the report', async ({ runInlineTest, mergeReports, showReport, page }) => { test('onError in the report', async ({ runInlineTest, mergeReports, showReport, page }) => {
@ -868,7 +868,7 @@ test('preserve config fields', async ({ runInlineTest, mergeReports }) => {
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.jsonl/), expect.stringMatching(/report-3-of-3.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-1-of-3.*.zip/), expect.stringMatching(/report-3-of-3.*.zip/), 'resources']);
const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js'), '-c', test.info().outputPath('merge.config.ts')] }); const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', test.info().outputPath('echo-reporter.js'), '-c', test.info().outputPath('merge.config.ts')] });
expect(exitCode).toBe(0); expect(exitCode).toBe(0);
const json = JSON.parse(fs.readFileSync(test.info().outputPath('config.json')).toString()); const json = JSON.parse(fs.readFileSync(test.info().outputPath('config.json')).toString());
@ -1025,7 +1025,7 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show
await runInlineTest(files); await runInlineTest(files);
const reportFiles = await fs.promises.readdir(reportDir); const reportFiles = await fs.promises.readdir(reportDir);
reportFiles.sort(); reportFiles.sort();
expect(reportFiles).toEqual([expect.stringMatching(/report-.*.jsonl/), 'resources']); expect(reportFiles).toEqual([expect.stringMatching(/report-.*.zip/), 'resources']);
// Run merger in a different directory to make sure relative paths will not be resolved // Run merger in a different directory to make sure relative paths will not be resolved
// relative to the current directory. // relative to the current directory.
const mergeCwd = test.info().outputPath('foo'); const mergeCwd = test.info().outputPath('foo');