feat(test-runner): export reporter api as `@playwright/test/reporter` (#7692)

packages/common/.npmignore

@@ -22,10 +22,13 @@ lib/**/injected/
 # Include generated types and entrypoint.
 !types/*
 !index.d.ts
+!reporter.d.ts
 # Include main entrypoint.
 !index.js
+!reporter.js
 # Include main entrypoint for ES Modules.
 !index.mjs
+!reporter.mjs
 # Include installer.
 !install.js
 # Include essentials.

packages/installation-tests/installation-tests.sh

@@ -53,6 +53,7 @@ function copy_test_scripts {
   cp "${SCRIPTS_PATH}/driver-client.js" .
   cp "${SCRIPTS_PATH}/sample.spec.js" .
   cp "${SCRIPTS_PATH}/read-json-report.js" .
+  cp "${SCRIPTS_PATH}/playwright-test-types.ts" .
 }
 
 function run_tests {
@@ -107,6 +108,7 @@ function test_screencast {
 
 function test_typescript_types {
   initialize_test "${FUNCNAME[0]}"
+  copy_test_scripts
 
   # install all packages.
   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CORE_TGZ}
@@ -114,6 +116,7 @@ function test_typescript_types {
   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_FIREFOX_TGZ}
   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_WEBKIT_TGZ}
   PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_CHROMIUM_TGZ}
+  PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install ${PLAYWRIGHT_TEST_TGZ}
 
   # typecheck all packages.
   for PKG_NAME in "playwright" \
@@ -126,6 +129,9 @@ function test_typescript_types {
     echo "import { Page } from '${PKG_NAME}';" > "${PKG_NAME}.ts" && npx -p typescript@3.7.5 tsc "${PKG_NAME}.ts"
   done;
 
+  echo "Checking types of @playwright/test"
+  echo npx -p typescript@3.7.5 tsc "playwright-test-types.ts"
+
   echo "${FUNCNAME[0]} success"
 }
 

packages/installation-tests/playwright-test-types.ts

@@ -0,0 +1,57 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from '@playwright/test';
+import { Reporter, Test } from '@playwright/test/reporter';
+
+test.use({ locale: 'en-US' });
+
+test.describe('block', () => {
+  test.beforeAll(async ({ browser }) => {
+  });
+  test.afterAll(async ({ browser }) => {
+  });
+  test.beforeEach(async ({ page }) => {
+  });
+  test.afterEach(async ({ page }) => {
+  });
+  test('should work', async ({ page, browserName }, testInfo) => {
+    test.skip(browserName === 'chromium');
+    await page.click(testInfo.title);
+    testInfo.annotations.push({ type: 'foo' });
+    await page.fill(testInfo.outputPath('foo', 'bar'), testInfo.outputDir);
+  });
+});
+
+const test2 = test.extend<{ foo: string, bar: number }>({
+  foo: '123',
+  bar: async ({ foo }, use) => {
+    await use(parseInt(foo, 10));
+  },
+});
+
+test2('should work 2', async ({ foo, bar }) => {
+  bar += parseInt(foo, 10);
+  expect(bar).toBe(123 * 2);
+});
+
+export class MyReporter implements Reporter {
+  onTestBegin(test: Test) {
+    test.titlePath().slice();
+    if (test.results[0].status === test.expectedStatus)
+      console.log(`Nice test ${test.title} at ${test.location.file}`);
+  }
+}

packages/playwright-test/reporter.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './types/testReporter';

packages/playwright-test/reporter.js

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// We only export types in reporter.d.ts.

packages/playwright-test/reporter.mjs

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// We only export types in reporter.d.ts.

src/test/dispatcher.ts

@@ -18,7 +18,7 @@ import child_process from 'child_process';
 import path from 'path';
 import { EventEmitter } from 'events';
 import { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams } from './ipc';
-import type { TestResult, Reporter, TestStatus } from './reporter';
+import type { TestResult, Reporter, TestStatus } from '../../types/testReporter';
 import { Suite, Test } from './test';
 import { Loader } from './loader';
 

src/test/ipc.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import type { TestError } from './reporter';
+import type { TestError } from '../../types/testReporter';
 import type { Config, TestStatus } from './types';
 
 export type SerializedLoaderData = {

src/test/loader.ts

@@ -23,7 +23,7 @@ import { SerializedLoaderData } from './ipc';
 import * as path from 'path';
 import * as url from 'url';
 import { ProjectImpl } from './project';
-import { Reporter } from './reporter';
+import { Reporter } from '../../types/testReporter';
 import { LaunchConfig } from '../../types/test';
 
 export class Loader {

src/test/reporter.ts

@@ -1,66 +0,0 @@
-/**
- * Copyright Microsoft Corporation. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type { FullConfig, TestStatus, TestError } from './types';
-export type { FullConfig, TestStatus, TestError } from './types';
-
-export interface Location {
-  file: string;
-  line: number;
-  column: number;
-}
-export interface Suite {
-  title: string;
-  location: Location;
-  suites: Suite[];
-  tests: Test[];
-  titlePath(): string[];
-  allTests(): Test[];
-}
-export interface Test {
-  title: string;
-  location: Location;
-  results: TestResult[];
-  expectedStatus: TestStatus;
-  timeout: number;
-  annotations: { type: string, description?: string }[];
-  retries: number;
-  titlePath(): string[];
-  status(): 'skipped' | 'expected' | 'unexpected' | 'flaky';
-  ok(): boolean;
-}
-export interface TestResult {
-  retry: number;
-  workerIndex: number,
-  duration: number;
-  status?: TestStatus;
-  error?: TestError;
-  attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
-  stdout: (string | Buffer)[];
-  stderr: (string | Buffer)[];
-}
-export interface FullResult {
-  status: 'passed' | 'failed' | 'timedout' | 'interrupted';
-}
-export interface Reporter {
-  onBegin?(config: FullConfig, suite: Suite): void;
-  onTestBegin?(test: Test): void;
-  onStdOut?(chunk: string | Buffer, test?: Test): void;
-  onStdErr?(chunk: string | Buffer, test?: Test): void;
-  onTestEnd?(test: Test, result: TestResult): void;
-  onError?(error: TestError): void;
-  onEnd?(result: FullResult): void | Promise<void>;
-}

src/test/reporters/base.ts

@@ -21,7 +21,7 @@ import fs from 'fs';
 import milliseconds from 'ms';
 import path from 'path';
 import StackUtils from 'stack-utils';
-import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter';
+import { FullConfig, TestStatus, Test, Suite, TestResult, TestError, Reporter, FullResult } from '../../../types/testReporter';
 
 const stackUtils = new StackUtils();
 

src/test/reporters/dot.ts

@@ -16,7 +16,7 @@
 
 import colors from 'colors/safe';
 import { BaseReporter } from './base';
-import { FullResult, Test, TestResult } from '../reporter';
+import { FullResult, Test, TestResult } from '../../../types/testReporter';
 
 class DotReporter extends BaseReporter {
   private _counter = 0;

src/test/reporters/empty.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Reporter } from '../reporter';
+import { Reporter } from '../../../types/testReporter';
 
 class EmptyReporter implements Reporter {
 }

src/test/reporters/json.ts

@@ -16,7 +16,7 @@
 
 import fs from 'fs';
 import path from 'path';
-import { FullConfig, Test, Suite, TestResult, TestError, FullResult, TestStatus, Location, Reporter } from '../reporter';
+import { FullConfig, Test, Suite, TestResult, TestError, FullResult, TestStatus, Location, Reporter } from '../../../types/testReporter';
 
 export interface JSONReport {
   config: Omit<FullConfig, 'projects'> & {

src/test/reporters/junit.ts

@@ -16,7 +16,7 @@
 
 import fs from 'fs';
 import path from 'path';
-import { FullConfig, FullResult, Reporter, Suite, Test } from '../reporter';
+import { FullConfig, FullResult, Reporter, Suite, Test } from '../../../types/testReporter';
 import { monotonicTime } from '../util';
 import { formatFailure, formatTestTitle, stripAscii } from './base';
 

src/test/reporters/line.ts

@@ -16,7 +16,7 @@
 
 import colors from 'colors/safe';
 import { BaseReporter, formatFailure, formatTestTitle } from './base';
-import { FullConfig, Test, Suite, TestResult, FullResult } from '../reporter';
+import { FullConfig, Test, Suite, TestResult, FullResult } from '../../../types/testReporter';
 
 class LineReporter extends BaseReporter {
   private _total = 0;

src/test/reporters/list.ts

@@ -19,7 +19,7 @@ import colors from 'colors/safe';
 // @ts-ignore
 import milliseconds from 'ms';
 import { BaseReporter, formatTestTitle } from './base';
-import { FullConfig, FullResult, Suite, Test, TestResult } from '../reporter';
+import { FullConfig, FullResult, Suite, Test, TestResult } from '../../../types/testReporter';
 
 // Allow it in the Visual Studio Code Terminal and the new Windows Terminal
 const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION;

src/test/reporters/multiplexer.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { FullConfig, Suite, Test, TestError, TestResult, Reporter, FullResult } from '../reporter';
+import { FullConfig, Suite, Test, TestError, TestResult, Reporter, FullResult } from '../../../types/testReporter';
 
 export class Multiplexer implements Reporter {
   private _reporters: Reporter[];

src/test/runner.ts

@@ -24,7 +24,7 @@ import { Dispatcher } from './dispatcher';
 import { createMatcher, FilePatternFilter, monotonicTime, raceAgainstDeadline } from './util';
 import { Test, Suite } from './test';
 import { Loader } from './loader';
-import { Reporter } from './reporter';
+import { Reporter } from '../../types/testReporter';
 import { Multiplexer } from './reporters/multiplexer';
 import DotReporter from './reporters/dot';
 import LineReporter from './reporters/line';

src/test/test.ts

@@ -15,7 +15,7 @@
  */
 
 import type { FixturePool } from './fixtures';
-import * as reporterTypes from './reporter';
+import * as reporterTypes from '../../types/testReporter';
 import type { TestTypeImpl } from './testType';
 import { Annotations, Location } from './types';
 

src/test/types.ts

@@ -15,9 +15,9 @@
  */
 
 import type { Fixtures } from '../../types/test';
-import type { Location } from './reporter';
+import type { Location } from '../../types/testReporter';
 export * from '../../types/test';
-export { Location } from './reporter';
+export { Location } from '../../types/testReporter';
 
 export type FixturesWithLocation = {
   fixtures: Fixtures;

types/testReporter.d.ts

@@ -0,0 +1,259 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { FullConfig, TestStatus, TestError } from './test';
+export type { FullConfig, TestStatus, TestError } from './test';
+
+/**
+ * Test or Suite location where it was defined.
+ */
+export interface Location {
+  /**
+   * Path to the file.
+   */
+  file: string;
+
+  /**
+   * Line number in the file.
+   */
+  line: number;
+
+  /**
+   * Column number in the file.
+   */
+  column: number;
+}
+
+/**
+ * A group of tests. All tests are reported in the following hierarchy:
+ * - Root suite
+ *   - Project suite #1 (for each project)
+ *     - File suite #1 (for each file in the project)
+ *       - Suites for any describe() calls
+ *         - Test #1 defined in the file or describe() group
+ *         - Test #2
+ *         ... < more tests >
+ *     - File suite #2
+ *     ... < more file suites >
+ *   - Second project suite
+ *   ... < more project suites >
+ */
+export interface Suite {
+  /**
+   * Suite title:
+   *   - Empty for root suite.
+   *   - Project name for project suite.
+   *   - File path for file suite.
+   *   - Title passed to describe() for describe suites
+   */
+  title: string;
+
+  /**
+   * Location where the suite is defined.
+   */
+  location: Location;
+
+  /**
+   * Child suites.
+   */
+  suites: Suite[];
+
+  /**
+   * Tests in the suite. Note that only tests defined directly in this suite
+   * are in the list. Any tests defined in nested describe() groups are listed
+   * in the child `suites`.
+   */
+  tests: Test[];
+
+  /**
+   * A list of titles from the root down to this suite.
+   */
+  titlePath(): string[];
+
+  /**
+   * Returns the list of all tests in this suite and its descendants.
+   */
+  allTests(): Test[];
+}
+
+/**
+ * A test, corresponds to test() call in a test file. When a single test() is
+ * running in multiple projects (or repeated multiple times), it will have multiple
+ * `Test` objects in corresponding projects' suites.
+ */
+export interface Test {
+  /**
+   * Test title as passed to the test() call.
+   */
+  title: string;
+
+  /**
+   * Location where the test is defined.
+   */
+  location: Location;
+
+  /**
+   * A list of titles from the root down to this test.
+   */
+  titlePath(): string[];
+
+  /**
+   * Expected status.
+   *   - Tests marked as test.skip() or test.fixme() are expected to be 'skipped'.
+   *   - Tests marked as test.fail() are expected to be 'failed'.
+   *   - Other tests are expected to be 'passed'.
+   */
+  expectedStatus: TestStatus;
+
+  /**
+   * The timeout given to the test. Affected by timeout in the configuration file,
+   * and calls to test.setTimeout() or test.slow().
+   */
+  timeout: number;
+
+  /**
+   * Annotations collected for this test. For example, calling
+   * `test.skip(true, 'just because')` will produce an annotation
+   * `{ type: 'skip', description: 'just because' }`.
+   */
+  annotations: { type: string, description?: string }[];
+
+  /**
+   * The maxmium number of retries given to this test in the configuration.
+   */
+  retries: number;
+
+  /**
+   * Results for each run of this test.
+   */
+  results: TestResult[];
+
+  /**
+   * Overall test status.
+   */
+  status(): 'skipped' | 'expected' | 'unexpected' | 'flaky';
+
+  /**
+   * Whether the test is considered running fine.
+   * Non-ok tests fail the test run with non-zero exit code.
+   */
+  ok(): boolean;
+}
+
+/**
+ * A result of a single test run.
+ */
+export interface TestResult {
+  /**
+   * When test is retries multiple times, each retry attempt is given a sequential number.
+   */
+  retry: number;
+
+  /**
+   * Index of the worker where the test was run.
+   */
+  workerIndex: number,
+
+  /**
+   * Running time in milliseconds.
+   */
+  duration: number;
+
+  /**
+   * The status of this test result.
+   */
+  status?: TestStatus;
+
+  /**
+   * An error from this test result, if any.
+   */
+  error?: TestError;
+
+  /**
+   * Any attachments created during the test run.
+   */
+  attachments: { name: string, path?: string, body?: Buffer, contentType: string }[];
+
+  /**
+   * Anything written to the standard output during the test run.
+   */
+  stdout: (string | Buffer)[];
+
+  /**
+   * Anything written to the standard error during the test run.
+   */
+  stderr: (string | Buffer)[];
+}
+
+/**
+ * Result of the full test run.
+ */
+export interface FullResult {
+  /**
+   * Status:
+   *   - 'passed' - everything went as expected.
+   *   - 'failed' - any test has failed.
+   *   - 'timedout' - the global time has been reached.
+   *   - 'interrupted' - interrupted by the user.
+   */
+  status: 'passed' | 'failed' | 'timedout' | 'interrupted';
+}
+
+/**
+ * Test runner notifies reporter about various events during the test run.
+ */
+export interface Reporter {
+  /**
+   * Called once before running tests.
+   * All tests have been already discovered and put into a hierarchy, see `Suite` description.
+   */
+  onBegin?(config: FullConfig, suite: Suite): void;
+
+  /**
+   * Called after a test has been started in the worker process.
+   */
+  onTestBegin?(test: Test): void;
+
+  /**
+   * Called when something has been written to the standard output in the worker process.
+   * When `test` is given, output happened while the test was running.
+   */
+  onStdOut?(chunk: string | Buffer, test?: Test): void;
+
+  /**
+   * Called when something has been written to the standard error in the worker process.
+   * When `test` is given, output happened while the test was running.
+   */
+  onStdErr?(chunk: string | Buffer, test?: Test): void;
+
+  /**
+   * Called after a test has been finished in the worker process.
+   */
+  onTestEnd?(test: Test, result: TestResult): void;
+
+  /**
+   * Called on some global error, for example unhandled expection in the worker process.
+   */
+  onError?(error: TestError): void;
+
+  /**
+   * Called after all tests has been run, or when testing has been interrupted.
+   */
+  onEnd?(result: FullResult): void | Promise<void>;
+}
+
+// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459
+export {};