feat(test-runner): document tagging, implement grep-invert (#7227)

docs/src/test-annotations.md

@@ -3,35 +3,77 @@ id: test-annotations
 title: "Annotations"
 ---
 
-Sadly, tests do not always pass. Luckily, Playwright Test supports test annotations to deal with failures, flakiness and tests that are not yet ready.
+<!-- TOC -->
+
+## Annotations
+
+Playwright Test supports test annotations to deal with failures, flakiness, skip, focus and tag tests:
+- `skip` marks the test as irrelevant. Playwright Test does not run such a test. Use this annotation when the test is not applicable in some configuration.
+- `fail` marks the test as failing. Playwright Test will run this test and ensure it does indeed fail. If the test does not fail, Playwright Test will complain.
+- `fixme` marks the test as failing. Playwright Test will not run this test, as opposite to the `fail` annotation. Use `fixme` when running the test is slow or crashy.
+- `slow` marks the test as slow and triples the test timeout.
+
+## Focus a test
+
+You can focus some tests. When there are focused tests, only these tests run.
 
 ```js js-flavor=js
-// example.spec.js
-const { test, expect } = require('@playwright/test');
-
-test('some feature', async ({ page, browserName }) => {
-  test.skip(browserName !== 'webkit', 'This feature is iOS-only');
-  // Test goes here.
-});
-
-test('another feature', async ({ page }) => {
-  test.fail(true, 'Broken, need to fix!');
-  // Test goes here.
+test.only('focus this test', async ({ page }) => {
+  // Run only focused tests in the entire project.
+});
+```
+
+```js js-flavor=ts
+test.only('focus this test', async ({ page }) => {
+  // Run only focused tests in the entire project.
+});
+```
+
+## Skip a test
+
+You can skip certain tests based on the condition.
+
+```js js-flavor=js
+test('skip this test', async ({ page, browserName }) => {
+  test.skip(browserName === 'firefox', 'Still working on it');
+});
+```
+
+```js js-flavor=ts
+test('skip this test', async ({ page, browserName }) => {
+  test.skip(browserName === 'firefox', 'Still working on it');
+});
+```
+
+## Group tests
+
+You can group tests to give them a logical name or to scope before/after hooks to the group.
+
+```js js-flavor=js
+const { test, expect } = require('@playwright/test');
+
+test.describe('two tests', () => {
+  test('one', async ({ page }) => {
+    // ...
+  });
+
+  test('two', async ({ page }) => {
+    // ...
+  });
 });
 ```
 
 ```js js-flavor=ts
-// example.spec.ts
 import { test, expect } from '@playwright/test';
 
-test('some feature', async ({ page, browserName }) => {
-  test.skip(browserName !== 'webkit', 'This feature is iOS-only');
-  // Test goes here.
-});
+test.describe('two tests', () => {
+  test('one', async ({ page }) => {
+    // ...
+  });
 
-test('broken feature', async ({ page }) => {
-  test.fail();
-  // Test goes here.
+  test('two', async ({ page }) => {
+    // ...
+  });
 });
 ```
 
@@ -42,3 +84,43 @@ Available annotations:
 - `fail` marks the test as failing. Playwright Test will run this test and ensure it does indeed fail. If the test does not fail, Playwright Test will complain.
 - `fixme` marks the test as failing. Playwright Test will not run this test, as opposite to the `fail` annotation. Use `fixme` when running the test is slow or crashy.
 - `slow` marks the test as slow and triples the test timeout.
+
+## Tag tests
+
+Sometimes you want to tag your tests as `@fast` or `@slow` and only run the tests that have the certain tag. We recommend that you use the `--grep` and `--grep-invert` command line flags for that:
+
+```js js-flavor=js
+const { test, expect } = require('@playwright/test');
+
+test('Test login page @fast', async ({ page }) => {
+  // ...
+});
+
+test('Test full report @slow', async ({ page }) => {
+  // ...
+});
+```
+
+```js js-flavor=ts
+import { test, expect } from '@playwright/test';
+
+test('Test login page @fast', async ({ page }) => {
+  // ...
+});
+
+test('Test full report @slow', async ({ page }) => {
+  // ...
+});
+```
+
+You will then be able to run only that test:
+
+```bash
+npx playwright test --grep @fast
+```
+
+Or if you want the opposite, you can skip the tests with a certain tag:
+
+```bash
+npx playwright test --grep-invert @slow
+```

src/test/cli.ts

@@ -44,6 +44,7 @@ export function addTestCommand(program: commander.CommanderStatic) {
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional "${tsConfig}"/"${jsConfig}"`);
   command.option('--forbid-only', `Fail if test.only is called (default: false)`);
   command.option('-g, --grep <grep>', `Only run tests matching this regular expression (default: ".*")`);
+  command.option('-gv, --grep-invert <grep>', `Only run tests that do not match this regular expression`);
   command.option('--global-timeout <timeout>', `Maximum time this test suite can run in milliseconds (default: unlimited)`);
   command.option('-j, --workers <workers>', `Number of concurrent workers, use 1 to run in a single worker (default: number of CPU cores / 2)`);
   command.option('--list', `Collect all the tests and report them, but do not run`);
@@ -147,6 +148,7 @@ function overridesFromOptions(options: { [key: string]: any }): Config {
     forbidOnly: options.forbidOnly ? true : undefined,
     globalTimeout: isDebuggerAttached ? 0 : (options.globalTimeout ? parseInt(options.globalTimeout, 10) : undefined),
     grep: options.grep ? forceRegExp(options.grep) : undefined,
+    grepInvert: options.grepInvert ? forceRegExp(options.grepInvert) : undefined,
     maxFailures: options.x ? 1 : (options.maxFailures ? parseInt(options.maxFailures, 10) : undefined),
     outputDir: options.output ? path.resolve(process.cwd(), options.output) : undefined,
     quiet: options.quiet ? options.quiet : undefined,

src/test/loader.ts

@@ -96,6 +96,7 @@ export class Loader {
     this._fullConfig.globalTeardown = takeFirst(this._configOverrides.globalTeardown, this._config.globalTeardown, baseFullConfig.globalTeardown);
     this._fullConfig.globalTimeout = takeFirst(this._configOverrides.globalTimeout, this._configOverrides.globalTimeout, this._config.globalTimeout, baseFullConfig.globalTimeout);
     this._fullConfig.grep = takeFirst(this._configOverrides.grep, this._config.grep, baseFullConfig.grep);
+    this._fullConfig.grepInvert = takeFirst(this._configOverrides.grepInvert, this._config.grepInvert, baseFullConfig.grepInvert);
     this._fullConfig.maxFailures = takeFirst(this._configOverrides.maxFailures, this._config.maxFailures, baseFullConfig.maxFailures);
     this._fullConfig.preserveOutput = takeFirst<PreserveOutput>(this._configOverrides.preserveOutput, this._config.preserveOutput, baseFullConfig.preserveOutput);
     this._fullConfig.reporter = takeFirst(toReporters(this._configOverrides.reporter), toReporters(this._config.reporter), baseFullConfig.reporter);
@@ -262,6 +263,17 @@ function validateConfig(config: Config) {
     }
   }
 
+  if ('grepInvert' in config && config.grepInvert !== undefined) {
+    if (Array.isArray(config.grepInvert)) {
+      config.grepInvert.forEach((item, index) => {
+        if (!isRegExp(item))
+          throw new Error(`config.grepInvert[${index}] must be a RegExp`);
+      });
+    } else if (!isRegExp(config.grepInvert)) {
+      throw new Error(`config.grep must be a RegExp`);
+    }
+  }
+
   if ('maxFailures' in config && config.maxFailures !== undefined) {
     if (typeof config.maxFailures !== 'number' || config.maxFailures < 0)
       throw new Error(`config.maxFailures must be a non-negative number`);
@@ -402,6 +414,7 @@ const baseFullConfig: FullConfig = {
   globalTeardown: null,
   globalTimeout: 0,
   grep: /.*/,
+  grepInvert: null,
   maxFailures: 0,
   preserveOutput: 'always',
   projects: [],

src/test/runner.ts

@@ -169,13 +169,17 @@ export class Runner {
 
       const outputDirs = new Set<string>();
       const grepMatcher = createMatcher(config.grep);
+      const grepInvertMatcher = config.grepInvert ? createMatcher(config.grepInvert) : null;
       for (const project of projects) {
         for (const file of files.get(project)!) {
           const fileSuite = fileSuites.get(file);
           if (!fileSuite)
             continue;
           for (const spec of fileSuite._allSpecs()) {
-            if (grepMatcher(spec._testFullTitle(project.config.name)))
+            const fullTitle = spec._testFullTitle(project.config.name);
+            if (grepInvertMatcher?.(fullTitle))
+              continue;
+            if (grepMatcher(fullTitle))
               project.generateTests(spec);
           }
         }

tests/playwright-test/match-grep.spec.ts

@@ -96,3 +96,9 @@ test('should grep by project name', async ({ runInlineTest }) => {
   expect(result.failed).toBe(0);
   expect(result.exitCode).toBe(0);
 });
+
+test('should grep invert test name', async ({ runInlineTest }) => {
+  const result = await runInlineTest(files, { 'grep-invert': 'BB' });
+  expect(result.passed).toBe(6);
+  expect(result.exitCode).toBe(0);
+});

types/test.d.ts

@@ -150,6 +150,11 @@ interface ConfigBase {
   grep?: RegExp | RegExp[];
 
   /**
+   * Filter out tests with a title matching one of the patterns.
+   */
+  grepInvert?: RegExp | RegExp[];
+
+   /**
    * The maximum number of test failures for this test run. After reaching this number,
    * testing will stop and exit with an error. Setting to zero (default) disables this behavior.
    */
@@ -223,6 +228,7 @@ export interface FullConfig {
   globalTeardown: string | null;
   globalTimeout: number;
   grep: RegExp | RegExp[];
+  grepInvert: RegExp | RegExp[] | null;
   maxFailures: number;
   preserveOutput: PreserveOutput;
   projects: FullProject[];