chore: remove config.groups (#17974)

packages/playwright-test/src/cli.ts

@@ -59,7 +59,6 @@ function addTestCommand(program: Command) {
   command.option('--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`);
   command.option('--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`);
   command.option('--project <project-name...>', `Only run tests from the specified list of projects (default: run all projects)`);
-  command.option('--group <project-group-name>', `Only run tests from the specified project group (default: run all projects from the 'default' group or just all projects if 'default' group is not defined).`);
   command.option('--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`);
   command.option('--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`);
   command.option('-u, --update-snapshots', `Update snapshots with actual results (default: only create missing snapshots)`);
@@ -173,7 +172,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
     testFileFilters,
     testTitleMatcher,
     projectFilter: opts.project || undefined,
-    projectGroup: opts.group,
     passWithNoTests: opts.passWithNoTests,
   });
   await stopProfiling(undefined);

packages/playwright-test/src/loader.ts

@@ -14,23 +14,22 @@
  * limitations under the License.
  */
 
-import { installTransform } from './transform';
-import type { Config, Project, ReporterDescription, FullProjectInternal, FullConfigInternal, Fixtures, FixturesWithLocation } from './types';
-import { getPackageJsonPath, mergeObjects, errorWithFile } from './util';
-import { setCurrentlyLoadingFileSuite } from './globals';
-import { Suite, type TestCase } from './test';
-import type { SerializedLoaderData, WorkerIsolation } from './ipc';
-import * as path from 'path';
-import * as url from 'url';
 import * as fs from 'fs';
 import * as os from 'os';
-import type { BuiltInReporter, ConfigCLIOverrides } from './runner';
+import * as path from 'path';
+import { calculateSha1, isRegExp } from 'playwright-core/lib/utils';
+import * as url from 'url';
 import type { Reporter } from '../types/testReporter';
-import { builtInReporters } from './runner';
-import { isRegExp, calculateSha1, isString, isObject } from 'playwright-core/lib/utils';
-import { serializeError } from './util';
 import { FixturePool, isFixtureOption } from './fixtures';
+import { setCurrentlyLoadingFileSuite } from './globals';
+import type { SerializedLoaderData, WorkerIsolation } from './ipc';
+import type { BuiltInReporter, ConfigCLIOverrides } from './runner';
+import { builtInReporters } from './runner';
+import { Suite, type TestCase } from './test';
 import type { TestTypeImpl } from './testType';
+import { installTransform } from './transform';
+import type { Config, Fixtures, FixturesWithLocation, FullConfigInternal, FullProjectInternal, Project, ReporterDescription } from './types';
+import { errorWithFile, getPackageJsonPath, mergeObjects, serializeError } from './util';
 
 export const defaultTimeout = 30000;
 
@@ -165,10 +164,6 @@ export class Loader {
     this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata);
     this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath));
     this._assignUniqueProjectIds(this._fullConfig.projects);
-
-    // TODO: restore or remove once groups are decided upon.
-    this._fullConfig.groups = (config as any).groups;
-    validateProjectGroups(this._configFile || '<default config>', this._fullConfig);
   }
 
   private _assignUniqueProjectIds(projects: FullProjectInternal[]) {
@@ -642,87 +637,6 @@ function validateProject(file: string, project: Project, title: string) {
   }
 }
 
-function validateProjectGroups(file: string, config: FullConfigInternal) {
-  if (config.groups === undefined)
-    return;
-  const projectNames = new Set(config.projects?.filter(p => !!p.name).map(p => p.name));
-  for (const [groupName, group] of Object.entries(config.groups)) {
-    function validateProjectReference(projectName: string) {
-      if (projectName.trim() === '')
-        throw errorWithFile(file, `config.groups.${groupName} refers to an empty project name`);
-      if (!projectNames.has(projectName))
-        throw errorWithFile(file, `config.groups.${groupName} refers to an unknown project '${projectName}'`);
-    }
-    group.forEach((step, stepIndex) => {
-      if (isString(step)) {
-        validateProjectReference(step);
-      } else if (Array.isArray(step)) {
-        const parallelProjectNames = new Set();
-        step.forEach((item, itemIndex) => {
-          let projectName;
-          if (isString(item)) {
-            validateProjectReference(item);
-            projectName = item;
-          } else if (isObject(item)) {
-            const project = item.project;
-            if (isString(project)) {
-              validateProjectReference(project);
-            } else if (Array.isArray(project)) {
-              project.forEach((name, projectIndex) => {
-                if (!isString(name))
-                  throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].project[${projectIndex}] contains non string value.`);
-                validateProjectReference(name);
-              });
-            }
-            projectName = project;
-            if ('grep' in item) {
-              if (Array.isArray(item.grep)) {
-                item.grep.forEach((item, grepIndex) => {
-                  if (!isRegExp(item))
-                    throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grep[${grepIndex}] must be a RegExp`);
-                });
-              } else if (!isRegExp(item.grep)) {
-                throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grep must be a RegExp`);
-              }
-            }
-            if ('grepInvert' in item) {
-              if (Array.isArray(item.grepInvert)) {
-                item.grepInvert.forEach((item, index) => {
-                  if (!isRegExp(item))
-                    throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grepInvert[${index}] must be a RegExp`);
-                });
-              } else if (!isRegExp(item.grepInvert)) {
-                throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].grepInvert must be a RegExp`);
-              }
-            }
-            for (const prop of ['testIgnore', 'testMatch'] as const) {
-              if (prop in item) {
-                const value = item[prop];
-                if (Array.isArray(value)) {
-                  value.forEach((item, index) => {
-                    if (typeof item !== 'string' && !isRegExp(item))
-                      throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].${prop}[${index}] must be a string or a RegExp`);
-                  });
-                } else if (typeof value !== 'string' && !isRegExp(value)) {
-                  throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}].${prop} must be a string or a RegExp`);
-                }
-              }
-            }
-          } else {
-            throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}] unexpected group entry ${JSON.stringify(step, null, 2)}`);
-          }
-          // We can relax this later.
-          if (parallelProjectNames.has(projectName))
-            throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}][${itemIndex}] group mentions project '${projectName}' twice in one parallel group`);
-          parallelProjectNames.add(projectName);
-        });
-      } else {
-        throw errorWithFile(file, `config.groups.${groupName}[${stepIndex}] unexpected group entry ${JSON.stringify(step, null, 2)}`);
-      }
-    });
-  }
-}
-
 export const baseFullConfig: FullConfigInternal = {
   forbidOnly: false,
   fullyParallel: false,

packages/playwright-test/src/runner.ts

@@ -15,39 +15,37 @@
  * limitations under the License.
  */
 
-import { rimraf, minimatch } from 'playwright-core/lib/utilsBundle';
 import * as fs from 'fs';
 import * as path from 'path';
+import { assert } from 'playwright-core/lib/utils';
+import { MultiMap } from 'playwright-core/lib/utils/multimap';
+import { raceAgainstTimeout } from 'playwright-core/lib/utils/timeoutRunner';
+import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle';
 import { promisify } from 'util';
+import type { FullResult, Reporter, TestError } from '../types/testReporter';
 import type { TestGroup } from './dispatcher';
 import { Dispatcher } from './dispatcher';
-import type { Matcher, TestFileFilter } from './util';
-import { createFileMatcher, createTitleMatcher, serializeError } from './util';
-import type { TestCase } from './test';
-import { Suite } from './test';
 import { Loader } from './loader';
-import type { FullResult, Reporter, TestError } from '../types/testReporter';
-import { Multiplexer } from './reporters/multiplexer';
-import { formatError } from './reporters/base';
-import { colors } from 'playwright-core/lib/utilsBundle';
-import DotReporter from './reporters/dot';
-import GitHubReporter from './reporters/github';
-import LineReporter from './reporters/line';
-import ListReporter from './reporters/list';
-import JSONReporter from './reporters/json';
-import JUnitReporter from './reporters/junit';
-import EmptyReporter from './reporters/empty';
-import HtmlReporter from './reporters/html';
-import type { Config, FullProjectInternal, ReporterInternal } from './types';
-import type { FullConfigInternal } from './types';
-import { raceAgainstTimeout } from 'playwright-core/lib/utils/timeoutRunner';
-import { SigIntWatcher } from './sigIntWatcher';
 import type { TestRunnerPlugin } from './plugins';
 import { setRunnerToAddPluginsTo } from './plugins';
-import { webServerPluginsForConfig } from './plugins/webServerPlugin';
 import { dockerPlugin } from './plugins/dockerPlugin';
-import { MultiMap } from 'playwright-core/lib/utils/multimap';
+import { webServerPluginsForConfig } from './plugins/webServerPlugin';
-import { isString, assert } from 'playwright-core/lib/utils';
+import { formatError } from './reporters/base';
+import DotReporter from './reporters/dot';
+import EmptyReporter from './reporters/empty';
+import GitHubReporter from './reporters/github';
+import HtmlReporter from './reporters/html';
+import JSONReporter from './reporters/json';
+import JUnitReporter from './reporters/junit';
+import LineReporter from './reporters/line';
+import ListReporter from './reporters/list';
+import { Multiplexer } from './reporters/multiplexer';
+import { SigIntWatcher } from './sigIntWatcher';
+import type { TestCase } from './test';
+import { Suite } from './test';
+import type { Config, FullConfigInternal, FullProjectInternal, ReporterInternal } from './types';
+import type { Matcher, TestFileFilter } from './util';
+import { createFileMatcher, createTitleMatcher, serializeError } from './util';
 
 const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
@@ -63,66 +61,15 @@ type ProjectConstraints = {
 // Project group is a sequence of run phases.
 class RunPhase {
   static collectRunPhases(options: RunOptions, config: FullConfigInternal): RunPhase[] {
-    let projectGroup = options.projectGroup;
-    if (options.projectFilter) {
-      if (projectGroup)
-        throw new Error('--group option can not be combined with --project');
-    } else {
-      if (!projectGroup && config.groups?.default && !options.testFileFilters?.length)
-        projectGroup = 'default';
-      if (projectGroup) {
-        if (config.shard)
-          throw new Error(`Project group '${projectGroup}' cannot be combined with --shard`);
-      }
-    }
-
     const phases: RunPhase[] = [];
-    if (projectGroup) {
+    const testFileMatcher = fileMatcherFrom(options.testFileFilters);
-      const group = config.groups?.[projectGroup];
+    const testTitleMatcher = options.testTitleMatcher;
-      if (!group)
+    const projects = options.projectFilter ?? config.projects.map(p => p.name);
-        throw new Error(`Cannot find project group '${projectGroup}' in the config`);
+    phases.push(new RunPhase(projects.map(projectName => ({
-      for (const entry of group) {
+      projectName,
-        if (isString(entry)) {
+      testFileMatcher,
-          phases.push(new RunPhase([{
+      testTitleMatcher
-            projectName: entry,
+    }))));
-            testFileMatcher: () => true,
-            testTitleMatcher: () => true,
-          }]));
-        } else {
-          const phase: ProjectConstraints[] = [];
-          for (const p of entry) {
-            if (isString(p)) {
-              phase.push({
-                projectName: p,
-                testFileMatcher: () => true,
-                testTitleMatcher: () => true,
-              });
-            } else {
-              const testMatch = p.testMatch ? createFileMatcher(p.testMatch) : () => true;
-              const testIgnore = p.testIgnore ? createFileMatcher(p.testIgnore) : () => false;
-              const grep = p.grep ? createTitleMatcher(p.grep) : () => true;
-              const grepInvert = p.grepInvert ? createTitleMatcher(p.grepInvert) : () => false;
-              const projects = isString(p.project) ? [p.project] : p.project;
-              phase.push(...projects.map(projectName => ({
-                projectName,
-                testFileMatcher: (file: string) => !testIgnore(file) && testMatch(file),
-                testTitleMatcher: (title: string) => !grepInvert(title) && grep(title),
-              })));
-            }
-          }
-          phases.push(new RunPhase(phase));
-        }
-      }
-    } else {
-      const testFileMatcher = fileMatcherFrom(options.testFileFilters);
-      const testTitleMatcher = options.testTitleMatcher;
-      const projects = options.projectFilter ?? config.projects.map(p => p.name);
-      phases.push(new RunPhase(projects.map(projectName => ({
-        projectName,
-        testFileMatcher,
-        testTitleMatcher
-      }))));
-    }
     return phases;
   }
 
@@ -152,7 +99,6 @@ type RunOptions = {
   testFileFilters: TestFileFilter[];
   testTitleMatcher: Matcher;
   projectFilter?: string[];
-  projectGroup?: string;
   passWithNoTests?: boolean;
 };
 
@@ -455,7 +401,6 @@ export class Runner {
     // Compute shards.
     const shard = config.shard;
     if (shard) {
-      assert(!options.projectGroup);
       assert(concurrentTestGroups.length === 1);
       const shardGroups: TestGroup[] = [];
       const shardTests = new Set<TestCase>();

packages/playwright-test/src/types.ts

@@ -55,14 +55,6 @@ export interface FullConfigInternal extends FullConfigPublic {
 
   // Overrides the public field.
   projects: FullProjectInternal[];
-
-  groups?: { [key: string]: Array<string | Array<string | {
-    project: string | string[],
-    grep?: RegExp | RegExp[],
-    grepInvert?: RegExp | RegExp[],
-    testMatch?: string | RegExp | Array<string | RegExp>,
-    testIgnore?: string | RegExp | Array<string | RegExp>
-  }>> };
 }
 
 /**

tests/playwright-test/config.spec.ts

@@ -480,147 +480,3 @@ test('should have correct types for the config', async ({ runTSC }) => {
   });
   expect(result.exitCode).toBe(0);
 });
-
-test('should throw when group has duplicate project references', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-        module.exports = {
-          projects: [
-            { name: 'a' },
-          ],
-          groups: {
-            default: [
-              ['a', 'a']
-            ]
-          }
-        };
-    `,
-    'a.test.ts': `
-        const { test } = pwt;
-        test('pass', async () => {});
-      `
-  });
-
-  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`config.groups.default[0][1] group mentions project 'a' twice in one parallel group`);
-});
-
-test('should throw when group grep has invalid type', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-        module.exports = {
-          projects: [
-            { name: 'a' },
-          ],
-          groups: {
-            default: [
-              [{ project: 'a', grep: 2022 }]
-            ]
-          }
-        };
-    `,
-    'a.test.ts': `
-        const { test } = pwt;
-        test('pass', async () => {});
-      `
-  });
-
-  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`config.groups.default[0][0].grep must be a RegExp`);
-});
-
-test('should throw when group grepInvert has invalid type', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-        module.exports = {
-          projects: [
-            { name: 'a' },
-          ],
-          groups: {
-            default: [
-              [{ project: 'a', grepInvert: [{}] }]
-            ]
-          }
-        };
-    `,
-    'a.test.ts': `
-        const { test } = pwt;
-        test('pass', async () => {});
-      `
-  });
-
-  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`config.groups.default[0][0].grepInvert[0] must be a RegExp`);
-});
-
-test('should throw when group testMatch has invalid type', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-        module.exports = {
-          projects: [
-            { name: 'a' },
-          ],
-          groups: {
-            all: [
-              [{ project: 'a', testMatch: [{}] }]
-            ]
-          }
-        };
-    `,
-    'a.test.ts': `
-        const { test } = pwt;
-        test('pass', async () => {});
-      `
-  });
-
-  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`config.groups.all[0][0].testMatch[0] must be a string or a RegEx`);
-});
-
-test('should throw when group testIgnore has invalid type', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-        module.exports = {
-          projects: [
-            { name: 'a' },
-          ],
-          groups: {
-            all: [
-              [{ project: 'a', testIgnore: [2022] }]
-            ]
-          }
-        };
-    `,
-    'a.test.ts': `
-        const { test } = pwt;
-        test('pass', async () => {});
-      `
-  });
-
-  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`config.groups.all[0][0].testIgnore[0] must be a string or a RegEx`);
-});
-
-test('should throw when group has unknown project reference', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-        module.exports = {
-          projects: [
-            { name: 'a' },
-          ],
-          groups: {
-            default: [
-              [{project: 'b'}]
-            ]
-          }
-        };
-    `,
-    'a.test.ts': `
-        const { test } = pwt;
-        test('pass', async () => {});
-      `
-  });
-
-  expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`config.groups.default refers to an unknown project 'b'`);
-});

tests/playwright-test/groups.spec.ts

@@ -1,366 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import type { PlaywrightTestConfig, TestInfo, PlaywrightTestProject } from '@playwright/test';
-import path from 'path';
-import { test, expect } from './playwright-test-fixtures';
-
-function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: PlaywrightTestConfig['groups'], projectTemplates?: { [name: string]: PlaywrightTestProject }): Record<string, string> {
-  const config: PlaywrightTestConfig = {
-    projects: names.map(name => ({ ...projectTemplates?.[name], name, testDir: testInfo.outputPath(name) })),
-    groups
-  };
-  const files = {};
-  for (const name of names) {
-    files[`${name}/${name}.spec.ts`] = `
-      const { test } = pwt;
-      test('${name} test', async () => {
-        await new Promise(f => setTimeout(f, 100));
-      });`;
-  }
-  function replacer(key, value) {
-    if (value instanceof RegExp)
-      return `RegExp(${value.toString()})`;
-    else
-      return value;
-  }
-  files['playwright.config.ts'] = `
-    import * as path from 'path';
-    module.exports = ${JSON.stringify(config, replacer, 2)};
-    `.replace(/"RegExp\((.*)\)"/g, '$1');
-  return files;
-}
-
-type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
-
-function formatTimeline(timeline: Timeline) {
-  return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
-}
-
-function projectNames(timeline: Timeline) {
-  const projectNames = Array.from(new Set(timeline.map(({ titlePath }) => titlePath[1])).keys());
-  projectNames.sort();
-  return projectNames;
-}
-
-function expectRunBefore(timeline: Timeline, before: string[], after: string[]) {
-  const begin = new Map<string, number>();
-  const end = new Map<string, number>();
-  for (let i = 0; i < timeline.length; i++) {
-    const projectName = timeline[i].titlePath[1];
-    const map = timeline[i].event === 'begin' ? begin : end;
-    const oldIndex = map.get(projectName) ?? i;
-    const newIndex = (timeline[i].event === 'begin') ? Math.min(i, oldIndex) : Math.max(i, oldIndex);
-    map.set(projectName, newIndex);
-  }
-  for (const b of before) {
-    for (const a of after) {
-      const bEnd = end.get(b) as number;
-      expect(bEnd === undefined, `Unknown project ${b}`).toBeFalsy();
-      const aBegin = begin.get(a) as number;
-      expect(aBegin === undefined, `Unknown project ${a}`).toBeFalsy();
-      if (bEnd < aBegin)
-        continue;
-      throw new Error(`Project '${b}' expected to finish before '${a}'\nTest run order was:\n${formatTimeline(timeline)}`);
-    }
-  }
-}
-
-test('should work', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: ['a']
-  });
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expect(passed).toBe(1);
-  expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
-a > a${path.sep}a.spec.ts > a test [end]`);
-});
-
-test('should order two projects', async ({ runGroups }, testInfo) => {
-  await test.step(`order a then b`, async () => {
-    const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-      default: [
-        'a',
-        'b'
-      ]
-    });
-    const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-    expect(exitCode).toBe(0);
-    expect(passed).toBe(2);
-    expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
-a > a${path.sep}a.spec.ts > a test [end]
-b > b${path.sep}b.spec.ts > b test [begin]
-b > b${path.sep}b.spec.ts > b test [end]`);
-  });
-  await test.step(`order b then a`, async () => {
-    const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-      default: [
-        'b',
-        'a'
-      ]
-    });
-    const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-    expect(exitCode).toBe(0);
-    expect(passed).toBe(2);
-    expect(formatTimeline(timeline)).toEqual(`b > b${path.sep}b.spec.ts > b test [begin]
-b > b${path.sep}b.spec.ts > b test [end]
-a > a${path.sep}a.spec.ts > a test [begin]
-a > a${path.sep}a.spec.ts > a test [end]`);
-  });
-});
-
-test('should order 1-3-1 projects', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      'e',
-      ['d', 'c', 'b'],
-      'a',
-    ]
-  });
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expectRunBefore(timeline, ['e'], ['d', 'c', 'b']);
-  expectRunBefore(timeline, ['d', 'c', 'b'], ['a']);
-  expect(passed).toBe(5);
-});
-
-test('should order 2-2-2 projects', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      ['a', 'b'],
-      ['d', 'c'],
-      ['e', 'f'],
-    ]
-  });
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expectRunBefore(timeline, ['a', 'b'], ['c', 'd']);
-  expectRunBefore(timeline, ['c', 'd'], ['e', 'f']);
-  expect(passed).toBe(6);
-});
-
-test('should run parallel groups sequentially without overlaps', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      ['a', 'b', 'c', 'd'],
-      ['a', 'b', 'c', 'd'],
-      ['a', 'b', 'c', 'd'],
-    ]
-  });
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-
-  const expectedEndOfFirstPhase = events => {
-    const firstProjectEndIndex = project => events.findIndex(e => e.event === 'end' && e.titlePath[1] === project);
-    return Math.max(...['a', 'b', 'c', 'd'].map(firstProjectEndIndex));
-  };
-  const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
-
-  let remainingTimeline = timeline;
-  for (let i = 0; i < 3; i++) {
-    const phaseEndIndex = expectedEndOfFirstPhase(remainingTimeline);
-    const firstPhase = formatPhaseEvents(remainingTimeline.slice(0, phaseEndIndex + 1));
-    firstPhase.sort();
-    expect(firstPhase, `check phase ${i}`).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end', 'c:begin', 'c:end', 'd:begin', 'd:end']);
-    remainingTimeline = remainingTimeline.slice(phaseEndIndex + 1);
-  }
-  expect(remainingTimeline.length).toBe(0);
-
-  expect(passed).toBe(12);
-});
-
-test('should support phase with multiple project names', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      [
-        { project: ['a', 'b', 'c'] }
-      ],
-      [
-        { project: ['d'] },
-        { project: ['e', 'f'] }
-      ],
-    ]
-  });
-
-  const { exitCode, passed } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expect(passed).toBe(6);
-});
-
-test('should support varios syntax', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      'a',
-      ['a', 'b'],
-      [
-        { project: ['a', 'b'] }
-      ],
-      [
-        { project: ['a', 'b'] },
-        'c',
-        { project: 'd' },
-      ],
-      [{ project: 'e' }],
-      'f'
-    ]
-  });
-  const { exitCode, passed } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expect(passed).toBe(11);
-});
-
-test('should support --group option', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      'a', 'b'
-    ],
-    foo: [
-      ['b', 'c']
-    ],
-    bar: [
-      'd', 'e'
-    ]
-  });
-  const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
-  {
-    const { exitCode, passed, timeline } =  await runGroups(configWithFiles, { group: 'default' });
-    expect(exitCode).toBe(0);
-    expect(passed).toBe(2);
-    expect(formatPhaseEvents(timeline)).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end']);
-  }
-  {
-    const { exitCode, passed, timeline } =  await runGroups(configWithFiles, { group: 'foo' });
-    expect(exitCode).toBe(0);
-    expect(passed).toBe(2);
-    const formatted = formatPhaseEvents(timeline);
-    formatted.sort();
-    expect(formatted).toEqual(['b:begin', 'b:end', 'c:begin', 'c:end']);
-  }
-  {
-    const { exitCode, passed, timeline } =  await runGroups(configWithFiles, { group: 'bar' });
-    expect(exitCode).toBe(0);
-    expect(passed).toBe(2);
-    expect(formatPhaseEvents(timeline)).toEqual(['d:begin', 'd:end', 'e:begin', 'e:end']);
-  }
-});
-
-test('should throw when unknown --group is passed', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      'a', 'b'
-    ],
-    foo: [
-      ['b', 'c']
-    ]
-  });
-  const { exitCode, output } =  await runGroups(configWithFiles, { group: 'bar' });
-  expect(exitCode).toBe(1);
-  expect(output).toContain(`Cannot find project group 'bar' in the config`);
-});
-
-test('should support testMatch and testIgnore', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      [
-        { project: ['a', 'b'], testMatch: ['**/a.spec.ts'] },
-        { project: ['c', 'd'], testMatch: [/.*c.spec.ts/, '**/*d*'] },
-        { project: ['e'], testIgnore: [/.*e.spec.ts/] },
-        { project: ['f'], testMatch: /does not match/ },
-      ],
-    ]
-  });
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expect(passed).toBe(3);
-  expect(projectNames(timeline)).toEqual(['a', 'c', 'd']);
-});
-
-test('should support grep and grepInvert', async ({ runGroups }, testInfo) => {
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      [
-        { project: ['a', 'b'], grep: /.*a test/ },
-        { project: ['c', 'd'], grepInvert: [/.*c test/] },
-        { project: ['e', 'f'], grep: /.*(e|f) test/, grepInvert: [/.*f test/] },
-      ],
-    ]
-  });
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expect(passed).toBe(3);
-  expect(projectNames(timeline)).toEqual(['a', 'd', 'e']);
-});
-
-test('should intercect gpoup and project level grep and grepInvert', async ({ runGroups }, testInfo) => {
-  const projectTemplates = {
-    'a': {
-      grep: /a test/,
-      grepInvert: [/no test/],
-    },
-    'b': {
-      grep: /.*b te.*/,
-      grepInvert: [/.*a test/],
-    },
-    'c': {
-      grepInvert: [/.*test/],
-    },
-    'd': {
-      grep: [/.*unkwnown test/],
-    },
-  };
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      [
-        { project: ['a', 'b', 'c', 'd', 'e'], grep: /.*(b|c|d|e) test/, grepInvert: /.*d test/ },
-      ],
-    ]
-  }, projectTemplates);
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expect(passed).toBe(2);
-  expect(projectNames(timeline)).toEqual(['b', 'e']);
-});
-
-test('should intercect gpoup and project level testMatch and testIgnore', async ({ runGroups }, testInfo) => {
-  const projectTemplates = {
-    'a': {
-      testMatch: /.*a.spec.ts/,
-      testIgnore: [/no test/],
-    },
-    'b': {
-      testMatch: '**/b.spec.ts',
-      testIgnore: [/.*a.spec.ts/],
-    },
-    'c': {
-      testIgnore: [/.*no-match.spec.ts/],
-    },
-    'd': {
-      testMatch: [/.*unkwnown/],
-    },
-  };
-  const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
-    default: [
-      [
-        { project: ['a', 'b', 'c', 'd'], testMatch: /.*(b|c|d).spec.ts/, testIgnore: /.*c.spec.ts/ },
-        { project: ['c', 'd', 'e', 'f'], testIgnore: /.*[^ef].spec.ts/ },
-      ],
-    ]
-  }, projectTemplates);
-  const { exitCode, passed, timeline } =  await runGroups(configWithFiles);
-  expect(exitCode).toBe(0);
-  expect(passed).toBe(3);
-  expect(projectNames(timeline)).toEqual(['b', 'e', 'f']);
-});