chore(ui): start adding ui mode tests (#21601)

packages/playwright-core/src/server/trace/viewer/traceViewer.ts

@@ -63,8 +63,6 @@ export async function showTraceViewer(traceUrls: string[], browserName: string,
     '--window-size=1280,800',
     '--test-type=',
   ] : [];
-  if (isUnderTest())
-    args.push(`--remote-debugging-port=0`);
 
   const context = await traceViewerPlaywright[traceViewerBrowser as 'chromium'].launchPersistentContext(serverSideCallMetadata(), '', {
     // TODO: store language in the trace.
@@ -74,7 +72,7 @@ export async function showTraceViewer(traceUrls: string[], browserName: string,
     ignoreDefaultArgs: ['--enable-automation'],
     headless,
     colorScheme: 'no-override',
-    useWebSocket: isUnderTest()
+    useWebSocket: isUnderTest(),
   });
 
   const controller = new ProgressController(serverSideCallMetadata(), context._browser);
@@ -84,6 +82,9 @@ export async function showTraceViewer(traceUrls: string[], browserName: string,
   await context.extendInjectedScript(consoleApiSource.source);
   const [page] = context.pages();
 
+  if (isUnderTest())
+    process.stderr.write('DevTools listening on: ' + context._browser.options.wsEndpoint + '\n');
+
   if (traceViewerBrowser === 'chromium')
     await installAppIcon(page);
   await syncLocalStorageWithSettings(page, 'traceviewer');

packages/playwright-core/src/utils/debug.ts

@@ -35,7 +35,7 @@ export function debugMode() {
   return debugEnv ? 'inspector' : '';
 }
 
-let _isUnderTest = false;
+let _isUnderTest = !!process.env.PWTEST_UNDER_TEST;
 export function setUnderTest() {
   _isUnderTest = true;
 }

packages/playwright-test/src/runner/uiMode.ts

@@ -16,7 +16,7 @@
 
 import { showTraceViewer } from 'playwright-core/lib/server';
 import type { Page } from 'playwright-core/lib/server/page';
-import { ManualPromise } from 'playwright-core/lib/utils';
+import { isUnderTest, ManualPromise } from 'playwright-core/lib/utils';
 import type { FullResult } from '../../reporter';
 import { clearCompilationCache, dependenciesForTestFile } from '../common/compilationCache';
 import type { FullConfigInternal } from '../common/types';
@@ -53,15 +53,6 @@ class UIMode {
     config._internal.configCLIOverrides.use.trace = { mode: 'on', sources: false };
 
     this._originalStderr = process.stderr.write.bind(process.stderr);
-    process.stdout.write = (chunk: string | Buffer) => {
-      this._dispatchEvent({ method: 'stdio', params: chunkToPayload('stdout', chunk) });
-      return true;
-    };
-    process.stderr.write = (chunk: string | Buffer) => {
-      this._dispatchEvent({ method: 'stdio', params: chunkToPayload('stderr', chunk) });
-      return true;
-    };
-
     this._installGlobalWatcher();
   }
 
@@ -101,7 +92,15 @@ class UIMode {
   }
 
   async showUI() {
-    this._page = await showTraceViewer([], 'chromium', { app: 'watch.html' });
+    this._page = await showTraceViewer([], 'chromium', { app: 'watch.html', headless: isUnderTest() && process.env.PWTEST_HEADED_FOR_TEST !== '1' });
+    process.stdout.write = (chunk: string | Buffer) => {
+      this._dispatchEvent({ method: 'stdio', params: chunkToPayload('stdout', chunk) });
+      return true;
+    };
+    process.stderr.write = (chunk: string | Buffer) => {
+      this._dispatchEvent({ method: 'stdio', params: chunkToPayload('stderr', chunk) });
+      return true;
+    };
     const exitPromise = new ManualPromise();
     this._page.on('close', () => exitPromise.resolve());
     let queue = Promise.resolve();

packages/trace-viewer/src/ui/watchMode.tsx

@@ -303,6 +303,7 @@ export const TestList: React.FC<{
     treeState={treeState}
     setTreeState={setTreeState}
     rootItem={rootItem}
+    dataTestId='test-tree'
     render={treeItem => {
       return <div className='hbox watch-mode-list-item'>
         <div className='watch-mode-list-item-title'>{treeItem.title}</div>
@@ -653,9 +654,12 @@ function createTree(rootSuite: Suite | undefined, projects: Map<string, boolean>
     visitSuite(projectSuite.title, projectSuite, rootItem);
   }
 
-  const propagateStatus = (treeItem: TreeItem) => {
+  const sortAndPropagateStatus = (treeItem: TreeItem) => {
     for (const child of treeItem.children)
-      propagateStatus(child);
+      sortAndPropagateStatus(child);
+
+    if (treeItem.kind === 'group' && treeItem.parent)
+      treeItem.children.sort((a, b) => a.location.line - b.location.line);
 
     let allPassed = treeItem.children.length > 0;
     let allSkipped = treeItem.children.length > 0;
@@ -678,7 +682,7 @@ function createTree(rootSuite: Suite | undefined, projects: Map<string, boolean>
     else if (allPassed)
       treeItem.status = 'passed';
   };
-  propagateStatus(rootItem);
+  sortAndPropagateStatus(rootItem);
   return rootItem;
 }
 

packages/web/src/components/listView.css

@@ -42,6 +42,10 @@
   z-index: 10;
 }
 
+.list-view-indent {
+  min-width: 16px;
+}
+
 .list-view-content:focus .list-view-entry.selected {
   background-color: var(--vscode-list-activeSelectionBackground);
   color: var(--vscode-list-activeSelectionForeground);

packages/web/src/components/listView.tsx

@@ -59,7 +59,7 @@ export function ListView<T>({
     onHighlighted?.(highlightedItem);
   }, [onHighlighted, highlightedItem]);
 
-  return <div className='list-view vbox' data-testid={dataTestId}>
+  return <div className='list-view vbox' role='list' data-testid={dataTestId}>
     <div
       className='list-view-content'
       tabIndex={0}
@@ -115,14 +115,15 @@ export function ListView<T>({
         const rendered = render(item);
         return <div
           key={id?.(item) || index}
+          role='listitem'
           className={'list-view-entry' + selectedSuffix + highlightedSuffix + errorSuffix}
           onClick={() => onSelected?.(item)}
           onMouseEnter={() => setHighlightedItem(item)}
           onMouseLeave={() => setHighlightedItem(undefined)}
         >
-          {indentation ? <div style={{ minWidth: indentation * 16 }}></div> : undefined}
+          {indentation ? new Array(indentation).fill(0).map(() => <div className='list-view-indent'></div>) : undefined}
           {icon && <div
-            className={'codicon ' + (icon(item) || 'blank')}
+            className={'codicon ' + (icon(item) || 'codicon-blank')}
             style={{ minWidth: 16, marginRight: 4 }}
             onDoubleClick={e => {
               e.preventDefault();

packages/web/src/components/treeView.tsx

@@ -56,6 +56,7 @@ export function TreeView<T extends TreeItem>({
   treeState,
   setTreeState,
   noItemsMessage,
+  dataTestId,
 }: TreeViewProps<T>) {
   const treeItems = React.useMemo(() => {
     for (let item: TreeItem | undefined = selectedItem?.parent; item; item = item.parent)
@@ -66,6 +67,7 @@ export function TreeView<T extends TreeItem>({
   return <TreeListView
     items={[...treeItems.keys()]}
     id={item => item.id}
+    dataTestId={dataTestId}
     render={item => {
       const rendered = render(item as T);
       return <>

tests/config/commonFixtures.ts

@@ -83,18 +83,24 @@ export class TestChildProcess {
 
   async close() {
     if (!this.process.killed)
-      this._killProcessGroup();
+      this._killProcessGroup('SIGINT');
     return this.exited;
   }
 
-  private _killProcessGroup() {
+  async kill() {
+    if (!this.process.killed)
+      this._killProcessGroup('SIGKILL');
+    return this.exited;
+  }
+
+  private _killProcessGroup(signal: 'SIGINT' | 'SIGKILL') {
     if (!this.process.pid || this.process.killed)
       return;
     try {
       if (process.platform === 'win32')
         execSync(`taskkill /pid ${this.process.pid} /T /F /FI "MEMUSAGE gt 0"`, { stdio: 'ignore' });
       else
-        process.kill(-this.process.pid, 'SIGKILL');
+        process.kill(-this.process.pid, signal);
     } catch (e) {
       // the process might have already stopped
     }

tests/playwright-test/playwright-test-fixtures.ts

@@ -28,7 +28,7 @@ import type { TestInfo } from './stable-test-runner';
 import { expect } from './stable-test-runner';
 import { test as base } from './stable-test-runner';
 
-const removeFolderAsync = promisify(rimraf);
+export const removeFolderAsync = promisify(rimraf);
 
 export type CliRunResult = {
   exitCode: number,
@@ -54,10 +54,10 @@ type TSCResult = {
   exitCode: number;
 };
 
-type Files = { [key: string]: string | Buffer };
+export type Files = { [key: string]: string | Buffer };
 type Params = { [key: string]: string | number | boolean | string[] };
 
-async function writeFiles(testInfo: TestInfo, files: Files, initial: boolean) {
+export async function writeFiles(testInfo: TestInfo, files: Files, initial: boolean) {
   const baseDir = testInfo.outputPath();
 
   if (initial && !Object.keys(files).some(name => name.includes('package.json'))) {
@@ -76,7 +76,7 @@ async function writeFiles(testInfo: TestInfo, files: Files, initial: boolean) {
   return baseDir;
 }
 
-const cliEntrypoint = path.join(__dirname, '../../packages/playwright-core/cli.js');
+export const cliEntrypoint = path.join(__dirname, '../../packages/playwright-core/cli.js');
 
 async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], baseDir: string, params: any, env: NodeJS.ProcessEnv, options: RunOptions): Promise<RunResult> {
   const paramList: string[] = [];
@@ -190,7 +190,7 @@ async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess']
   return { exitCode, output: testProcess.output.toString() };
 }
 
-function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+export function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
   return {
     ...process.env,
     // BEGIN: Reserved CI
@@ -216,7 +216,7 @@ function cleanEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
   };
 }
 
-type RunOptions = {
+export type RunOptions = {
   sendSIGINTAfter?: number;
   additionalArgs?: string[];
   cwd?: string,
@@ -254,7 +254,7 @@ export const test = base
           testProcess = watchPlaywrightTest(childProcess, baseDir, { ...env, PWTEST_CACHE_DIR: cacheDir }, options);
           return testProcess;
         });
-        await testProcess?.close();
+        await testProcess?.kill();
         await removeFolderAsync(cacheDir);
       },
 

tests/playwright-test/ui-mode-fixtures.ts

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import type { TestChildProcess } from '../config/commonFixtures';
+import { cleanEnv, cliEntrypoint, removeFolderAsync, test as base, writeFiles } from './playwright-test-fixtures';
+import type { Files, RunOptions } from './playwright-test-fixtures';
+import type { Browser, Page, TestInfo } from './stable-test-runner';
+
+type Fixtures = {
+  runUITest: (files: Files, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<Page>;
+};
+
+export function dumpTestTree(page: Page): () => Promise<string> {
+  return () => page.getByTestId('test-tree').evaluate(async treeElement => {
+    function iconName(iconElement: Element): string {
+      const icon = iconElement.className.replace('codicon codicon-', '');
+      if (icon === 'chevron-right')
+        return '►';
+      if (icon === 'chevron-down')
+        return '▼';
+      if (icon === 'blank')
+        return ' ';
+      if (icon === 'circle-outline')
+        return '◯';
+      if (icon === 'check')
+        return '✅';
+      if (icon === 'error')
+        return '❌';
+      return icon;
+    }
+
+    const result: string[] = [];
+    const listItems = treeElement.querySelectorAll('[role=listitem]');
+    for (const listItem of listItems) {
+      const iconElements = listItem.querySelectorAll('.codicon');
+      const treeIcon = iconName(iconElements[0]);
+      const statusIcon = iconName(iconElements[1]);
+      const indent = listItem.querySelectorAll('.list-view-indent').length;
+      const selected = listItem.classList.contains('selected') ? ' <=' : '';
+      result.push('    ' + '  '.repeat(indent) + treeIcon + ' ' + statusIcon + ' ' + listItem.textContent + selected);
+    }
+    return '\n' + result.join('\n') + '\n  ';
+  });
+}
+
+export const test = base
+    .extend<Fixtures>({
+      runUITest: async ({ childProcess, playwright, headless }, use, testInfo: TestInfo) => {
+        const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
+        let testProcess: TestChildProcess | undefined;
+        let browser: Browser | undefined;
+        await use(async (files: Files, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => {
+          const baseDir = await writeFiles(testInfo, files, true);
+          testProcess = childProcess({
+            command: ['node', cliEntrypoint, 'ui', ...(options.additionalArgs || [])],
+            env: {
+              ...cleanEnv(env),
+              PWTEST_UNDER_TEST: '1',
+              PWTEST_CACHE_DIR: cacheDir,
+              PWTEST_HEADED_FOR_TEST: headless ? '0' : '1',
+            },
+            cwd: options.cwd ? path.resolve(baseDir, options.cwd) : baseDir,
+          });
+          await testProcess.waitForOutput('DevTools listening on');
+          const line = testProcess.output.split('\n').find(l => l.includes('DevTools listening on'));
+          const wsEndpoint = line!.split(' ')[3];
+          browser = await playwright.chromium.connectOverCDP(wsEndpoint);
+          const [context] = browser.contexts();
+          const [page] = context.pages();
+          return page;
+        });
+        await browser?.close();
+        await testProcess?.close();
+        await removeFolderAsync(cacheDir);
+      },
+    });
+
+export { expect } from './stable-test-runner';

tests/playwright-test/ui-mode-test-tree.spec.ts

@@ -0,0 +1,119 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, dumpTestTree } from './ui-mode-fixtures';
+
+test.describe.configure({ mode: 'parallel' });
+
+const basicTestTree = {
+  'a.test.ts': `
+    import { test, expect } from '@playwright/test';
+    test('passes', () => {});
+    test('fails', () => {});
+    test.describe('suite', () => {
+      test('inner passes', () => {});
+      test('inner fails', () => {});
+    });
+  `,
+  'b.test.ts': `
+    import { test, expect } from '@playwright/test';
+    test('passes', () => {});
+    test('fails', () => {});
+  `,
+};
+
+test('should list tests', async ({ runUITest }) => {
+  const page = await runUITest(basicTestTree);
+  await expect.poll(dumpTestTree(page)).toBe(`
+    ▼ ◯ a.test.ts
+        ◯ passes
+        ◯ fails
+      ► ◯ suite
+    ▼ ◯ b.test.ts
+        ◯ passes
+        ◯ fails
+  `);
+});
+
+test('should traverse up/down', async ({ runUITest }) => {
+  const page = await runUITest(basicTestTree);
+  await page.getByText('a.test.ts').click();
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ▼ ◯ a.test.ts <=
+        ◯ passes
+        ◯ fails
+      ► ◯ suite
+  `);
+
+  await page.keyboard.press('ArrowDown');
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ▼ ◯ a.test.ts
+        ◯ passes <=
+        ◯ fails
+      ► ◯ suite
+  `);
+  await page.keyboard.press('ArrowDown');
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ▼ ◯ a.test.ts
+        ◯ passes
+        ◯ fails <=
+      ► ◯ suite
+  `);
+
+  await page.keyboard.press('ArrowUp');
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ▼ ◯ a.test.ts
+        ◯ passes <=
+        ◯ fails
+      ► ◯ suite
+  `);
+});
+
+test('should expand / collapse groups', async ({ runUITest }) => {
+  const page = await runUITest(basicTestTree);
+
+  await page.getByText('suite').click();
+  await page.keyboard.press('ArrowRight');
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ▼ ◯ a.test.ts
+        ◯ passes
+        ◯ fails
+      ▼ ◯ suite <=
+          ◯ inner passes
+          ◯ inner fails
+  `);
+
+  await page.keyboard.press('ArrowLeft');
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ▼ ◯ a.test.ts
+        ◯ passes
+        ◯ fails
+      ► ◯ suite <=
+  `);
+
+  await page.getByText('passes').first().click();
+  await page.keyboard.press('ArrowLeft');
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ▼ ◯ a.test.ts <=
+        ◯ passes
+        ◯ fails
+  `);
+
+  await page.keyboard.press('ArrowLeft');
+  await expect.poll(dumpTestTree(page)).toContain(`
+    ► ◯ a.test.ts <=
+  `);
+});