chore: move best practices into the journal (#37694)
infra / docs & lint (push) Waiting to run Details
infra / Lint snippets (push) Waiting to run Details
components / ${{ matrix.os }} - Node.js ${{ matrix.node-version }} (18, macos-latest) (push) Waiting to run Details
components / ${{ matrix.os }} - Node.js ${{ matrix.node-version }} (18, ubuntu-latest) (push) Waiting to run Details
components / ${{ matrix.os }} - Node.js ${{ matrix.node-version }} (18, windows-latest) (push) Waiting to run Details
components / ${{ matrix.os }} - Node.js ${{ matrix.node-version }} (20, ubuntu-latest) (push) Waiting to run Details
components / ${{ matrix.os }} - Node.js ${{ matrix.node-version }} (22, ubuntu-latest) (push) Waiting to run Details
MCP / ${{ matrix.os }} (macos-15) (push) Waiting to run Details
MCP / ${{ matrix.os }} (ubuntu-latest) (push) Waiting to run Details
MCP / ${{ matrix.os }} (windows-latest) (push) Waiting to run Details
tests others / Stress - ${{ matrix.os }} (macos-latest) (push) Waiting to run Details
tests others / Stress - ${{ matrix.os }} (ubuntu-latest) (push) Waiting to run Details
tests others / Stress - ${{ matrix.os }} (windows-latest) (push) Waiting to run Details
tests others / WebView2 (push) Waiting to run Details
tests others / time library - ${{ matrix.clock }} (frozen) (push) Waiting to run Details
tests others / time library - ${{ matrix.clock }} (realtime) (push) Waiting to run Details
tests others / time test runner - ${{ matrix.clock }} (frozen) (push) Waiting to run Details
tests others / time test runner - ${{ matrix.clock }} (realtime) (push) Waiting to run Details
tests others / Electron - ${{ matrix.os }} (macos-latest) (push) Waiting to run Details
tests others / Electron - ${{ matrix.os }} (ubuntu-latest) (push) Waiting to run Details
tests others / Electron - ${{ matrix.os }} (windows-latest) (push) Waiting to run Details
tests 1 / ${{ matrix.os }} (${{ matrix.browser }} - Node.js ${{ matrix.node-version }}) (chromium, 18, ubuntu-22.04) (push) Waiting to run Details
tests 1 / ${{ matrix.os }} (${{ matrix.browser }} - Node.js ${{ matrix.node-version }}) (chromium, 20, ubuntu-22.04) (push) Waiting to run Details
tests 1 / ${{ matrix.os }} (${{ matrix.browser }} - Node.js ${{ matrix.node-version }}) (chromium, 22, ubuntu-22.04) (push) Waiting to run Details
tests 1 / ${{ matrix.os }} (${{ matrix.browser }} - Node.js ${{ matrix.node-version }}) (chromium, 24, ubuntu-22.04) (push) Waiting to run Details
tests 1 / ${{ matrix.os }} (${{ matrix.browser }} - Node.js ${{ matrix.node-version }}) (firefox, 18, ubuntu-22.04) (push) Waiting to run Details
tests 1 / ${{ matrix.os }} (${{ matrix.browser }} - Node.js ${{ matrix.node-version }}) (webkit, 18, ubuntu-22.04) (push) Waiting to run Details
tests 1 / ${{ matrix.os }} (chromium tip-of-tree) (ubuntu-22.04) (push) Waiting to run Details
tests 1 / Test Runner (18, macos-latest, 1, 2) (push) Waiting to run Details
tests 1 / Test Runner (18, macos-latest, 2, 2) (push) Waiting to run Details
tests 1 / Test Runner (18, ubuntu-latest, 1, 2) (push) Waiting to run Details
tests 1 / Test Runner (18, ubuntu-latest, 2, 2) (push) Waiting to run Details
tests 1 / Test Runner (18, windows-latest, 1, 2) (push) Waiting to run Details
tests 1 / Test Runner (18, windows-latest, 2, 2) (push) Waiting to run Details
tests 1 / Test Runner (20, ubuntu-latest, 1, 2) (push) Waiting to run Details
tests 1 / Test Runner (20, ubuntu-latest, 2, 2) (push) Waiting to run Details
tests 1 / Test Runner (22, ubuntu-latest, 1, 2) (push) Waiting to run Details
tests 1 / Test Runner (22, ubuntu-latest, 2, 2) (push) Waiting to run Details
tests 1 / Test Runner (24, ubuntu-latest, 1, 2) (push) Waiting to run Details
tests 1 / Test Runner (24, ubuntu-latest, 2, 2) (push) Waiting to run Details
tests 1 / Web Components (push) Waiting to run Details
tests 1 / VSCode Extension (push) Waiting to run Details
tests 1 / Installation Test ${{ matrix.os }} (macos-latest) (push) Waiting to run Details
tests 1 / Installation Test ${{ matrix.os }} (ubuntu-latest) (push) Waiting to run Details
tests 1 / Installation Test ${{ matrix.os }} (windows-latest) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (chromium, ubuntu-24.04) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (firefox, ubuntu-24.04) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (webkit, ubuntu-24.04) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (chromium, macos-14-large) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (chromium, macos-14-xlarge) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (chromium, macos-15-large) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (chromium, macos-15-xlarge) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (firefox, macos-14-large) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (firefox, macos-14-xlarge) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (firefox, macos-15-large) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (firefox, macos-15-xlarge) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (webkit, macos-14-large) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (webkit, macos-14-xlarge) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (webkit, macos-15-large) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (webkit, macos-15-xlarge) (push) Waiting to run Details
tests 2 / ${{ matrix.os }} (${{ matrix.browser }}) (webkit, macos-26-xlarge) (push) Waiting to run Details
tests 2 / Windows (chromium) (push) Waiting to run Details
tests 2 / Windows (firefox) (push) Waiting to run Details
tests 2 / Windows (webkit) (push) Waiting to run Details
tests 2 / Installation Test ${{ matrix.os }} (${{ matrix.node_version }}) (20, ubuntu-latest) (push) Waiting to run Details
tests 2 / Installation Test ${{ matrix.os }} (${{ matrix.node_version }}) (22, ubuntu-latest) (push) Waiting to run Details
tests 2 / Installation Test ${{ matrix.os }} (${{ matrix.node_version }}) (24, ubuntu-latest) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (chromium, macos-14-xlarge) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (chromium, ubuntu-24.04) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (chromium, windows-latest) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (firefox, macos-14-xlarge) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (firefox, ubuntu-24.04) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (firefox, windows-latest) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (webkit, macos-14-xlarge) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (webkit, ubuntu-22.04) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (webkit, ubuntu-24.04) (push) Waiting to run Details
tests 2 / headed ${{ matrix.browser }} (${{ matrix.os }}) (webkit, windows-latest) (push) Waiting to run Details
tests 2 / Transport (driver) (push) Waiting to run Details
tests 2 / Transport (service) (push) Waiting to run Details
tests 2 / Tracing ${{ matrix.browser }} ${{ matrix.channel }} (chromium, chromium-tip-of-tree, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Tracing ${{ matrix.browser }} ${{ matrix.channel }} (chromium, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Tracing ${{ matrix.browser }} ${{ matrix.channel }} (firefox, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Tracing ${{ matrix.browser }} ${{ matrix.channel }} (webkit, ubuntu-24.04) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (chrome, macos-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (chrome, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (chrome, windows-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (chrome-beta, macos-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (chrome-beta, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (chrome-beta, windows-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge, macos-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge, windows-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge-beta, macos-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge-beta, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge-beta, windows-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge-dev, macos-latest) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge-dev, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Test ${{ matrix.channel }} on ${{ matrix.runs-on }} (msedge-dev, windows-latest) (push) Waiting to run Details
tests 2 / Chromium tip-of-tree ${{ matrix.os }}${{ matrix.headed }} (, macos-13) (push) Waiting to run Details
tests 2 / Chromium tip-of-tree ${{ matrix.os }}${{ matrix.headed }} (, windows-latest) (push) Waiting to run Details
tests 2 / Chromium tip-of-tree ${{ matrix.os }}${{ matrix.headed }} (--headed, macos-13) (push) Waiting to run Details
tests 2 / Chromium tip-of-tree ${{ matrix.os }}${{ matrix.headed }} (--headed, ubuntu-22.04) (push) Waiting to run Details
tests 2 / Chromium tip-of-tree ${{ matrix.os }}${{ matrix.headed }} (--headed, windows-latest) (push) Waiting to run Details
tests 2 / Chromium tip-of-tree headless-shell-${{ matrix.os }} (ubuntu-22.04) (push) Waiting to run Details
tests 2 / Firefox Beta ${{ matrix.os }} (macos-latest) (push) Waiting to run Details
tests 2 / Firefox Beta ${{ matrix.os }} (ubuntu-22.04) (push) Waiting to run Details
tests 2 / Firefox Beta ${{ matrix.os }} (windows-latest) (push) Waiting to run Details
tests 2 / build-playwright-driver (push) Waiting to run Details
tests 2 / Test channel=chromium (macos-latest) (push) Waiting to run Details
tests 2 / Test channel=chromium (ubuntu-latest) (push) Waiting to run Details
tests 2 / Test channel=chromium (windows-latest) (push) Waiting to run Details
tests Video / Video Linux (chromium, ubuntu-22.04) (push) Waiting to run Details
tests Video / Video Linux (chromium, ubuntu-24.04) (push) Waiting to run Details
tests Video / Video Linux (firefox, ubuntu-22.04) (push) Waiting to run Details
tests Video / Video Linux (firefox, ubuntu-24.04) (push) Waiting to run Details
tests Video / Video Linux (webkit, ubuntu-22.04) (push) Waiting to run Details
tests Video / Video Linux (webkit, ubuntu-24.04) (push) Waiting to run Details
Internal Tests / trigger (push) Waiting to run Details

This commit is contained in:
Pavel Feldman 2025-10-02 17:32:30 -07:00 committed by GitHub
parent 530761fe7c
commit 42a623f9b6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 30 additions and 41 deletions

View File

@ -24,6 +24,7 @@ application behavior.
- Test title must match the scenario name
- Includes a comment with the step text before each step execution. Do not duplicate comments if step requires
multiple actions.
- Always use best practices from the log when generating tests.
<example-generation>
For following plan:
@ -55,16 +56,4 @@ application behavior.
});
});
```
</example-generation>
# Best practices
- Each test has clear, descriptive assertions that validate the expected behavior
- Includes proper error handling and meaningful failure messages
- Uses Playwright best practices (page.waitForLoadState, expect.toBeVisible, etc.)
- Do not improvise, do not add directives that were not asked for
- Uses reliable locators (preferring data-testid, role-based, or text-based selectors over fragile CSS selectors)
- Uses local variables for locators that are used multiple times
- Uses explicit waits rather than arbitrary timeouts
- Never waits for networkidle or use other discouraged or deprecated apis
- Is self-contained and can run independently
- Is deterministic and not prone to flaky behavior
</example-generation>

View File

@ -21,6 +21,7 @@ application behavior.
- Test title must match the scenario name
- Includes a comment with the step text before each step execution. Do not duplicate comments if step requires
multiple actions.
- Always use best practices from the log when generating tests.
<example-generation>
For following plan:
@ -53,17 +54,5 @@ application behavior.
});
```
</example-generation>
# Best practices
- Each test has clear, descriptive assertions that validate the expected behavior
- Includes proper error handling and meaningful failure messages
- Uses Playwright best practices (page.waitForLoadState, expect.toBeVisible, etc.)
- Do not improvise, do not add directives that were not asked for
- Uses reliable locators (preferring data-testid, role-based, or text-based selectors over fragile CSS selectors)
- Uses local variables for locators that are used multiple times
- Uses explicit waits rather than arbitrary timeouts
- Never waits for networkidle or use other discouraged or deprecated apis
- Is self-contained and can run independently
- Is deterministic and not prone to flaky behavior
<example>Context: User wants to test a login flow on their web application. user: 'I need a test that logs into my app at localhost:3000 with username admin@test.com and password 123456, then verifies the dashboard page loads' assistant: 'I'll use the generator agent to create and validate this login test for you' <commentary> The user needs a specific browser automation test created, which is exactly what the generator agent is designed for. </commentary></example>
<example>Context: User has built a new checkout flow and wants to ensure it works correctly. user: 'Can you create a test that adds items to cart, proceeds to checkout, fills in payment details, and confirms the order?' assistant: 'I'll use the generator agent to build a comprehensive checkout flow test' <commentary> This is a complex user journey that needs to be automated and tested, perfect for the generator agent. </commentary></example>

View File

@ -46,6 +46,7 @@ application behavior.
- Test title must match the scenario name
- Includes a comment with the step text before each step execution. Do not duplicate comments if step requires
multiple actions.
- Always use best practices from the log when generating tests.
<example-generation>
For following plan:
@ -79,18 +80,6 @@ application behavior.
```
</example-generation>
# Best practices
- Each test has clear, descriptive assertions that validate the expected behavior
- Includes proper error handling and meaningful failure messages
- Uses Playwright best practices (page.waitForLoadState, expect.toBeVisible, etc.)
- Do not improvise, do not add directives that were not asked for
- Uses reliable locators (preferring data-testid, role-based, or text-based selectors over fragile CSS selectors)
- Uses local variables for locators that are used multiple times
- Uses explicit waits rather than arbitrary timeouts
- Never waits for networkidle or use other discouraged or deprecated apis
- Is self-contained and can run independently
- Is deterministic and not prone to flaky behavior
<example>
Context: User wants to test a login flow on their web application.
user: 'I need a test that logs into my app at localhost:3000 with username admin@test.com and password 123456, then

View File

@ -65,6 +65,7 @@ export class GeneratorJournal {
\`\`\`ts
${step.code}
\`\`\``).join('\n\n'));
result.push(bestPracticesMarkdown);
return result.join('\n\n');
}
}
@ -177,3 +178,16 @@ export class TestContext {
async close() {
}
}
const bestPracticesMarkdown = `
# Best practices
- Do not improvise, do not add directives that were not asked for
- Use clear, descriptive assertions to validate the expected behavior
- Use reliable locators from this log
- Use local variables for locators that are used multiple times
- Use Playwright waiting assertions and best practices from this log
- NEVER! use page.waitForLoadState()
- NEVER! use page.waitForNavigation()
- NEVER! use page.waitForTimeout()
- NEVER! use page.evaluate()
`;

View File

@ -101,7 +101,7 @@ test('generator_setup_page', async ({ startClient }) => {
expect(await client.callTool({
name: 'generator_read_log',
arguments: {},
})).toHaveTextResponse(`# Plan
})).toHaveTextResponse(expect.stringContaining(`# Plan
Test plan
@ -125,7 +125,11 @@ Test plan
### Click submit button
\`\`\`ts
await page.getByRole('button', { name: 'Submit' }).click();
\`\`\``);
\`\`\`
# Best practices
`));
});
test('click after generator_log_action', async ({ startClient }) => {
@ -164,7 +168,7 @@ test('click after generator_log_action', async ({ startClient }) => {
expect(await client.callTool({
name: 'generator_read_log',
arguments: {},
})).toHaveTextResponse(`# Plan
})).toHaveTextResponse(expect.stringContaining(`# Plan
Test plan
@ -188,7 +192,11 @@ Test plan
### Click submit button
\`\`\`ts
await page.getByRole('button', { name: 'Submit' }).click();
\`\`\``);
\`\`\`
# Best practices
`));
});
test('generator_setup_page is required', async ({ startClient }) => {