diff --git a/.github/workflows/tests_components.yml b/.github/workflows/tests_components.yml index abc904d8919c6..6daa301a2a6aa 100644 --- a/.github/workflows/tests_components.yml +++ b/.github/workflows/tests_components.yml @@ -9,6 +9,9 @@ on: paths-ignore: - 'browser_patches/**' - 'docs/**' + - 'packages/playwright-core/src/cli/client/**' + - 'packages/playwright-core/src/cli/daemon/**' + - 'packages/playwright-core/src/mcp/**' - 'packages/playwright/src/mcp/**' - 'tests/mcp/**' branches: diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.yml index 506946d4c5635..acbe235c2a80b 100644 --- a/.github/workflows/tests_mcp.yml +++ b/.github/workflows/tests_mcp.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-15, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] shardIndex: [1, 2] shardTotal: [2] runs-on: ${{ matrix.os }} diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts index 6b2c7561dc0ab..80c1548474110 100644 --- a/packages/playwright/src/reporters/junit.ts +++ b/packages/playwright/src/reporters/junit.ts @@ -24,7 +24,7 @@ import { stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; import type { JUnitReporterOptions } from '../../types/test'; -import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter'; +import type { FullConfig, FullResult, Suite, TestCase, TestResult } from '../../types/testReporter'; class JUnitReporter implements ReporterV2 { private config!: FullConfig; @@ -38,10 +38,12 @@ class JUnitReporter implements ReporterV2 { private resolvedOutputFile: string | undefined; private stripANSIControlSequences = false; private includeProjectInTestName = false; + private includeRetries = false; constructor(options: JUnitReporterOptions & CommonReporterOptions) { this.stripANSIControlSequences = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_STRIP_ANSI', !!options.stripANSIControlSequences); this.includeProjectInTestName = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_PROJECT_IN_TEST_NAME', !!options.includeProjectInTestName); + this.includeRetries = getAsBooleanFromENV('PLAYWRIGHT_JUNIT_INCLUDE_RETRIES', !!options.includeRetries); this.configDir = options.configDir; this.resolvedOutputFile = resolveOutputFile('JUNIT', options)?.outputFile; } @@ -143,6 +145,9 @@ class JUnitReporter implements ReporterV2 { } private async _addTestCase(suiteName: string, namePrefix: string, test: TestCase, entries: XMLEntry[]): Promise<'failure' | 'error' | null> { + const isRetried = this.includeRetries && test.results.length > 1; + const isFlaky = isRetried && test.ok(); + const entry = { name: 'testcase', attributes: { @@ -150,7 +155,14 @@ class JUnitReporter implements ReporterV2 { name: namePrefix + test.titlePath().slice(3).join(' › '), // filename classname: suiteName, - time: (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000 + // For flaky tests, use the last (successful) result's duration. + // For permanent failures with retries, use the first result's duration. + // Otherwise, use total duration across all results. + time: isFlaky + ? test.results[test.results.length - 1].duration / 1000 + : isRetried + ? test.results[0].duration / 1000 + : (test.results.reduce((acc, value) => acc + value.duration, 0)) / 1000 }, children: [] as XMLEntry[] @@ -185,34 +197,40 @@ class JUnitReporter implements ReporterV2 { } let classification: 'failure' | 'error' | null = null; - if (!test.ok()) { - const errorInfo = classifyError(test); - if (errorInfo) { - classification = errorInfo.elementName; - entry.children.push({ - name: errorInfo.elementName, - attributes: { - message: errorInfo.message, - type: errorInfo.type, - }, - text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test)) - }); - } else { - classification = 'failure'; - entry.children.push({ - name: 'failure', - attributes: { - message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`, - type: 'FAILURE', - }, - text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test)) - }); + + if (isFlaky) { + // Flaky test (eventually passed): use Maven Surefire /. + // No element — flaky tests count as passed. + for (const result of test.results) { + if (result.status === 'passed' || result.status === 'skipped') + continue; + entry.children.push(buildSurefireRetryEntry(result, 'flaky')); } + // classification stays null — flaky tests are not counted as failures. + } else if (isRetried) { + // Permanent failure (failed all retries): use + Maven Surefire /. + classification = this._addFailureEntry(test, entry); + // Add / for each subsequent retry. + for (let i = 1; i < test.results.length; i++) { + const result = test.results[i]; + if (result.status === 'passed' || result.status === 'skipped') + continue; + entry.children.push(buildSurefireRetryEntry(result, 'rerun')); + } + } else if (!test.ok()) { + // Standard failure (no retries, or includeRetries is false). + classification = this._addFailureEntry(test, entry); } const systemOut: string[] = []; const systemErr: string[] = []; - for (const result of test.results) { + // When retries are included, top-level output comes from the primary result only: + // flaky → last (successful) result; permanent failure → first result. + // Without retries: all results (original behavior). + const outputResults = isRetried + ? [isFlaky ? test.results[test.results.length - 1] : test.results[0]] + : test.results; + for (const result of outputResults) { for (const item of result.stdout) systemOut.push(item.toString()); for (const item of result.stderr) @@ -245,40 +263,92 @@ class JUnitReporter implements ReporterV2 { entry.children.push({ name: 'system-err', text: systemErr.join('') }); return classification; } -} -function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null { - for (const result of test.results) { - const error = result.error; - if (!error) - continue; - - const rawMessage = stripAnsiEscapes(error.message || error.value || ''); - - // Parse "ErrorName: message" format from serialized error. - const nameMatch = rawMessage.match(/^(\w+): /); - const errorName = nameMatch ? nameMatch[1] : ''; - const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage; - const firstLine = messageBody.split('\n')[0].trim(); - - // Check for expect/assertion failure pattern. - const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/); - if (matcherMatch) { - const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`; - return { - elementName: 'failure', - type: matcherName, - message: firstLine, - }; + private _addFailureEntry(test: TestCase, entry: XMLEntry): 'failure' | 'error' { + const errorInfo = classifyError(test); + if (errorInfo) { + entry.children!.push({ + name: errorInfo.elementName, + attributes: { message: errorInfo.message, type: errorInfo.type }, + text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test)) + }); + return errorInfo.elementName; } + entry.children!.push({ + name: 'failure', + attributes: { + message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`, + type: 'FAILURE', + }, + text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test)) + }); + return 'failure'; + } + +} - // Thrown error. +/** + * Builds a Maven Surefire retry entry (/ or /) + * with per-result stackTrace, system-out, and system-err as children. + */ +function buildSurefireRetryEntry(result: TestResult, prefix: 'flaky' | 'rerun'): XMLEntry { + const errorInfo = classifyResultError(result); + const baseName = errorInfo?.elementName === 'error' ? 'Error' : 'Failure'; + const elementName = `${prefix}${baseName}`; + const children: XMLEntry[] = []; + const stackTrace = result.error?.stack || result.error?.message || result.error?.value || ''; + children.push({ name: 'stackTrace', text: stripAnsiEscapes(stackTrace) }); + const resultOut = result.stdout.map(s => s.toString()).join(''); + const resultErr = result.stderr.map(s => s.toString()).join(''); + if (resultOut) + children.push({ name: 'system-out', text: resultOut }); + if (resultErr) + children.push({ name: 'system-err', text: resultErr }); + return { + name: elementName, + attributes: { message: errorInfo?.message || '', type: errorInfo?.type || 'FAILURE', time: result.duration / 1000 }, + children, + }; +} + +function classifyResultError(result: TestResult): { elementName: 'failure' | 'error'; type: string; message: string } | null { + const error = result.error; + if (!error) + return null; + + const rawMessage = stripAnsiEscapes(error.message || error.value || ''); + + // Parse "ErrorName: message" format from serialized error. + const nameMatch = rawMessage.match(/^(\w+): /); + const errorName = nameMatch ? nameMatch[1] : ''; + const messageBody = nameMatch ? rawMessage.slice(nameMatch[0].length) : rawMessage; + const firstLine = messageBody.split('\n')[0].trim(); + + // Check for expect/assertion failure pattern. + const matcherMatch = rawMessage.match(/expect\(.*?\)\.(not\.)?(\w+)/); + if (matcherMatch) { + const matcherName = `expect.${matcherMatch[1] || ''}${matcherMatch[2]}`; return { - elementName: 'error', - type: errorName || 'Error', + elementName: 'failure', + type: matcherName, message: firstLine, }; } + + // Thrown error. + return { + elementName: 'error', + type: errorName || 'Error', + message: firstLine, + }; +} + +function classifyError(test: TestCase): { elementName: 'failure' | 'error'; type: string; message: string } | null { + for (const result of test.results) { + const info = classifyResultError(result); + if (info) + return info; + } return null; } diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index c30f7bca04dab..742c4b9ebfa02 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -20,7 +20,7 @@ export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; export type ListReporterOptions = { printSteps?: boolean }; -export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean }; +export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean }; export type JsonReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string; diff --git a/tests/mcp/cli-fixtures.ts b/tests/mcp/cli-fixtures.ts index 56d0076323805..a6a05ef7f33d2 100644 --- a/tests/mcp/cli-fixtures.ts +++ b/tests/mcp/cli-fixtures.ts @@ -65,7 +65,7 @@ export const test = baseTest.extend<{ function cliEnv() { return { PLAYWRIGHT_DAEMON_SESSION_DIR: test.info().outputPath('daemon'), - PLAYWRIGHT_DAEMON_SOCKETS_DIR: path.join(test.info().project.outputDir, 'daemon-sockets'), + PLAYWRIGHT_DAEMON_SOCKETS_DIR: path.join(test.info().project.outputDir, 'ds'), }; } diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts index 6781ef4049cc9..ed8f1fe5793e5 100644 --- a/tests/playwright-test/reporter-junit.spec.ts +++ b/tests/playwright-test/reporter-junit.spec.ts @@ -633,5 +633,87 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(time).toBe(result.report.stats.duration / 1000); expect(time).toBeGreaterThan(1); }); + + test('should emit flakyFailure for flaky tests when includeRetries is enabled', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + retries: 2, + reporter: [['junit', { includeRetries: true }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + expect(testInfo.retry).toBe(2); + }); + `, + }, { reporter: '' }); + const xml = parseXML(result.output); + // Single testcase for the test; flaky tests count as passed. + expect(xml['testsuites']['$']['tests']).toBe('1'); + expect(xml['testsuites']['$']['failures']).toBe('0'); + expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); + expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('0'); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['$']['name']).toBe('one'); + // No element — test eventually passed. + expect(testcase['failure']).toBeFalsy(); + // Two elements for the two failed attempts. + expect(testcase['flakyFailure'].length).toBe(2); + expect(testcase['flakyFailure'][0]['stackTrace']).toBeTruthy(); + expect(testcase['flakyFailure'][1]['stackTrace']).toBeTruthy(); + expect(result.exitCode).toBe(0); + }); + + test('should not include retries by default', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + `, + }, { retries: 1, reporter: 'junit' }); + const xml = parseXML(result.output); + // Default behavior: single testcase, no flakyFailure elements. + expect(xml['testsuites']['$']['tests']).toBe('1'); + const testcases = xml['testsuites']['testsuite'][0]['testcase']; + expect(testcases.length).toBe(1); + expect(testcases[0]['$']['name']).toBe('one'); + expect(testcases[0]['flakyFailure']).toBeFalsy(); + expect(result.exitCode).toBe(0); + }); + + test('should emit rerunFailure for permanent failures when includeRetries is enabled', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['junit', { includeRetries: true }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + expect(1).toBe(0); + }); + `, + }, { reporter: '' }); + const xml = parseXML(result.output); + // Single testcase; permanent failure counts as 1 test with 1 failure. + expect(xml['testsuites']['$']['tests']).toBe('1'); + expect(xml['testsuites']['$']['failures']).toBe('1'); + expect(xml['testsuites']['testsuite'][0]['$']['tests']).toBe('1'); + expect(xml['testsuites']['testsuite'][0]['$']['failures']).toBe('1'); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['$']['name']).toBe('one'); + // for the first attempt. + expect(testcase['failure']).toBeTruthy(); + // for the retry. + expect(testcase['rerunFailure'].length).toBe(1); + expect(testcase['rerunFailure'][0]['stackTrace']).toBeTruthy(); + expect(result.exitCode).toBe(1); + }); }); } diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index e2529457cc718..93bbfe5e2e4c2 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -19,7 +19,7 @@ export * from 'playwright-core'; export type BlobReporterOptions = { outputDir?: string, fileName?: string }; export type ListReporterOptions = { printSteps?: boolean }; -export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean }; +export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean }; export type JsonReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string;