From ac421b048f5f70b794ddd310fbb71db67a580bd4 Mon Sep 17 00:00:00 2001 From: miliberlin Date: Mon, 16 Feb 2026 16:46:03 +0100 Subject: [PATCH 1/6] fix: skip PlaywrightCheck constructs and check files during pw-test [RED-155] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When pw-test runs, PlaywrightCheck constructs defined in .check.ts files should not be loaded — they interfere with the ad-hoc Playwright test run by adding a second check and clobbering dependency resolution. Two changes in parseProject() when currentCommand === 'pw-test': 1. Filter PlaywrightCheck instances from checklyConfigConstructs 2. Skip loadAllCheckFiles() entirely Co-Authored-By: Claude Opus 4.6 --- packages/cli/src/services/project-parser.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/services/project-parser.ts b/packages/cli/src/services/project-parser.ts index e20180cfb..220747532 100644 --- a/packages/cli/src/services/project-parser.ts +++ b/packages/cli/src/services/project-parser.ts @@ -165,7 +165,11 @@ export async function parseProject (opts: ProjectParseOpts): Promise { ignoreWorkspaces: !enableWorkspaces, }) - checklyConfigConstructs?.forEach( + const filteredConstructs = currentCommand === 'pw-test' + ? checklyConfigConstructs?.filter(c => !(c instanceof PlaywrightCheck)) + : checklyConfigConstructs + + filteredConstructs?.forEach( construct => project.addResource(construct.type, construct.logicalId, construct), ) Session.project = project @@ -186,7 +190,9 @@ export async function parseProject (opts: ProjectParseOpts): Promise { // TODO: Do we really need all of the ** globs, or could we just put node_modules? const ignoreDirectories = ['**/node_modules/**', '**/.git/**', ...ignoreDirectoriesMatch] - await loadAllCheckFiles(directory, checkMatch, ignoreDirectories) + if (currentCommand !== 'pw-test') { + await loadAllCheckFiles(directory, checkMatch, ignoreDirectories) + } // Load sequentially because otherwise Session.checkFileAbsolutePath and // Session.checkFilePath are going to be subject to race conditions. From 78ffee03a3dbb6351833629d6dd01489d4ad8f4f Mon Sep 17 00:00:00 2001 From: miliberlin Date: Mon, 16 Feb 2026 18:10:01 +0100 Subject: [PATCH 2/6] fix: update webServer test fixture to use playwrightConfigPath for ad-hoc check creation Since loadAllCheckFiles() is now skipped for pw-test, the test fixture needs playwrightConfigPath in the config so loadPlaywrightChecks() creates the ad-hoc PlaywrightCheck that triggers the webServer validation. Co-Authored-By: Claude Opus 4.6 --- .../playwright-check/test-cases/test-webServer/checkly.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-webServer/checkly.config.ts b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-webServer/checkly.config.ts index bae253372..8abf471e6 100644 --- a/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-webServer/checkly.config.ts +++ b/packages/cli/src/constructs/__tests__/fixtures/playwright-check/test-cases/test-webServer/checkly.config.ts @@ -5,6 +5,7 @@ const config = defineConfig({ logicalId: 'playwright-check-fixture', checks: { checkMatch: '**/*.check.ts', + playwrightConfigPath: './playwright.config.ts', }, }) From 2aa286861c5f981f0c578e47ef60714acab9f4be Mon Sep 17 00:00:00 2001 From: miliberlin Date: Tue, 17 Feb 2026 17:09:07 +0100 Subject: [PATCH 3/6] refactor: replace currentCommand with explicit behavior flags [RED-155] Replace caller-identity branching (currentCommand === 'pw-test') with explicit flags: ignoreCheckDefinitions and skipWebServerValidation. Remove Session.currentCommand and Session.includeFlagProvided. Co-Authored-By: Claude Opus 4.6 --- packages/cli/src/commands/debug/parse-project.ts | 4 ++-- packages/cli/src/commands/pw-test.ts | 4 ++-- packages/cli/src/constructs/playwright-check.ts | 4 ++-- packages/cli/src/constructs/project.ts | 3 +-- packages/cli/src/services/project-parser.ts | 15 +++++++-------- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/packages/cli/src/commands/debug/parse-project.ts b/packages/cli/src/commands/debug/parse-project.ts index d0c554947..48bbe5add 100644 --- a/packages/cli/src/commands/debug/parse-project.ts +++ b/packages/cli/src/commands/debug/parse-project.ts @@ -107,9 +107,9 @@ export default class ParseProjectCommand extends Command { checklyConfigConstructs, playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath, include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, - includeFlagProvided: includeFlag.length > 0, playwrightChecks: checklyConfig.checks?.playwrightChecks, - currentCommand: emulatePwTest ? 'pw-test' : undefined, + ignoreCheckDefinitions: emulatePwTest, + skipWebServerValidation: emulatePwTest && !(includeFlag.length > 0), }) const diagnostics = new Diagnostics() diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts index 8970b1489..ade0da820 100644 --- a/packages/cli/src/commands/pw-test.ts +++ b/packages/cli/src/commands/pw-test.ts @@ -190,9 +190,9 @@ export default class PwTestCommand extends AuthCommand { checklyConfigConstructs, playwrightConfigPath, include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, - includeFlagProvided: includeFlag.length > 0, playwrightChecks: [playwrightCheck], - currentCommand: 'pw-test', + ignoreCheckDefinitions: true, + skipWebServerValidation: !(includeFlag.length > 0), checkFilter: check => { // Skip non Playwright checks if (!(check instanceof PlaywrightCheck)) { diff --git a/packages/cli/src/constructs/playwright-check.ts b/packages/cli/src/constructs/playwright-check.ts index b32c416f9..03ca0c145 100644 --- a/packages/cli/src/constructs/playwright-check.ts +++ b/packages/cli/src/constructs/playwright-check.ts @@ -283,8 +283,8 @@ export class PlaywrightCheck extends RuntimeCheck { } protected async validateWebServerConfig (diagnostics: Diagnostics): Promise { - // Only show webServer warning for pw-test command when --include is not provided - if (Session.currentCommand !== 'pw-test' || Session.includeFlagProvided) { + // Skip webServer validation unless explicitly requested via skipWebServerValidation + if (!Session.skipWebServerValidation) { return } diff --git a/packages/cli/src/constructs/project.ts b/packages/cli/src/constructs/project.ts index fb7d624ca..67542ed3a 100644 --- a/packages/cli/src/constructs/project.ts +++ b/packages/cli/src/constructs/project.ts @@ -248,8 +248,7 @@ export class Session { static parsers = new Map() static constructExports: ConstructExport[] = [] static ignoreDirectoriesMatch: string[] = [] - static currentCommand?: 'pw-test' | 'test' | 'deploy' - static includeFlagProvided?: boolean + static skipWebServerValidation?: boolean static packageManager: PackageManager = npmPackageManager static workspace: Result = Err(new Error(`Workspace support not initialized`)) diff --git a/packages/cli/src/services/project-parser.ts b/packages/cli/src/services/project-parser.ts index 220747532..3dab298cb 100644 --- a/packages/cli/src/services/project-parser.ts +++ b/packages/cli/src/services/project-parser.ts @@ -45,9 +45,9 @@ type ProjectParseOpts = { checklyConfigConstructs?: Construct[] playwrightConfigPath?: string include?: string | string[] - includeFlagProvided?: boolean playwrightChecks?: PlaywrightSlimmedProp[] - currentCommand?: 'pw-test' | 'test' | 'deploy' + ignoreCheckDefinitions?: boolean + skipWebServerValidation?: boolean enableWorkspaces?: boolean } @@ -144,9 +144,9 @@ export async function parseProject (opts: ProjectParseOpts): Promise { checklyConfigConstructs, playwrightConfigPath, include, - includeFlagProvided, playwrightChecks, - currentCommand, + ignoreCheckDefinitions, + skipWebServerValidation, enableWorkspaces = true, } = opts const project = new Project(projectLogicalId, { @@ -165,7 +165,7 @@ export async function parseProject (opts: ProjectParseOpts): Promise { ignoreWorkspaces: !enableWorkspaces, }) - const filteredConstructs = currentCommand === 'pw-test' + const filteredConstructs = ignoreCheckDefinitions ? checklyConfigConstructs?.filter(c => !(c instanceof PlaywrightCheck)) : checklyConfigConstructs @@ -182,15 +182,14 @@ export async function parseProject (opts: ProjectParseOpts): Promise { Session.defaultRuntimeId = defaultRuntimeId Session.verifyRuntimeDependencies = verifyRuntimeDependencies ?? true Session.ignoreDirectoriesMatch = ignoreDirectoriesMatch - Session.currentCommand = currentCommand - Session.includeFlagProvided = includeFlagProvided + Session.skipWebServerValidation = skipWebServerValidation Session.packageManager = packageManager Session.workspace = workspace // TODO: Do we really need all of the ** globs, or could we just put node_modules? const ignoreDirectories = ['**/node_modules/**', '**/.git/**', ...ignoreDirectoriesMatch] - if (currentCommand !== 'pw-test') { + if (!ignoreCheckDefinitions) { await loadAllCheckFiles(directory, checkMatch, ignoreDirectories) } From 1bbf380a6d7356048726e5ced279b5be77d0d31c Mon Sep 17 00:00:00 2001 From: miliberlin Date: Tue, 17 Feb 2026 17:18:10 +0100 Subject: [PATCH 4/6] refactor: rename skipWebServerValidation to warnOnWebServerConfig Positive flag name better conveys intent: warn users about webServer config needing additional files when --include isn't provided. Co-Authored-By: Claude Opus 4.6 --- packages/cli/src/commands/debug/parse-project.ts | 2 +- packages/cli/src/commands/pw-test.ts | 2 +- packages/cli/src/constructs/playwright-check.ts | 3 +-- packages/cli/src/constructs/project.ts | 2 +- packages/cli/src/services/project-parser.ts | 6 +++--- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/commands/debug/parse-project.ts b/packages/cli/src/commands/debug/parse-project.ts index 48bbe5add..61050c11b 100644 --- a/packages/cli/src/commands/debug/parse-project.ts +++ b/packages/cli/src/commands/debug/parse-project.ts @@ -109,7 +109,7 @@ export default class ParseProjectCommand extends Command { include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, playwrightChecks: checklyConfig.checks?.playwrightChecks, ignoreCheckDefinitions: emulatePwTest, - skipWebServerValidation: emulatePwTest && !(includeFlag.length > 0), + warnOnWebServerConfig: emulatePwTest && !(includeFlag.length > 0), }) const diagnostics = new Diagnostics() diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts index ade0da820..069d888ec 100644 --- a/packages/cli/src/commands/pw-test.ts +++ b/packages/cli/src/commands/pw-test.ts @@ -192,7 +192,7 @@ export default class PwTestCommand extends AuthCommand { include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, playwrightChecks: [playwrightCheck], ignoreCheckDefinitions: true, - skipWebServerValidation: !(includeFlag.length > 0), + warnOnWebServerConfig: !(includeFlag.length > 0), checkFilter: check => { // Skip non Playwright checks if (!(check instanceof PlaywrightCheck)) { diff --git a/packages/cli/src/constructs/playwright-check.ts b/packages/cli/src/constructs/playwright-check.ts index 03ca0c145..734fd3790 100644 --- a/packages/cli/src/constructs/playwright-check.ts +++ b/packages/cli/src/constructs/playwright-check.ts @@ -283,8 +283,7 @@ export class PlaywrightCheck extends RuntimeCheck { } protected async validateWebServerConfig (diagnostics: Diagnostics): Promise { - // Skip webServer validation unless explicitly requested via skipWebServerValidation - if (!Session.skipWebServerValidation) { + if (!Session.warnOnWebServerConfig) { return } diff --git a/packages/cli/src/constructs/project.ts b/packages/cli/src/constructs/project.ts index 67542ed3a..94207780b 100644 --- a/packages/cli/src/constructs/project.ts +++ b/packages/cli/src/constructs/project.ts @@ -248,7 +248,7 @@ export class Session { static parsers = new Map() static constructExports: ConstructExport[] = [] static ignoreDirectoriesMatch: string[] = [] - static skipWebServerValidation?: boolean + static warnOnWebServerConfig?: boolean static packageManager: PackageManager = npmPackageManager static workspace: Result = Err(new Error(`Workspace support not initialized`)) diff --git a/packages/cli/src/services/project-parser.ts b/packages/cli/src/services/project-parser.ts index 3dab298cb..7fd904163 100644 --- a/packages/cli/src/services/project-parser.ts +++ b/packages/cli/src/services/project-parser.ts @@ -47,7 +47,7 @@ type ProjectParseOpts = { include?: string | string[] playwrightChecks?: PlaywrightSlimmedProp[] ignoreCheckDefinitions?: boolean - skipWebServerValidation?: boolean + warnOnWebServerConfig?: boolean enableWorkspaces?: boolean } @@ -146,7 +146,7 @@ export async function parseProject (opts: ProjectParseOpts): Promise { include, playwrightChecks, ignoreCheckDefinitions, - skipWebServerValidation, + warnOnWebServerConfig, enableWorkspaces = true, } = opts const project = new Project(projectLogicalId, { @@ -182,7 +182,7 @@ export async function parseProject (opts: ProjectParseOpts): Promise { Session.defaultRuntimeId = defaultRuntimeId Session.verifyRuntimeDependencies = verifyRuntimeDependencies ?? true Session.ignoreDirectoriesMatch = ignoreDirectoriesMatch - Session.skipWebServerValidation = skipWebServerValidation + Session.warnOnWebServerConfig = warnOnWebServerConfig Session.packageManager = packageManager Session.workspace = workspace From 1c62dfa283242d05f52215e85308895cec6922d6 Mon Sep 17 00:00:00 2001 From: miliberlin Date: Tue, 17 Feb 2026 17:49:50 +0100 Subject: [PATCH 5/6] refactor: rename flag to loadPlaywrightChecksOnly and skip all check discovery Consolidate two flags (skipCheckFileLoading, filterPlaywrightFromConfig) into loadPlaywrightChecksOnly. Also skip browser and multi-step check loading under this flag since they get filtered out anyway. Co-Authored-By: Claude Opus 4.6 --- .../cli/src/commands/debug/parse-project.ts | 2 +- packages/cli/src/commands/pw-test.ts | 2 +- packages/cli/src/services/project-parser.ts | 17 ++++++++--------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/commands/debug/parse-project.ts b/packages/cli/src/commands/debug/parse-project.ts index 61050c11b..6ee8a1e2a 100644 --- a/packages/cli/src/commands/debug/parse-project.ts +++ b/packages/cli/src/commands/debug/parse-project.ts @@ -108,7 +108,7 @@ export default class ParseProjectCommand extends Command { playwrightConfigPath: checklyConfig.checks?.playwrightConfigPath, include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, playwrightChecks: checklyConfig.checks?.playwrightChecks, - ignoreCheckDefinitions: emulatePwTest, + loadPlaywrightChecksOnly: emulatePwTest, warnOnWebServerConfig: emulatePwTest && !(includeFlag.length > 0), }) diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts index 069d888ec..970d7cbfd 100644 --- a/packages/cli/src/commands/pw-test.ts +++ b/packages/cli/src/commands/pw-test.ts @@ -191,7 +191,7 @@ export default class PwTestCommand extends AuthCommand { playwrightConfigPath, include: includeFlag.length ? includeFlag : checklyConfig.checks?.include, playwrightChecks: [playwrightCheck], - ignoreCheckDefinitions: true, + loadPlaywrightChecksOnly: true, warnOnWebServerConfig: !(includeFlag.length > 0), checkFilter: check => { // Skip non Playwright checks diff --git a/packages/cli/src/services/project-parser.ts b/packages/cli/src/services/project-parser.ts index 7fd904163..fa9c10836 100644 --- a/packages/cli/src/services/project-parser.ts +++ b/packages/cli/src/services/project-parser.ts @@ -46,7 +46,7 @@ type ProjectParseOpts = { playwrightConfigPath?: string include?: string | string[] playwrightChecks?: PlaywrightSlimmedProp[] - ignoreCheckDefinitions?: boolean + loadPlaywrightChecksOnly?: boolean warnOnWebServerConfig?: boolean enableWorkspaces?: boolean } @@ -145,7 +145,7 @@ export async function parseProject (opts: ProjectParseOpts): Promise { playwrightConfigPath, include, playwrightChecks, - ignoreCheckDefinitions, + loadPlaywrightChecksOnly, warnOnWebServerConfig, enableWorkspaces = true, } = opts @@ -165,7 +165,7 @@ export async function parseProject (opts: ProjectParseOpts): Promise { ignoreWorkspaces: !enableWorkspaces, }) - const filteredConstructs = ignoreCheckDefinitions + const filteredConstructs = loadPlaywrightChecksOnly ? checklyConfigConstructs?.filter(c => !(c instanceof PlaywrightCheck)) : checklyConfigConstructs @@ -189,14 +189,13 @@ export async function parseProject (opts: ProjectParseOpts): Promise { // TODO: Do we really need all of the ** globs, or could we just put node_modules? const ignoreDirectories = ['**/node_modules/**', '**/.git/**', ...ignoreDirectoriesMatch] - if (!ignoreCheckDefinitions) { + if (!loadPlaywrightChecksOnly) { await loadAllCheckFiles(directory, checkMatch, ignoreDirectories) + // Load sequentially because otherwise Session.checkFileAbsolutePath and + // Session.checkFilePath are going to be subject to race conditions. + await loadAllBrowserChecks(directory, browserCheckMatch, ignoreDirectories, project) + await loadAllMultiStepChecks(directory, multiStepCheckMatch, ignoreDirectories, project) } - - // Load sequentially because otherwise Session.checkFileAbsolutePath and - // Session.checkFilePath are going to be subject to race conditions. - await loadAllBrowserChecks(directory, browserCheckMatch, ignoreDirectories, project) - await loadAllMultiStepChecks(directory, multiStepCheckMatch, ignoreDirectories, project) await loadPlaywrightChecks(directory, playwrightChecks, playwrightConfigPath, include) // private-location must be processed after all checks and groups are loaded. From 01e3ac2c83b26b4118ca8f3adb1a6b0d88d9e867 Mon Sep 17 00:00:00 2001 From: miliberlin Date: Wed, 18 Feb 2026 10:05:33 +0100 Subject: [PATCH 6/6] fix: reset warnOnWebServerConfig in Session.reset() Co-Authored-By: Claude Opus 4.6 --- packages/cli/src/constructs/project.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/src/constructs/project.ts b/packages/cli/src/constructs/project.ts index 94207780b..ad013e6bd 100644 --- a/packages/cli/src/constructs/project.ts +++ b/packages/cli/src/constructs/project.ts @@ -271,6 +271,7 @@ export class Session { this.parsers = new Map() this.constructExports = [] this.ignoreDirectoriesMatch = [] + this.warnOnWebServerConfig = false this.packageManager = npmPackageManager this.workspace = Err(new Error(`Workspace support not initialized`)) this.resetSharedFiles()