From 5143d38cb8767a0951717e71f8ada6036e5e4149 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jan 2025 04:37:00 +0200 Subject: [PATCH 01/18] added result object, improved transport of tests --- .github/workflows/webdriver.yml | 2 +- examples/codecept.config.js | 26 +- examples/github_test.js | 83 +++--- examples/selenoid-example/browsers.json | 22 -- examples/selenoid-example/codecept.conf.js | 29 --- examples/selenoid-example/git_test.js | 16 -- lib/ai.js | 251 +++++++++--------- lib/codecept.js | 9 +- lib/command/run-workers.js | 38 +-- lib/command/workers/runTests.js | 213 ++------------- lib/container.js | 16 ++ lib/listener/artifacts.js | 19 -- lib/listener/result.js | 13 + lib/listener/steps.js | 5 - lib/listener/store.js | 4 + lib/mocha/asyncWrapper.js | 2 +- lib/mocha/cli.js | 53 ++-- lib/mocha/suite.js | 27 ++ lib/mocha/test.js | 74 +++++- lib/mocha/types.d.ts | 2 + lib/plugin/analyze.js | 290 +++++++++++++++++++++ lib/plugin/customReporter.js | 34 +++ lib/plugin/debugErrors.js | 67 ----- lib/plugin/pageInfo.js | 145 +++++++++++ lib/plugin/screenshotOnFail.js | 6 +- lib/result.js | 94 +++++++ lib/step/base.js | 50 +++- lib/utils.js | 8 + lib/workers.js | 57 ++-- test/runner/interface_test.js | 1 - 30 files changed, 1024 insertions(+), 632 deletions(-) delete mode 100644 examples/selenoid-example/browsers.json delete mode 100644 examples/selenoid-example/codecept.conf.js delete mode 100644 examples/selenoid-example/git_test.js delete mode 100644 lib/listener/artifacts.js create mode 100644 lib/listener/result.js create mode 100644 lib/plugin/analyze.js create mode 100644 lib/plugin/customReporter.js delete mode 100644 lib/plugin/debugErrors.js create mode 100644 lib/plugin/pageInfo.js create mode 100644 lib/result.js diff --git a/.github/workflows/webdriver.yml b/.github/workflows/webdriver.yml index 98cef5bbd..74e1a9882 100644 --- a/.github/workflows/webdriver.yml +++ b/.github/workflows/webdriver.yml @@ -38,7 +38,7 @@ jobs: PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true - name: start a server run: 'php -S 127.0.0.1:8000 -t test/data/app &' - - name: Check CodeceptJS can be started + - name: check run: './bin/codecept.js check -c test/acceptance/codecept.WebDriver.js' - name: run unit tests run: ./node_modules/.bin/mocha test/helper/WebDriver_test.js --exit diff --git a/examples/codecept.config.js b/examples/codecept.config.js index 2f4b03d64..9223a3a77 100644 --- a/examples/codecept.config.js +++ b/examples/codecept.config.js @@ -1,4 +1,4 @@ -require('./heal_recipes'); +require('./heal_recipes') exports.config = { output: './output', @@ -34,22 +34,24 @@ exports.config = { }, gherkin: { features: './features/*.feature', - steps: [ - './step_definitions/steps.js', - ], + steps: ['./step_definitions/steps.js'], }, plugins: { tryTo: { enabled: true, }, - heal: { + analyze: { enabled: true, }, + // heal: { + // enabled: true, + // }, + // customReporter: { + // enabled: true, + // }, wdio: { enabled: false, - services: [ - 'selenium-standalone', - ], + services: ['selenium-standalone'], }, stepByStepReport: {}, autoDelay: { @@ -65,6 +67,7 @@ exports.config = { enabled: true, }, }, + tests: './*_test.js', // timeout: 100, multiple: { @@ -73,11 +76,8 @@ exports.config = { }, default: { grep: 'signin', - browsers: [ - 'chrome', - 'firefox', - ], + browsers: ['chrome', 'firefox'], }, }, name: 'tests', -}; +} diff --git a/examples/github_test.js b/examples/github_test.js index e8f274c21..a2c66fa18 100644 --- a/examples/github_test.js +++ b/examples/github_test.js @@ -1,36 +1,36 @@ // / -Feature('GitHub'); +Feature('GitHub') Before(({ I }) => { - I.amOnPage('https://github.com'); -}); + I.amOnPage('https://github.com') + I.see('GitLab') +}) xScenario('test ai features', ({ I }) => { - I.amOnPage('https://getbootstrap.com/docs/5.1/examples/checkout/'); - pause(); -}); + I.amOnPage('https://getbootstrap.com/docs/5.1/examples/checkout/') +}) Scenario('Incorrect search for Codeceptjs', ({ I }) => { - I.fillField('.search-input', 'CodeceptJS'); - I.pressKey('Enter'); - I.waitForElement('[data-testid=search-sub-header]', 10); - I.see('Supercharged End 2 End Testing'); -}); + I.fillField('.search-input', 'CodeceptJS') + I.pressKey('Enter') + I.waitForElement('[data-testid=search-sub-header]', 10) + I.see('Supercharged End 2 End Testing') +}) Scenario('Visit Home Page @retry', async ({ I }) => { // .retry({ retries: 3, minTimeout: 1000 }) - I.retry(2).see('GitHub'); - I.retry(3).see('ALL'); - I.retry(2).see('IMAGES'); -}); + I.retry(2).see('GitHub') + I.retry(3).see('ALL') + I.retry(2).see('IMAGES') +}) Scenario('search @grop', { timeout: 6 }, ({ I }) => { - I.amOnPage('https://github.com/search'); + I.amOnPage('https://github.com/search') const a = { b: { c: 'asdasdasd', }, - }; + } const b = { users: { admin: { @@ -42,35 +42,38 @@ Scenario('search @grop', { timeout: 6 }, ({ I }) => { other: (world = '') => `Hello ${world}`, }, urls: {}, - }; - I.fillField('Search GitHub', 'CodeceptJS'); + } + I.fillField('Search GitHub', 'CodeceptJS') // pause({ a, b }); - I.pressKey('Enter'); - I.wait(3); + I.pressKey('Enter') + I.wait(3) // pause(); - I.see('Codeception/CodeceptJS', locate('.repo-list .repo-list-item').first()); -}); + I.see('Codeception/CodeceptJS', locate('.repo-list .repo-list-item').first()) +}) Scenario('signin @sign', { timeout: 6 }, ({ I, loginPage }) => { - I.say('it should not enter'); - loginPage.login('something@totest.com', '123456'); - I.see('Incorrect username or password.', '.flash-error'); -}).tag('normal').tag('important').tag('@slow'); + I.say('it should not enter') + loginPage.login('something@totest.com', '123456') + I.see('Incorrect username or password.', '.flash-error') +}) + .tag('normal') + .tag('important') + .tag('@slow') Scenario('signin2', { timeout: 1 }, ({ I, Smth }) => { - Smth.openAndLogin(); - I.see('Incorrect username or password.', '.flash-error'); -}); + Smth.openAndLogin() + I.see('Incorrect username or password.', '.flash-error') +}) Scenario('register', ({ I }) => { within('.js-signup-form', () => { - I.fillField('user[login]', 'User'); - I.fillField('user[email]', 'user@user.com'); - I.fillField('user[password]', 'user@user.com'); - I.fillField('q', 'aaa'); - I.click('button'); - }); - I.see('There were problems creating your account.'); - I.click('Explore'); - I.seeInCurrentUrl('/explore'); -}); + I.fillField('user[login]', 'User') + I.fillField('user[email]', 'user@user.com') + I.fillField('user[password]', 'user@user.com') + I.fillField('q', 'aaa') + I.click('button') + }) + I.see('There were problems creating your account.') + I.click('Explore') + I.seeInCurrentUrl('/explore') +}) diff --git a/examples/selenoid-example/browsers.json b/examples/selenoid-example/browsers.json deleted file mode 100644 index d715f44cc..000000000 --- a/examples/selenoid-example/browsers.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "chrome": { - "default": "latest", - "versions": { - "latest": { - "image": "selenoid/chrome:latest", - "port": "4444", - "path": "/" - } - } - }, - "firefox": { - "default": "latest", - "versions": { - "latest": { - "image": "selenoid/firefox:latest", - "port": "4444", - "path": "/wd/hub" - } - } - } -} \ No newline at end of file diff --git a/examples/selenoid-example/codecept.conf.js b/examples/selenoid-example/codecept.conf.js deleted file mode 100644 index 59666c7e1..000000000 --- a/examples/selenoid-example/codecept.conf.js +++ /dev/null @@ -1,29 +0,0 @@ -exports.config = { - tests: './*_test.js', - output: './output', - helpers: { - WebDriver: { - url: 'http://localhost', - browser: 'chrome', - }, - }, - - plugins: { - selenoid: { - enabled: true, - deletePassed: true, - autoCreate: true, - autoStart: true, - sessionTimeout: '30m', - enableVideo: true, - enableLog: true, - }, - allure: { - enabled: false, - }, - }, - include: {}, - bootstrap: null, - mocha: {}, - name: 'example', -}; diff --git a/examples/selenoid-example/git_test.js b/examples/selenoid-example/git_test.js deleted file mode 100644 index 1727c1bdc..000000000 --- a/examples/selenoid-example/git_test.js +++ /dev/null @@ -1,16 +0,0 @@ -Feature('Git'); - -Scenario('Demo Test Github', ({ I }) => { - I.amOnPage('https://github.com/login'); - I.see('GitHub'); - I.fillField('login', 'randomuser_kmk'); - I.fillField('password', 'randomuser_kmk'); - I.click('Sign in'); - I.see('Repositories'); -}); - -Scenario('Demo Test GitLab', ({ I }) => { - I.amOnPage('https://gitlab.com'); - I.dontSee('GitHub'); - I.see('GitLab'); -}); diff --git a/lib/ai.js b/lib/ai.js index 86dffcd3b..2104dafec 100644 --- a/lib/ai.js +++ b/lib/ai.js @@ -1,40 +1,44 @@ -const debug = require('debug')('codeceptjs:ai'); -const output = require('./output'); -const event = require('./event'); -const { removeNonInteractiveElements, minifyHtml, splitByChunks } = require('./html'); +const debug = require('debug')('codeceptjs:ai') +const output = require('./output') +const event = require('./event') +const { removeNonInteractiveElements, minifyHtml, splitByChunks } = require('./html') const defaultHtmlConfig = { maxLength: 50000, simplify: true, minify: true, html: {}, -}; +} const defaultPrompts = { - writeStep: (html, input) => [{ - role: 'user', - content: `I am test engineer writing test in CodeceptJS + writeStep: (html, input) => [ + { + role: 'user', + content: `I am test engineer writing test in CodeceptJS I have opened web page and I want to use CodeceptJS to ${input} on this page Provide me valid CodeceptJS code to accomplish it Use only locators from this HTML: \n\n${html}`, - }, + }, ], healStep: (html, { step, error, prevSteps }) => { - return [{ - role: 'user', - content: `As a test automation engineer I am testing web application using CodeceptJS. + return [ + { + role: 'user', + content: `As a test automation engineer I am testing web application using CodeceptJS. I want to heal a test that fails. Here is the list of executed steps: ${prevSteps.map(s => s.toString()).join(', ')} Propose how to adjust ${step.toCode()} step to fix the test. Use locators in order of preference: semantic locator by text, CSS, XPath. Use codeblocks marked with \`\`\` Here is the error message: ${error.message} Here is HTML code of a page where the failure has happened: \n\n${html}`, - }]; + }, + ] }, - generatePageObject: (html, extraPrompt = '', rootLocator = null) => [{ - role: 'user', - content: `As a test automation engineer I am creating a Page Object for a web application using CodeceptJS. + generatePageObject: (html, extraPrompt = '', rootLocator = null) => [ + { + role: 'user', + content: `As a test automation engineer I am creating a Page Object for a web application using CodeceptJS. Here is an sample page object: const { I } = inject(); @@ -60,72 +64,73 @@ module.exports = { ${extraPrompt} ${rootLocator ? `All provided elements are inside '${rootLocator}'. Declare it as root variable and for every locator use locate(...).inside(root)` : ''} Add only locators from this HTML: \n\n${html}`, - }], -}; + }, + ], +} class AiAssistant { constructor() { - this.totalTime = 0; - this.numTokens = 0; + this.totalTime = 0 + this.numTokens = 0 - this.reset(); - this.connectToEvents(); + this.reset() + this.connectToEvents() } enable(config = {}) { - debug('Enabling AI assistant'); - this.isEnabled = true; + debug('Enabling AI assistant') + this.isEnabled = true - const { html, prompts, ...aiConfig } = config; + const { html, prompts, ...aiConfig } = config - this.config = Object.assign(this.config, aiConfig); - this.htmlConfig = Object.assign(defaultHtmlConfig, html); - this.prompts = Object.assign(defaultPrompts, prompts); + this.config = Object.assign(this.config, aiConfig) + this.htmlConfig = Object.assign(defaultHtmlConfig, html) + this.prompts = Object.assign(defaultPrompts, prompts) - debug('Config', this.config); + debug('Config', this.config) } reset() { - this.numTokens = 0; - this.isEnabled = false; + this.numTokens = 0 + this.isEnabled = false this.config = { maxTokens: 1000000, request: null, response: parseCodeBlocks, // lets limit token usage to 1M - }; - this.minifiedHtml = null; - this.response = null; - this.totalTime = 0; + } + this.minifiedHtml = null + this.response = null + this.totalTime = 0 } disable() { - this.isEnabled = false; + this.isEnabled = false } connectToEvents() { event.dispatcher.on(event.all.result, () => { if (this.isEnabled && this.numTokens > 0) { - const numTokensK = Math.ceil(this.numTokens / 1000); - const maxTokensK = Math.ceil(this.config.maxTokens / 1000); - output.print(`AI assistant took ${this.totalTime}s and used ~${numTokensK}K input tokens. Tokens limit: ${maxTokensK}K`); + const numTokensK = Math.ceil(this.numTokens / 1000) + const maxTokensK = Math.ceil(this.config.maxTokens / 1000) + output.print(`AI assistant took ${this.totalTime}s and used ~${numTokensK}K input tokens. Tokens limit: ${maxTokensK}K`) } - }); + }) } checkRequestFn() { if (!this.isEnabled) { - debug('AI assistant is disabled'); - return; + debug('AI assistant is disabled') + return } - if (this.config.request) return; + if (this.config.request) return const noRequestErrorMessage = ` - No request function is set for AI assistant. - Please implement your own request function and set it in the config. + No request function is set for AI assistant. - [!] AI request was decoupled from CodeceptJS. To connect to OpenAI or other AI service, please implement your own request function and set it in the config. + [!] AI request was decoupled from CodeceptJS. To connect to OpenAI or other AI service. + Please implement your own request function and set it in the config. Example (connect to OpenAI): @@ -134,82 +139,80 @@ class AiAssistant { const OpenAI = require('openai'); const openai = new OpenAI({ apiKey: process.env['OPENAI_API_KEY'] }) const response = await openai.chat.completions.create({ - model: 'gpt-3.5-turbo-0125', + model: 'gpt-4o-mini', messages, }); return response?.data?.choices[0]?.message?.content; } } - `.trim(); + `.trim() - throw new Error(noRequestErrorMessage); + throw new Error(noRequestErrorMessage) } async setHtmlContext(html) { - let processedHTML = html; + let processedHTML = html if (this.htmlConfig.simplify) { - processedHTML = removeNonInteractiveElements(processedHTML, this.htmlConfig); + processedHTML = removeNonInteractiveElements(processedHTML, this.htmlConfig) } - if (this.htmlConfig.minify) processedHTML = await minifyHtml(processedHTML); - if (this.htmlConfig.maxLength) processedHTML = splitByChunks(processedHTML, this.htmlConfig.maxLength)[0]; + if (this.htmlConfig.minify) processedHTML = await minifyHtml(processedHTML) + if (this.htmlConfig.maxLength) processedHTML = splitByChunks(processedHTML, this.htmlConfig.maxLength)[0] - this.minifiedHtml = processedHTML; + this.minifiedHtml = processedHTML } getResponse() { - return this.response || ''; + return this.response || '' } async createCompletion(messages) { - if (!this.isEnabled) return ''; - - debug('Request', messages); - - this.checkRequestFn(); - - this.response = null; - - this.calculateTokens(messages); + if (!this.isEnabled) return '' try { - const startTime = process.hrtime(); - this.response = await this.config.request(messages); - const endTime = process.hrtime(startTime); - const executionTimeInSeconds = endTime[0] + endTime[1] / 1e9; - - this.totalTime += Math.round(executionTimeInSeconds); - debug('AI response time', executionTimeInSeconds); - debug('Response', this.response); - this.stopWhenReachingTokensLimit(); - return this.response; + this.checkRequestFn() + debug('Request', messages) + + this.response = null + + this.calculateTokens(messages) + const startTime = process.hrtime() + this.response = await this.config.request(messages) + const endTime = process.hrtime(startTime) + const executionTimeInSeconds = endTime[0] + endTime[1] / 1e9 + + this.totalTime += Math.round(executionTimeInSeconds) + debug('AI response time', executionTimeInSeconds) + debug('Response', this.response) + this.stopWhenReachingTokensLimit() + return this.response } catch (err) { - debug(err.response); - output.print(''); - output.error(`AI service error: ${err.message}`); - if (err?.response?.data?.error?.code) output.error(err?.response?.data?.error?.code); - if (err?.response?.data?.error?.message) output.error(err?.response?.data?.error?.message); - this.stopWhenReachingTokensLimit(); - return ''; + debug(err.response) + output.print('') + output.error(`AI service error: ${err.message}`) + if (err?.response?.data?.error?.code) output.error(err?.response?.data?.error?.code) + if (err?.response?.data?.error?.message) output.error(err?.response?.data?.error?.message) + this.stopWhenReachingTokensLimit() + return '' } } async healFailedStep(failureContext) { - if (!this.isEnabled) return []; - if (!failureContext.html) throw new Error('No HTML context provided'); + if (!this.isEnabled) return [] + if (!failureContext.html) throw new Error('No HTML context provided') - await this.setHtmlContext(failureContext.html); + await this.setHtmlContext(failureContext.html) if (!this.minifiedHtml) { - debug('HTML context is empty after removing non-interactive elements & minification'); - return []; + debug('HTML context is empty after removing non-interactive elements & minification') + return [] } - const response = await this.createCompletion(this.prompts.healStep(this.minifiedHtml, failureContext)); - if (!response) return []; + const response = await this.createCompletion(this.prompts.healStep(this.minifiedHtml, failureContext)) + if (!response) return [] - return this.config.response(response); + return this.config.response(response) } /** @@ -219,13 +222,13 @@ class AiAssistant { * @returns */ async generatePageObject(extraPrompt = null, locator = null) { - if (!this.isEnabled) return []; - if (!this.minifiedHtml) throw new Error('No HTML context provided'); + if (!this.isEnabled) return [] + if (!this.minifiedHtml) throw new Error('No HTML context provided') - const response = await this.createCompletion(this.prompts.generatePageObject(this.minifiedHtml, locator, extraPrompt)); - if (!response) return []; + const response = await this.createCompletion(this.prompts.generatePageObject(this.minifiedHtml, locator, extraPrompt)) + if (!response) return [] - return this.config.response(response); + return this.config.response(response) } calculateTokens(messages) { @@ -233,66 +236,72 @@ class AiAssistant { // this approach was tested via https://platform.openai.com/tokenizer // we need it to display current tokens usage so users could analyze effectiveness of AI - const inputString = messages.map(m => m.content).join(' ').trim(); - const numWords = (inputString.match(/[^\s\-:=]+/g) || []).length; + const inputString = messages + .map(m => m.content) + .join(' ') + .trim() + const numWords = (inputString.match(/[^\s\-:=]+/g) || []).length // 2.5 token is constant for average HTML input - const tokens = numWords * 2.5; + const tokens = numWords * 2.5 - this.numTokens += tokens; + this.numTokens += tokens - return tokens; + return tokens } stopWhenReachingTokensLimit() { - if (this.numTokens < this.config.maxTokens) return; + if (this.numTokens < this.config.maxTokens) return - output.print(`AI assistant has reached the limit of ${this.config.maxTokens} tokens in this session. It will be disabled now`); - this.disable(); + output.print(`AI assistant has reached the limit of ${this.config.maxTokens} tokens in this session. It will be disabled now`) + this.disable() } async writeSteps(input) { - if (!this.isEnabled) return; - if (!this.minifiedHtml) throw new Error('No HTML context provided'); + if (!this.isEnabled) return + if (!this.minifiedHtml) throw new Error('No HTML context provided') - const snippets = []; + const snippets = [] - const response = await this.createCompletion(this.prompts.writeStep(this.minifiedHtml, input)); - if (!response) return; - snippets.push(...this.config.response(response)); + const response = await this.createCompletion(this.prompts.writeStep(this.minifiedHtml, input)) + if (!response) return + snippets.push(...this.config.response(response)) - debug(snippets[0]); + debug(snippets[0]) - return snippets[0]; + return snippets[0] } } function parseCodeBlocks(response) { // Regular expression pattern to match code snippets - const codeSnippetPattern = /```(?:javascript|js|typescript|ts)?\n([\s\S]+?)\n```/g; + const codeSnippetPattern = /```(?:javascript|js|typescript|ts)?\n([\s\S]+?)\n```/g // Array to store extracted code snippets - const codeSnippets = []; + const codeSnippets = [] - response = response.split('\n').map(line => line.trim()).join('\n'); + response = response + .split('\n') + .map(line => line.trim()) + .join('\n') // Iterate over matches and extract code snippets - let match; + let match while ((match = codeSnippetPattern.exec(response)) !== null) { - codeSnippets.push(match[1]); + codeSnippets.push(match[1]) } // Remove "Scenario", "Feature", and "require()" lines const modifiedSnippets = codeSnippets.map(snippet => { - const lines = snippet.split('\n'); + const lines = snippet.split('\n') - const filteredLines = lines.filter(line => !line.includes('I.amOnPage') && !line.startsWith('Scenario') && !line.startsWith('Feature') && !line.includes('= require(')); + const filteredLines = lines.filter(line => !line.includes('I.amOnPage') && !line.startsWith('Scenario') && !line.startsWith('Feature') && !line.includes('= require(')) - return filteredLines.join('\n'); + return filteredLines.join('\n') // remove snippets that move from current url - }); // .filter(snippet => !line.includes('I.amOnPage')); + }) // .filter(snippet => !line.includes('I.amOnPage')); - return modifiedSnippets.filter(snippet => !!snippet); + return modifiedSnippets.filter(snippet => !!snippet) } -module.exports = new AiAssistant(); +module.exports = new AiAssistant() diff --git a/lib/codecept.js b/lib/codecept.js index f0626cbcc..a3e61744a 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -9,6 +9,7 @@ const event = require('./event') const runHook = require('./hooks') const output = require('./output') const { emptyFolder } = require('./utils') +const Result = require('./result') /** * CodeceptJS runner @@ -105,8 +106,8 @@ class Codecept { // default hooks runHook(require('./listener/store')) runHook(require('./listener/steps')) - runHook(require('./listener/artifacts')) runHook(require('./listener/config')) + runHook(require('./listener/result')) runHook(require('./listener/helpers')) runHook(require('./listener/globalTimeout')) runHook(require('./listener/globalRetry')) @@ -199,13 +200,13 @@ class Codecept { mocha.files = mocha.files.filter(t => fsPath.basename(t, '.js') === test || t === test) } const done = () => { - event.emit(event.all.result, this) - event.emit(event.all.after, this) + event.emit(event.all.result, container.result()) + event.emit(event.all.after, container.result()) resolve() } try { - event.emit(event.all.before, this) + event.emit(event.all.before, container.result()) mocha.run(() => done()) } catch (e) { output.error(e.stack) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 31a2c598e..4d597f7b0 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -35,18 +35,6 @@ module.exports = async function (workerCount, selectedRuns, options) { const workers = new Workers(numberOfWorkers, config) workers.overrideConfig(overrideConfigs) - workers.on(event.suite.before, suite => { - suiteArr.push(suite) - }) - - workers.on(event.step.passed, step => { - stepArr.push(step) - }) - - workers.on(event.step.failed, step => { - stepArr.push(step) - }) - workers.on(event.test.failed, test => { failedTestArr.push(test) output.test.failed(test) @@ -62,31 +50,9 @@ module.exports = async function (workerCount, selectedRuns, options) { output.test.skipped(test) }) - workers.on(event.all.result, () => { - // expose test stats after all workers finished their execution - function addStepsToTest(test, stepArr) { - stepArr.test.steps.forEach(step => { - if (test.steps.length === 0) { - test.steps.push(step) - } - }) - } - - stepArr.forEach(step => { - passedTestArr.forEach(test => { - if (step.test.title === test.title) { - addStepsToTest(test, step) - } - }) - - failedTestArr.forEach(test => { - if (step.test.title === test.title) { - addStepsToTest(test, step) - } - }) - }) - + workers.on(event.all.result, result => { event.dispatcher.emit(event.workers.result, { + ...result?.simplify(), suites: suiteArr, tests: { passed: passedTestArr, diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index f32725d35..09900f075 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -75,217 +75,52 @@ function filterTests() { } function initializeListeners() { - function simplifyError(error) { - if (error) { - const { stack, uncaught, message, actual, expected } = error - - return { - stack, - uncaught, - message, - actual, - expected, - } - } - - return null - } - function simplifyTest(test, err = null) { - test = { ...test } - - if (test.start && !test.duration) { - const end = new Date() - test.duration = end - test.start - } - - if (test.err) { - err = simplifyError(test.err) - test.status = 'failed' - } else if (err) { - err = simplifyError(err) - test.status = 'failed' - } - const parent = {} - if (test.parent) { - parent.title = test.parent.title - } - - if (test.opts) { - Object.keys(test.opts).forEach(k => { - if (typeof test.opts[k] === 'object') delete test.opts[k] - if (typeof test.opts[k] === 'function') delete test.opts[k] - }) - } - - return { - opts: test.opts || {}, - tags: test.tags || [], - uid: test.uid, - workerIndex, - retries: test._retries, - title: test.title, - status: test.status, - notes: test.notes || [], - meta: test.meta || {}, - artifacts: test.artifacts || [], - duration: test.duration || 0, - err, - parent, - steps: test.steps && test.steps.length > 0 ? simplifyStepsInTestObject(test.steps, err) : [], - } - } - - function simplifyStepsInTestObject(steps, err) { - steps = [...steps] - const _steps = [] - - for (step of steps) { - const _args = [] - - if (step.args) { - for (const arg of step.args) { - // check if arg is a JOI object - if (arg && arg.$_root) { - _args.push(JSON.stringify(arg).slice(0, 300)) - // check if arg is a function - } else if (arg && typeof arg === 'function') { - _args.push(arg.name) - } else { - _args.push(arg) - } - } - } - - _steps.push({ - actor: step.actor, - name: step.name, - status: step.status, - args: JSON.stringify(_args), - startedAt: step.startedAt, - startTime: step.startTime, - endTime: step.endTime, - finishedAt: step.finishedAt, - duration: step.duration, - err, - }) - } - - return _steps - } - - function simplifyStep(step, err = null) { - step = { ...step } - - if (step.startTime && !step.duration) { - const end = new Date() - step.duration = end - step.startTime - } - - if (step.err) { - err = simplifyError(step.err) - step.status = 'failed' - } else if (err) { - err = simplifyError(err) - step.status = 'failed' - } - - const parent = {} - if (step.metaStep) { - parent.title = step.metaStep.actor - } - - if (step.opts) { - Object.keys(step.opts).forEach(k => { - if (typeof step.opts[k] === 'object') delete step.opts[k] - if (typeof step.opts[k] === 'function') delete step.opts[k] - }) - } - - return { - opts: step.opts || {}, - workerIndex, - title: step.name, - status: step.status, - duration: step.duration || 0, - err, - parent, - test: simplifyTest(step.test), - } - } - - collectStats() // suite - event.dispatcher.on(event.suite.before, suite => sendToParentThread({ event: event.suite.before, workerIndex, data: simplifyTest(suite) })) - event.dispatcher.on(event.suite.after, suite => sendToParentThread({ event: event.suite.after, workerIndex, data: simplifyTest(suite) })) + event.dispatcher.on(event.suite.before, suite => sendToParentThread({ event: event.suite.before, workerIndex, data: suite.simplify() })) + event.dispatcher.on(event.suite.after, suite => sendToParentThread({ event: event.suite.after, workerIndex, data: suite.simplify() })) // calculate duration event.dispatcher.on(event.test.started, test => (test.start = new Date())) // tests - event.dispatcher.on(event.test.before, test => sendToParentThread({ event: event.test.before, workerIndex, data: simplifyTest(test) })) - event.dispatcher.on(event.test.after, test => sendToParentThread({ event: event.test.after, workerIndex, data: simplifyTest(test) })) + event.dispatcher.on(event.test.before, test => sendToParentThread({ event: event.test.before, workerIndex, data: test.simplify() })) + event.dispatcher.on(event.test.after, test => sendToParentThread({ event: event.test.after, workerIndex, data: test.simplify() })) // we should force-send correct errors to prevent race condition - event.dispatcher.on(event.test.finished, (test, err) => sendToParentThread({ event: event.test.finished, workerIndex, data: simplifyTest(test, err) })) - event.dispatcher.on(event.test.failed, (test, err) => sendToParentThread({ event: event.test.failed, workerIndex, data: simplifyTest(test, err) })) - event.dispatcher.on(event.test.passed, (test, err) => sendToParentThread({ event: event.test.passed, workerIndex, data: simplifyTest(test, err) })) - event.dispatcher.on(event.test.started, test => sendToParentThread({ event: event.test.started, workerIndex, data: simplifyTest(test) })) - event.dispatcher.on(event.test.skipped, test => sendToParentThread({ event: event.test.skipped, workerIndex, data: simplifyTest(test) })) + event.dispatcher.on(event.test.finished, (test, err) => sendToParentThread({ event: event.test.finished, workerIndex, data: { ...test.simplify(), err } })) + event.dispatcher.on(event.test.failed, (test, err) => sendToParentThread({ event: event.test.failed, workerIndex, data: { ...test.simplify(), err } })) + event.dispatcher.on(event.test.passed, (test, err) => sendToParentThread({ event: event.test.passed, workerIndex, data: { ...test.simplify(), err } })) + event.dispatcher.on(event.test.started, test => sendToParentThread({ event: event.test.started, workerIndex, data: test.simplify() })) + event.dispatcher.on(event.test.skipped, test => sendToParentThread({ event: event.test.skipped, workerIndex, data: test.simplify() })) // steps - event.dispatcher.on(event.step.finished, step => sendToParentThread({ event: event.step.finished, workerIndex, data: simplifyStep(step) })) - event.dispatcher.on(event.step.started, step => sendToParentThread({ event: event.step.started, workerIndex, data: simplifyStep(step) })) - event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: simplifyStep(step) })) - event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: simplifyStep(step) })) + event.dispatcher.on(event.step.finished, step => sendToParentThread({ event: event.step.finished, workerIndex, data: step.simplify() })) + event.dispatcher.on(event.step.started, step => sendToParentThread({ event: event.step.started, workerIndex, data: step.simplify() })) + event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: step.simplify() })) + event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: step.simplify() })) - event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: simplifyTest(test, err) })) - event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: simplifyTest(test, err) })) - event.dispatcher.on(event.all.failures, data => sendToParentThread({ event: event.all.failures, workerIndex, data })) + event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: { ...test.simplify(), err } })) + event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: { ...test.simplify(), err } })) + event.dispatcher.once(event.all.after, () => { + sendToParentThread({ event: event.all.after, workerIndex, data: container.result().simplify() }) + }) // all - event.dispatcher.once(event.all.result, () => parentPort.close()) + event.dispatcher.once(event.all.result, () => { + sendToParentThread({ event: event.all.result, workerIndex, data: container.result().simplify() }) + parentPort?.close() + }) } function disablePause() { global.pause = () => {} } -function collectStats() { - const stats = { - passes: 0, - failures: 0, - skipped: 0, - tests: 0, - pending: 0, - } - event.dispatcher.on(event.test.skipped, () => { - stats.skipped++ - }) - event.dispatcher.on(event.test.passed, () => { - stats.passes++ - }) - event.dispatcher.on(event.test.failed, test => { - if (test.ctx._runnable.title.includes('hook: AfterSuite')) { - stats.failedHooks += 1 - } - stats.failures++ - }) - event.dispatcher.on(event.test.skipped, () => { - stats.pending++ - }) - event.dispatcher.on(event.test.finished, () => { - stats.tests++ - }) - event.dispatcher.once(event.all.after, () => { - sendToParentThread({ event: event.all.after, data: stats }) - }) -} - function sendToParentThread(data) { - parentPort.postMessage(data) + parentPort?.postMessage(data) } function listenToParentThread() { - parentPort.on('message', eventData => { + parentPort?.on('message', eventData => { container.append({ support: eventData.data }) }) } diff --git a/lib/container.js b/lib/container.js index d8d25ebfd..6a041bb14 100644 --- a/lib/container.js +++ b/lib/container.js @@ -9,6 +9,7 @@ const recorder = require('./recorder') const event = require('./event') const WorkerStorage = require('./workerStorage') const store = require('./store') +const Result = require('./result') const ai = require('./ai') let asyncHelperPromise @@ -25,6 +26,8 @@ let container = { */ mocha: {}, translation: {}, + /** @type {Result | null} */ + result: null, } /** @@ -54,6 +57,7 @@ class Container { container.translation = loadTranslation(config.translation || null, config.vocabularies || []) container.proxySupport = createSupportObjects(config.include || {}) container.plugins = createPlugins(config.plugins || {}, opts) + container.result = new Result() createActor(config.include?.I) @@ -127,6 +131,18 @@ class Container { return container.mocha } + /** + * Get result + * + * @returns {Result} + */ + static result() { + if (!container.result) { + container.result = new Result() + } + return container.result + } + /** * Append new services to container * diff --git a/lib/listener/artifacts.js b/lib/listener/artifacts.js deleted file mode 100644 index da4cddeb7..000000000 --- a/lib/listener/artifacts.js +++ /dev/null @@ -1,19 +0,0 @@ -const event = require('../event') -const recorder = require('../recorder') - -/** - * Create and clean up empty artifacts - */ -module.exports = function () { - event.dispatcher.on(event.test.before, test => { - test.artifacts = {} - }) - - event.dispatcher.on(event.test.after, test => { - recorder.add('clean up empty artifacts', () => { - for (const key in test.artifacts || {}) { - if (!test.artifacts[key]) delete test.artifacts[key] - } - }) - }) -} diff --git a/lib/listener/result.js b/lib/listener/result.js new file mode 100644 index 000000000..fe9762017 --- /dev/null +++ b/lib/listener/result.js @@ -0,0 +1,13 @@ +const event = require('../event') +const container = require('../container') + +module.exports = function () { + event.dispatcher.on(event.hook.failed, err => { + container.result().addStats({ failedHooks: 1 }) + }) + + event.dispatcher.on(event.test.started, test => { + container.result().addStats({ tests: 1 }) + container.result().addTest(test) + }) +} diff --git a/lib/listener/steps.js b/lib/listener/steps.js index aa33d1e14..c7b5beb1c 100644 --- a/lib/listener/steps.js +++ b/lib/listener/steps.js @@ -71,8 +71,6 @@ module.exports = function () { }) event.dispatcher.on(event.step.started, step => { - step.startedAt = +new Date() - step.test = currentTest store.currentStep = step if (currentHook && Array.isArray(currentHook.steps)) { return currentHook.steps.push(step) @@ -82,9 +80,6 @@ module.exports = function () { }) event.dispatcher.on(event.step.finished, step => { - step.finishedAt = +new Date() - if (step.startedAt) step.duration = step.finishedAt - step.startedAt - debug(`Step '${step}' finished; Duration: ${step.duration || 0}ms`) store.currentStep = null store.stepOptions = null }) diff --git a/lib/listener/store.js b/lib/listener/store.js index 763aa1edc..73377e922 100644 --- a/lib/listener/store.js +++ b/lib/listener/store.js @@ -2,6 +2,10 @@ const event = require('../event') const store = require('../store') module.exports = function () { + event.dispatcher.on(event.all.before, result => { + store.result = result + }) + event.dispatcher.on(event.test.before, test => { store.currentTest = test }) diff --git a/lib/mocha/asyncWrapper.js b/lib/mocha/asyncWrapper.js index 68902a912..398ce2260 100644 --- a/lib/mocha/asyncWrapper.js +++ b/lib/mocha/asyncWrapper.js @@ -112,7 +112,7 @@ module.exports.injected = function (fn, suite, hookName) { const errHandler = err => { recorder.session.start('teardown') recorder.cleanAsyncErr() - event.emit(event.test.failed, suite, err) + if (hookName == 'before' || hookName == 'beforeSuite') suite.eachTest(test => event.emit(event.test.failed, test, err)) if (hookName === 'after') event.emit(event.test.after, suite) if (hookName === 'afterSuite') event.emit(event.suite.after, suite) recorder.add(() => doneFn(err)) diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index 64fd72d49..365d618af 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -1,12 +1,12 @@ const { reporters: { Base }, } = require('mocha') -const figures = require('figures') const ms = require('ms') +const figures = require('figures') const event = require('../event') const AssertionFailedError = require('../assert/error') const output = require('../output') - +const { cloneTest } = require('./test') const cursor = Base.cursor let currentMetaStep = [] let codeceptjsEventDispatchersRegistered = false @@ -32,6 +32,16 @@ class Cli extends Base { output.print(output.styles.debug(`Plugins: ${Object.keys(Containter.plugins()).join(', ')}`)) } + if (level >= 3) { + process.on('warning', warning => { + console.log('\nWarning Details:') + console.log('Name:', warning.name) + console.log('Message:', warning.message) + console.log('Stack:', warning.stack) + console.log('-------------------') + }) + } + runner.on('start', () => { console.log() }) @@ -108,9 +118,6 @@ class Cli extends Base { } currentMetaStep = metaSteps output.stepShift = 3 + 2 * shift - if (step.helper.constructor.name !== 'ExpectHelper') { - output.step(step) - } }) event.dispatcher.on(event.step.finished, () => { @@ -138,16 +145,18 @@ class Cli extends Base { } } - this.stats.pending += skippedCount - this.stats.tests += skippedCount + const container = require('../container') + container.result().addStats({ pending: skippedCount, tests: skippedCount }) }) runner.on('end', this.result.bind(this)) } result() { + const container = require('../container') const stats = this.stats - stats.failedHooks = 0 + container.result().addStats(stats) + container.result().finish() console.log() // passes @@ -160,7 +169,8 @@ class Cli extends Base { // failures if (stats.failures) { // append step traces - this.failures.map(test => { + this.failures = this.failures.map(test => { + // we will change the stack trace, so we need to clone the test const err = test.err let log = '' @@ -204,7 +214,7 @@ class Cli extends Base { try { let stack = err.stack - stack = stack.replace(originalMessage, '') + stack = (stack || '').replace(originalMessage, '') stack = stack ? stack.split('\n') : [] if (stack[0] && stack[0].includes(err.message)) { @@ -215,14 +225,15 @@ class Cli extends Base { stack = stack.slice(0, 3) } - err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}` - - // clone err object so stack trace adjustments won't affect test other reports - test.err = err - return test + err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}`.trim() } catch (e) { - throw Error(e) + console.error(e) } + + // we will change the stack trace, so we need to clone the test + test = cloneTest(test) + test.err = err + return test }) const originalLog = Base.consoleLog @@ -235,13 +246,9 @@ class Cli extends Base { console.log() } - this.failures.forEach(failure => { - if (failure.constructor.name === 'Hook') { - stats.failedHooks += 1 - } - }) - event.emit(event.all.failures, { failuresLog, stats }) - output.result(stats.passes, stats.failures, stats.pending, ms(stats.duration), stats.failedHooks) + container.result().addFailures(failuresLog) + + output.result(stats.passes, stats.failures, stats.pending, ms(stats.duration)) if (stats.failures && output.level() < 3) { output.print(output.styles.debug('Run with --verbose flag to see complete NodeJS stacktrace')) diff --git a/lib/mocha/suite.js b/lib/mocha/suite.js index be96439d6..7282f930b 100644 --- a/lib/mocha/suite.js +++ b/lib/mocha/suite.js @@ -34,6 +34,10 @@ function enhanceMochaSuite(suite) { } } + suite.simplify = function () { + return serializeSuite(this) + } + return suite } @@ -49,7 +53,30 @@ function createSuite(parent, title) { return enhanceMochaSuite(suite) } +function serializeSuite(suite) { + suite = { ...suite } + + return { + opts: suite.opts || {}, + tags: suite.tags || [], + retries: suite._retries, + title: suite.title, + status: suite.status, + notes: suite.notes || [], + meta: suite.meta || {}, + duration: suite.duration || 0, + } +} + +function deserializeSuite(suite) { + suite = Object.assign(new MochaSuite(suite.title), suite) + enhanceMochaSuite(suite) + return suite +} + module.exports = { createSuite, enhanceMochaSuite, + serializeSuite, + deserializeSuite, } diff --git a/lib/mocha/test.js b/lib/mocha/test.js index 5c326a346..6c2769da1 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -2,7 +2,7 @@ const Test = require('mocha/lib/test') const Suite = require('mocha/lib/suite') const { test: testWrapper } = require('./asyncWrapper') const { enhanceMochaSuite } = require('./suite') -const { genTestId } = require('../utils') +const { genTestId, serializeError, clearString } = require('../utils') /** * Factory function to create enhanced tests @@ -51,6 +51,18 @@ function enhanceMochaTest(test) { test.uid = genTestId(test) } + test.toFileName = function () { + let fileName = clearString(test.title) + if (fileName.indexOf('{') !== -1) { + fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim() + } + // TODO: add suite title to file name + // if (test.parent && test.parent.title) { + // fileName = `${clearString(test.parent.title)}_${fileName}` + // } + return fileName + } + test.applyOptions = function (opts) { if (!opts) opts = {} test.opts = opts @@ -59,17 +71,73 @@ function enhanceMochaTest(test) { if (opts.retries) this.retries(opts.retries) } + test.simplify = function () { + return serializeTest(this) + } + return test } -function repackTestForWorkersTransport(test) { +function deserializeTest(test) { test = Object.assign(new Test(test.title || '', () => {}), test) test.parent = Object.assign(new Suite(test.parent.title), test.parent) + enhanceMochaTest(test) + enhanceMochaSuite(test.parent) return test } +function serializeTest(test, err = null) { + test = { ...test } + + if (test.start && !test.duration) { + const end = +new Date() + test.duration = end - test.start + } + + if (test.err) { + err = serializeError(test.err) + test.status = 'failed' + } else if (err) { + err = serializeError(err) + test.status = 'failed' + } + const parent = {} + if (test.parent) { + parent.title = test.parent.title + } + + if (test.opts) { + Object.keys(test.opts).forEach(k => { + if (typeof test.opts[k] === 'object') delete test.opts[k] + if (typeof test.opts[k] === 'function') delete test.opts[k] + }) + } + + return { + opts: test.opts || {}, + tags: test.tags || [], + uid: test.uid, + retries: test._retries, + title: test.title, + status: test.status, + notes: test.notes || [], + meta: test.meta || {}, + artifacts: test.artifacts || [], + duration: test.duration || 0, + err, + parent, + steps: [...test.steps].map(step => step.simplify()), + } +} + +function cloneTest(test) { + return deserializeTest(serializeTest(test)) +} + module.exports = { createTest, enhanceMochaTest, - repackTestForWorkersTransport, + serializeTest, + deserializeTest, + cloneTest, } diff --git a/lib/mocha/types.d.ts b/lib/mocha/types.d.ts index 2bdb55a56..7372ba553 100644 --- a/lib/mocha/types.d.ts +++ b/lib/mocha/types.d.ts @@ -20,6 +20,8 @@ declare global { totalTimeout?: number addToSuite(suite: Mocha.Suite): void applyOptions(opts: Record): void + simplify(): Record + toFileName(): string addNote(type: string, note: string): void codeceptjs: boolean } diff --git a/lib/plugin/analyze.js b/lib/plugin/analyze.js new file mode 100644 index 000000000..710a70e1f --- /dev/null +++ b/lib/plugin/analyze.js @@ -0,0 +1,290 @@ +const debug = require('debug')('codeceptjs:analyze') +const { isMainThread } = require('node:worker_threads') +const container = require('../container') +const ai = require('../ai') +const colors = require('chalk') +const ora = require('ora-classic') +const event = require('../event') +const output = require('../output') +const { ansiRegExp } = require('../utils') + +const MAX_DATA_LENGTH = 5000 + +const defaultConfig = { + clusterize: 5, + analyze: 3, + prompts: { + clusterize: testsAndErrors => { + const serializedFailedTests = testsAndErrors + .map(({ test, error }, index) => { + return ` + TEST #${index + 1}: ${serializeTest(test)} + ERROR: ${serializeError(error).slice(0, MAX_DATA_LENGTH / testsAndErrors.length)}` + }) + .join('\n\n---\n\n') + + return [ + { + role: 'user', + content: ` + I am test analyst analyzing failed tests in CodeceptJS testing framework. + + Please analyze the following failed tests and classify them into groups by their cause. + If there is no common cause, say: "No common cause found". + + Provide a short description of the group and a list of failed tests that belong to this group. + Use percent sign to indicate the percentage of failed tests in the group if this percentage is greater than 30%. + + Here are failed tests: + + ${serializedFailedTests} + + Common categories of failures by order of priority: + + * Browser connection error / browser crash + * Network errors (server error, timeout, etc) + * HTML / page elements (not found, not visible, etc) + * Navigation errors (404, etc) + * Code errors (syntax error, JS errors, etc) + * Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc) + * Data errors (password incorrect, no options in select, invalid format, etc) + * Assertion failures + * Other errors + + + If there is no groups of tests, say: "No patterns found" + Preserve error messages but cut them if they are too long. + Respond clearly and directly, without introductory words or phrases like ‘Of course,’ ‘Here is the answer,’ etc. + Do not list more than 3 tests in the group. + Do not list more than 3 errors in the group. + If you identify that all tests in the group have the same tag, add this tag to the group report, otherwise ignore TAG section. + If you identify that all tests in the group have the same suite, add this suite to the group report, otherwise ignore SUITE section. + Pick different emojis for each group. + + Provide list of groups in following format: + + _______________________________ + + --- GROUP # <(percentage of failed tests)'> + CATEGORY + ERRORS , , ... + SUMMARY + STEPS (in format I.click(), I.see(), etc) + AFFECTED TESTS () , , ... + SUITE , (if all tests in the group have the same suite or suites) + TAG (if all tags in group have the same tag) + `, + }, + { + role: 'assistant', + content: `--- GROUP' + `, + }, + ] + }, + analyze: (test, error) => { + const testMessage = serializeTest(test) + const errorMessage = serializeError(error) + // TODO: if vision is on, attach screenshot to the prompt + + return [ + { + role: 'user', + content: ` + I am qa engineer analyzing failed tests in CodeceptJS testing framework. + Please analyze the following failed test and error its error and explain it. + + Pick one of the categories of failures and explain it. + + Common causes of failures: + + * Browser connection error / browser crash + * Network errors (server error, timeout, etc) + * HTML / page elements (not found, not visible, etc) + * Navigation errors (404, etc) + * Code errors (syntax error, JS errors, etc) + * Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc) + * Data errors (password incorrect, no options in select, invalid format, etc) + * Assertion failures + * Other errors + + + Here is the test and error: + ${testMessage} + ${errorMessage} + + Do not get to details, be concise. + If there is failed step, just write it in STEPS section. + If you have suggestions for the test, write them in SUMMARY section. + Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest. + Be concise, each section should not take more than one sentence. + + Response format: + + CATEGORY + STEPS + SUMMARY + + Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS. + `, + }, + ] + }, + }, +} + +/** + * + * @param {*} config + * @returns + */ +module.exports = function (config = {}) { + config = Object.assign(defaultConfig, config) + + let failedTestsAndErrors = [] + + event.dispatcher.on(event.test.failed, (test, error) => { + if (!ai.isEnabled) return + failedTestsAndErrors.push({ test, error }) + }) + + event.dispatcher.on(event.all.result, async () => { + if (!ai.isEnabled) { + console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.') + return + } + + if (!failedTestsAndErrors.length) return + if (!isMainThread) return // run only on main thread + + debug(failedTestsAndErrors.map(e => serializeTest(e.test) + '\n' + serializeError(e.error))) + + console.log() + console.log(colors.bold.white('🪄 AI REPORT:')) + + try { + if (failedTestsAndErrors.length >= config.clusterize) { + const response = await clusterize() + console.log(response) + return + } + + output.plugin('analyze', `Analyzing first ${config.analyze} failed tests...`) + + const uniqueErrors = failedTestsAndErrors.filter((item, index, array) => { + return array.findIndex(t => serializeError(t.error) === serializeError(item.error)) === index + }) + + for (let i = 0; i < config.analyze; i++) { + if (!uniqueErrors[i]) break + + const response = await analyze(uniqueErrors[i]) + if (!response) { + break + } + + console.log() + console.log('--------------------------------') + console.log(colors.bold.white(uniqueErrors[i].test.fullTitle())) + console.log(response) + } + } catch (err) { + console.error('Error analyzing failed tests', err) + } + + if (!container.plugins('pageInfo')) { + console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.') + } + }) + + async function clusterize() { + const spinner = ora('Clusterizing failures...').start() + const prompt = config.prompts.clusterize(failedTestsAndErrors) + try { + const response = await ai.createCompletion(prompt) + spinner.stop() + return formatResponse(response) + } catch (err) { + spinner.stop() + console.error('Error clusterizing failures', err.message) + } + } + + async function analyze(failedTestAndError) { + const spinner = ora('Analyzing failure...').start() + const prompt = config.prompts.analyze(failedTestAndError.test, failedTestAndError.error) + try { + const response = await ai.createCompletion(prompt) + spinner.stop() + return formatResponse(response) + } catch (err) { + spinner.stop() + console.error('Error analyzing failure:', err.message) + } + } +} + +function serializeError(error) { + let errorMessage = 'ERROR: ' + error.message + + if (error.inspect) { + errorMessage = 'ERROR: ' + error.inspect() + } + + if (error.stack) { + errorMessage += + '\n' + + error.stack + .replace(global.codecept_dir || '', '.') + .split('\n') + .map(line => line.replace(ansiRegExp(), '')) + .slice(0, 5) + .join('\n') + } + if (error.steps) { + errorMessage += '\n STEPS: ' + error.steps.map(s => s.toCode()).join('\n') + } + return errorMessage +} + +function serializeTest(test) { + if (!test.uid) return + + let testMessage = 'TEST TITLE: ' + test.title + + if (test.suite) { + testMessage += '\n SUITE: ' + test.suite.title + } + + if (test.steps?.length) { + const failedSteps = test.steps.filter(s => s.status === 'failed') + if (failedSteps.length) testMessage += '\n STEP: ' + failedSteps.map(s => s.toCode()).join('; ') + } + + const pageInfo = test.notes.find(n => n.type === 'pageInfo') + if (pageInfo) { + testMessage += '\n PAGE INFO: ' + pageInfo.text + } + + return testMessage +} + +function formatResponse(response) { + return response + .split('\n') + .map(line => line.trim()) + .filter(line => !/^[A-Z\s]+$/.test(line)) + .map(line => { + if (line.startsWith('ANALYSIS REPORT')) return line.replace('ANALYSIS REPORT', colors.bold.white('ANALYSIS REPORT ')) + if (line.startsWith('GROUP')) return line.replace('GROUP', colors.bold.bgWhite('GROUP ')) + if (line.startsWith('STEPS')) return line.replace('STEPS', colors.bold.bgBlue('STEP ')) + if (line.startsWith('AFFECTED TESTS')) return line.replace('AFFECTED TESTS', colors.bold.bgWhite('AFFECTED TESTS ')) + if (line.startsWith('ERRORS')) return line.replace('ERRORS', colors.bold.bgRed('ERRORS ')) + if (line.startsWith('TAG')) return line.replace('TAG', colors.bold.bgGray('TAG ')) + if (line.startsWith('SUITE')) return line.replace('SUITE', colors.bold.bgGray('SUITE ')) + if (line.startsWith('SUMMARY')) return line.replace('SUMMARY', colors.bold.bgYellow('SUMMARY ')) + if (line.startsWith('CATEGORY')) return line.replace('CATEGORY', colors.bold.bgGreen('CATEGORY ')) + return line + }) + .join('\n') +} diff --git a/lib/plugin/customReporter.js b/lib/plugin/customReporter.js new file mode 100644 index 000000000..9780636b8 --- /dev/null +++ b/lib/plugin/customReporter.js @@ -0,0 +1,34 @@ +const event = require('../event') + +/** + * Sample custom reporter for CodeceptJS. + */ +module.exports = function (config) { + event.dispatcher.on(event.hook.before, hook => { + if (config.onHookBefore) { + config.onHookBefore(hook) + } + }) + + event.dispatcher.on(event.test.before, test => { + if (config.onTestBefore) { + config.onTestBefore(test) + } + }) + + event.dispatcher.on(event.test.failed, test => { + if (config.onTestFailed) { + config.onTestFailed(test) + } + }) + + event.dispatcher.on(event.all.result, result => { + if (config.onAllResult) { + config.onResult(result) + } + + if (config.save) { + result.save() + } + }) +} diff --git a/lib/plugin/debugErrors.js b/lib/plugin/debugErrors.js deleted file mode 100644 index c24255a62..000000000 --- a/lib/plugin/debugErrors.js +++ /dev/null @@ -1,67 +0,0 @@ -const Container = require('../container') -const recorder = require('../recorder') -const event = require('../event') -const supportedHelpers = require('./standardActingHelpers') -const { scanForErrorMessages } = require('../html') -const { output } = require('..') - -const defaultConfig = { - errorClasses: ['error', 'warning', 'alert', 'danger'], -} - -/** - * Prints errors found in HTML code after each failed test. - * - * It scans HTML and searches for elements with error classes. - * If an element found prints a text from it to console and adds as artifact to the test. - * - * Enable this plugin in config: - * - * ```js - * plugins: { - * debugErrors: { - * enabled: true, - * } - * ``` - * - * Additional config options: - * - * * `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`) - * - */ -module.exports = function (config = {}) { - const helpers = Container.helpers() - let helper - - config = Object.assign(defaultConfig, config) - - for (const helperName of supportedHelpers) { - if (Object.keys(helpers).indexOf(helperName) > -1) { - helper = helpers[helperName] - } - } - - if (!helper) return // no helpers for screenshot - - event.dispatcher.on(event.test.failed, test => { - recorder.add('HTML snapshot failed test', async () => { - try { - const currentOutputLevel = output.level() - output.level(0) - const html = await helper.grabHTMLFrom('body') - output.level(currentOutputLevel) - - if (!html) return - - const errors = scanForErrorMessages(html, config.errorClasses) - if (errors.length) { - output.debug('Detected errors in HTML code') - errors.forEach(error => output.debug(error)) - test.artifacts.errors = errors - } - } catch (err) { - // not really needed - } - }) - }) -} diff --git a/lib/plugin/pageInfo.js b/lib/plugin/pageInfo.js new file mode 100644 index 000000000..836360fcb --- /dev/null +++ b/lib/plugin/pageInfo.js @@ -0,0 +1,145 @@ +const path = require('path') +const fs = require('fs') +const Container = require('../container') +const recorder = require('../recorder') +const event = require('../event') +const supportedHelpers = require('./standardActingHelpers') +const { scanForErrorMessages } = require('../html') +const { output } = require('..') +const { humanizeString, ucfirst } = require('../utils') + +const defaultConfig = { + errorClasses: ['error', 'warning', 'alert', 'danger'], + browserLogs: ['error'], +} + +/** + * Collects information from web page after each failed test and adds it to the test as an artifact. + * It is suggested to enable this plugin if you run tests on CI and you need to debug failed tests. + * This plugin can be paired with `analyze` plugin to provide more context. + * + * It collects URL, HTML errors (by classes), and browser logs. + * + * Enable this plugin in config: + * + * ```js + * plugins: { + * pageInfo: { + * enabled: true, + * } + * ``` + * + * Additional config options: + * + * * `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`) + * * `browserLogs` - list of types of errors to search for in browser logs (default: `['error']`) + * + */ +module.exports = function (config = {}) { + const helpers = Container.helpers() + let helper + + config = Object.assign(defaultConfig, config) + + for (const helperName of supportedHelpers) { + if (Object.keys(helpers).indexOf(helperName) > -1) { + helper = helpers[helperName] + } + } + + if (!helper) return // no helpers for screenshot + + event.dispatcher.on(event.test.failed, test => { + const pageState = {} + + recorder.add('URL of failed test', async () => { + try { + const url = await helper.grabCurrentUrl() + pageState.url = url + } catch (err) { + // not really needed + } + }) + recorder.add('HTML snapshot failed test', async () => { + try { + const currentOutputLevel = output.level() + output.level(0) + const html = await helper.grabHTMLFrom('body') + output.level(currentOutputLevel) + + if (!html) return + + const errors = scanForErrorMessages(html, config.errorClasses) + if (errors.length) { + output.debug('Detected errors in HTML code') + errors.forEach(error => output.debug(error)) + pageState.htmlErrors = errors + } + } catch (err) { + // not really needed + } + }) + + recorder.add('Browser logs for failed test', async () => { + try { + const logs = await helper.grabBrowserLogs() + + if (!logs) return + + pageState.browserErrors = getBrowserErrors(logs, config.browserLogs) + } catch (err) { + // not really needed + } + }) + + recorder.add('Save page info', () => { + test.addNote('pageInfo', pageStateToMarkdown(pageState)) + + const pageStateFileName = path.join(global.output_dir, `${test.toFileName()}.pageInfo.md`) + fs.writeFileSync(pageStateFileName, pageStateToMarkdown(pageState)) + test.artifacts.pageInfo = pageStateFileName + return pageState + }) + }) +} + +function pageStateToMarkdown(pageState) { + let markdown = '' + + for (const [key, value] of Object.entries(pageState)) { + if (!value) continue + let result = '' + + if (Array.isArray(value)) { + result = value.map(v => `- ${JSON.stringify(v, null, 2)}`).join('\n') + } else if (typeof value === 'string') { + result = `${value}` + } else { + result = JSON.stringify(value, null, 2) + } + + if (!result.trim()) continue + + markdown += `### ${ucfirst(humanizeString(key))}\n\n` + markdown += result + markdown += '\n\n' + } + + markdown += `### ${ucfirst(humanizeString(key))}\n\n` + + return markdown +} + +function getBrowserErrors(logs, type = ['error']) { + // Playwright console messages + let errors = logs + .filter(log => log.type) + .map(l => ({ type: l.type(), text: l.text() })) + .filter(l => type.includes(l.type)) + .map(l => l.text) + + // If console log coming from other helpers they may need other analysis + // TODO: Add other helpers analysis + + return errors +} diff --git a/lib/plugin/screenshotOnFail.js b/lib/plugin/screenshotOnFail.js index bf7a461dc..b8cc23907 100644 --- a/lib/plugin/screenshotOnFail.js +++ b/lib/plugin/screenshotOnFail.js @@ -79,12 +79,10 @@ module.exports = function (config) { recorder.add( 'screenshot of failed test', async () => { - let fileName = clearString(test.title) const dataType = 'image/png' // This prevents data driven to be included in the failed screenshot file name - if (fileName.indexOf('{') !== -1) { - fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim() - } + let fileName = test.toFileName() + if (test.ctx && test.ctx.test && test.ctx.test.type === 'hook') fileName = clearString(`${test.title}_${test.ctx.test.title}`) if (options.uniqueScreenshotNames && test) { const uuid = _getUUID(test) diff --git a/lib/result.js b/lib/result.js new file mode 100644 index 000000000..75130d6d5 --- /dev/null +++ b/lib/result.js @@ -0,0 +1,94 @@ +const fs = require('fs') +const path = require('path') + +class Result { + constructor() { + this._stats = { + passes: 0, + failures: 0, + tests: 0, + pending: 0, + failedHooks: 0, + } + + this._startTime = new Date() + this._endTime = null + + /** @type {CodeceptJS.Test[]} */ + this._tests = [] + + /** @type {String[]} */ + this._failures = [] + + this.start() + } + + start() { + this._startTime = new Date() + } + + finish() { + this._endTime = new Date() + } + + get tests() { + return this._tests + } + + get failures() { + return this._failures + } + + get stats() { + return this._stats + } + + get startTime() { + return this._startTime + } + + addTest(test) { + this._tests.push(test) + } + + /** + * Add failures to result + * + * @param {String[]} newFailures + */ + addFailures(newFailures) { + this._failures.push(...newFailures) + } + + get hasFailures() { + return this._failures.length > 0 + } + + get duration() { + return this._endTime ? +this._endTime - +this._startTime : 0 + } + + simplify() { + return { + stats: this.stats, + duration: this.duration, + tests: this._tests.map(test => test.simplify()), + failures: this._failures, + } + } + + save(fileName) { + if (!fileName) fileName = 'result.json' + fs.writeFileSync(path.join(codeceptjs.outputDir, fileName), JSON.stringify(this.simplify(), null, 2)) + } + + addStats(newStats = {}) { + this._stats.passes += newStats.passes || 0 + this._stats.failures += newStats.failures || 0 + this._stats.tests += newStats.tests || 0 + this._stats.pending += newStats.pending || 0 + this._stats.failedHooks += newStats.failedHooks || 0 + } +} + +module.exports = Result diff --git a/lib/step/base.js b/lib/step/base.js index 47bbe4f5c..85b66626e 100644 --- a/lib/step/base.js +++ b/lib/step/base.js @@ -1,9 +1,9 @@ const color = require('chalk') const Secret = require('../secret') const { getCurrentTimeout } = require('./timeout') -const { ucfirst, humanizeString } = require('../utils') +const { ucfirst, humanizeString, serializeError } = require('../utils') -const STACK_LINE = 4 +const STACK_LINE = 5 /** * Each command in test executed through `I.` object is wrapped in Step. @@ -37,6 +37,9 @@ class Step { /** @member {string} */ this.stack = '' + this.startTime = 0 + this.endTime = 0 + this.setTrace() } @@ -159,6 +162,49 @@ class Step { return this.constructor.name === 'MetaStep' } + get duration() { + return this.endTime - this.startTime + } + + simplify() { + const step = this + + const parent = {} + if (step.metaStep) { + parent.title = step.metaStep.actor + } + + if (step.opts) { + Object.keys(step.opts).forEach(k => { + if (typeof step.opts[k] === 'object') delete step.opts[k] + if (typeof step.opts[k] === 'function') delete step.opts[k] + }) + } + + const args = [] + if (step.args) { + for (const arg of step.args) { + // check if arg is a JOI object + if (arg && arg.$_root) { + args.push(JSON.stringify(arg).slice(0, 300)) + } else if (arg && typeof arg === 'function') { + args.push(arg.name) + } else { + args.push(arg) + } + } + } + + return { + opts: step.opts || {}, + title: step.name, + args: JSON.stringify(args), + status: step.status, + duration: step.duration || 0, + parent, + } + } + /** @return {boolean} */ hasBDDAncestor() { let hasBDD = false diff --git a/lib/utils.js b/lib/utils.js index 2aac45685..9310b3350 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -542,3 +542,11 @@ module.exports.humanizeString = function (string) { _result[0] = _result[0] === 'i' ? this.ucfirst(_result[0]) : _result[0] return _result.join(' ').trim() } + +module.exports.serializeError = function (error) { + if (error) { + const { stack, uncaught, message, actual, expected } = error + return { stack, uncaught, message, actual, expected } + } + return null +} diff --git a/lib/workers.js b/lib/workers.js index c66aa2c40..52607eb35 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -11,8 +11,10 @@ const { isFunction, fileExists } = require('./utils') const { replaceValueDeep, deepClone } = require('./utils') const mainConfig = require('./config') const output = require('./output') +const store = require('./store') const event = require('./event') -const { repackTestForWorkersTransport: repackTest } = require('./mocha/test') +const { deserializeTest } = require('./mocha/test') +const { deserializeSuite } = require('./mocha/suite') const recorder = require('./recorder') const runHook = require('./hooks') const WorkerStorage = require('./workerStorage') @@ -230,17 +232,10 @@ class Workers extends EventEmitter { super() this.setMaxListeners(50) this.codecept = initializeCodecept(config.testConfig, config.options) - this.failuresLog = [] this.errors = [] this.numberOfWorkers = 0 this.closedWorkers = 0 this.workers = [] - this.stats = { - passes: 0, - failures: 0, - tests: 0, - pending: 0, - } this.testGroups = [] createOutputDir(config.testConfig) @@ -353,8 +348,6 @@ class Workers extends EventEmitter { } run() { - this.stats.start = new Date() - this.stats.failedHooks = 0 recorder.startUnlessRunning() event.dispatcher.emit(event.workers.before) process.env.RUNS_WITH_WORKERS = 'true' @@ -380,7 +373,7 @@ class Workers extends EventEmitter { * @returns {Boolean} */ isFailed() { - return (this.stats.failures || this.errors.length) > 0 + return (Container.result().failures.length || this.errors.length) > 0 } _listenWorkerEvents(worker) { @@ -393,33 +386,33 @@ class Workers extends EventEmitter { } switch (message.event) { - case event.all.failures: - this.failuresLog = this.failuresLog.concat(message.data.failuresLog) - this._appendStats(message.data.stats) + case event.all.result: + Container.result().addFailures(message.data.failures) + Container.result().addStats(message.data.stats) break case event.suite.before: - this.emit(event.suite.before, repackTest(message.data)) + this.emit(event.suite.before, deserializeSuite(message.data)) break case event.test.before: - this.emit(event.test.before, repackTest(message.data)) + this.emit(event.test.before, deserializeTest(message.data)) break case event.test.started: - this.emit(event.test.started, repackTest(message.data)) + this.emit(event.test.started, deserializeTest(message.data)) break case event.test.failed: - this.emit(event.test.failed, repackTest(message.data)) + this.emit(event.test.failed, deserializeTest(message.data)) break case event.test.passed: - this.emit(event.test.passed, repackTest(message.data)) + this.emit(event.test.passed, deserializeTest(message.data)) break case event.test.skipped: - this.emit(event.test.skipped, repackTest(message.data)) + this.emit(event.test.skipped, deserializeTest(message.data)) break case event.test.finished: - this.emit(event.test.finished, repackTest(message.data)) + this.emit(event.test.finished, deserializeTest(message.data)) break case event.test.after: - this.emit(event.test.after, repackTest(message.data)) + this.emit(event.test.after, deserializeTest(message.data)) break case event.step.finished: this.emit(event.step.finished, message.data) @@ -455,28 +448,20 @@ class Workers extends EventEmitter { } else { process.exitCode = 0 } - // removed this.finishedTests because in all /lib only first argument (!this.isFailed()) is used) - this.emit(event.all.result, !this.isFailed()) - this.emit('end') // internal event - } - _appendStats(newStats) { - this.stats.passes += newStats.passes - this.stats.failures += newStats.failures - this.stats.tests += newStats.tests - this.stats.pending += newStats.pending - this.stats.failedHooks += newStats.failedHooks + this.emit(event.all.result, Container.result()) + this.emit('end') // internal event } printResults() { - this.stats.end = new Date() - this.stats.duration = this.stats.end - this.stats.start + const result = Container.result() + result.finish() // Reset process for logs in main thread output.process(null) output.print() - this.failuresLog = this.failuresLog + this.failuresLog = result.failures .filter(log => log.length && typeof log[1] === 'number') // mocha/lib/reporters/base.js .map(([format, num, title, message, stack], i) => [format, i + 1, title, message, stack]) @@ -487,7 +472,7 @@ class Workers extends EventEmitter { this.failuresLog.forEach(log => output.print(...log)) } - output.result(this.stats.passes, this.stats.failures, this.stats.pending, ms(this.stats.duration), this.stats.failedHooks) + output.result(result.stats.passes, result.stats.failures, result.stats.pending, ms(result.duration), result.stats.failedHooks) process.env.RUNS_WITH_WORKERS = 'false' } } diff --git a/test/runner/interface_test.js b/test/runner/interface_test.js index daa40b342..72d8023bc 100644 --- a/test/runner/interface_test.js +++ b/test/runner/interface_test.js @@ -156,7 +156,6 @@ describe('CodeceptJS Interface', () => { expect(output).toContain('OK') expect(output).toContain('0 passed') expect(output).toContain('2 skipped') - console.log(err) if (process.env.CI) { // we notify that no tests were executed, which is not expected on CI expect(err).toBeTruthy() From 87608a4de073d54e69f36ae69d821c57be6843f3 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jan 2025 04:44:04 +0200 Subject: [PATCH 02/18] fixed setting name for test --- lib/mocha/test.js | 26 ++++++++++++++------------ lib/plugin/pageInfo.js | 4 ++-- lib/plugin/screenshotOnFail.js | 6 +++--- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/mocha/test.js b/lib/mocha/test.js index 6c2769da1..bc648c281 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -51,18 +51,6 @@ function enhanceMochaTest(test) { test.uid = genTestId(test) } - test.toFileName = function () { - let fileName = clearString(test.title) - if (fileName.indexOf('{') !== -1) { - fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim() - } - // TODO: add suite title to file name - // if (test.parent && test.parent.title) { - // fileName = `${clearString(test.parent.title)}_${fileName}` - // } - return fileName - } - test.applyOptions = function (opts) { if (!opts) opts = {} test.opts = opts @@ -134,8 +122,22 @@ function cloneTest(test) { return deserializeTest(serializeTest(test)) } +function testToFileName(test) { + let fileName = clearString(test.title) + if (fileName.indexOf('{') !== -1) { + fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim() + } + if (test.ctx && test.ctx.test && test.ctx.test.type === 'hook') fileName = clearString(`${test.title}_${test.ctx.test.title}`) + // TODO: add suite title to file name + // if (test.parent && test.parent.title) { + // fileName = `${clearString(test.parent.title)}_${fileName}` + // } + return fileName +} + module.exports = { createTest, + testToFileName, enhanceMochaTest, serializeTest, deserializeTest, diff --git a/lib/plugin/pageInfo.js b/lib/plugin/pageInfo.js index 836360fcb..d350f8c68 100644 --- a/lib/plugin/pageInfo.js +++ b/lib/plugin/pageInfo.js @@ -7,7 +7,7 @@ const supportedHelpers = require('./standardActingHelpers') const { scanForErrorMessages } = require('../html') const { output } = require('..') const { humanizeString, ucfirst } = require('../utils') - +const { testToFileName } = require('../mocha/test') const defaultConfig = { errorClasses: ['error', 'warning', 'alert', 'danger'], browserLogs: ['error'], @@ -95,7 +95,7 @@ module.exports = function (config = {}) { recorder.add('Save page info', () => { test.addNote('pageInfo', pageStateToMarkdown(pageState)) - const pageStateFileName = path.join(global.output_dir, `${test.toFileName()}.pageInfo.md`) + const pageStateFileName = path.join(global.output_dir, `${testToFileName(test)}.pageInfo.md`) fs.writeFileSync(pageStateFileName, pageStateToMarkdown(pageState)) test.artifacts.pageInfo = pageStateFileName return pageState diff --git a/lib/plugin/screenshotOnFail.js b/lib/plugin/screenshotOnFail.js index b8cc23907..f35b9d052 100644 --- a/lib/plugin/screenshotOnFail.js +++ b/lib/plugin/screenshotOnFail.js @@ -5,8 +5,9 @@ const Container = require('../container') const recorder = require('../recorder') const event = require('../event') const output = require('../output') -const { fileExists, clearString } = require('../utils') +const { fileExists } = require('../utils') const Codeceptjs = require('../index') +const { testToFileName } = require('../mocha/test') const defaultConfig = { uniqueScreenshotNames: false, @@ -81,9 +82,8 @@ module.exports = function (config) { async () => { const dataType = 'image/png' // This prevents data driven to be included in the failed screenshot file name - let fileName = test.toFileName() + let fileName = testToFileName(test) - if (test.ctx && test.ctx.test && test.ctx.test.type === 'hook') fileName = clearString(`${test.title}_${test.ctx.test.title}`) if (options.uniqueScreenshotNames && test) { const uuid = _getUUID(test) fileName = `${fileName.substring(0, 10)}_${uuid}.failed.png` From 5b4b9a6f51df0438e99b183f8c58a37beaac6d04 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jan 2025 04:51:59 +0200 Subject: [PATCH 03/18] fixed workers tests --- lib/result.js | 5 +++++ lib/workers.js | 2 +- test/unit/worker_test.js | 28 ++++++++++++++-------------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/lib/result.js b/lib/result.js index 75130d6d5..a9b6ff447 100644 --- a/lib/result.js +++ b/lib/result.js @@ -31,6 +31,10 @@ class Result { this._endTime = new Date() } + get hasFailed() { + return this.tests.some(test => test.state === 'failed') + } + get tests() { return this._tests } @@ -70,6 +74,7 @@ class Result { simplify() { return { + hasFailed: this.hasFailed, stats: this.stats, duration: this.duration, tests: this._tests.map(test => test.simplify()), diff --git a/lib/workers.js b/lib/workers.js index 52607eb35..74ab182b6 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -443,7 +443,7 @@ class Workers extends EventEmitter { _finishRun() { event.dispatcher.emit(event.workers.after, { tests: this.workers.map(worker => worker.tests) }) - if (this.isFailed()) { + if (Container.result().hasFailed) { process.exitCode = 1 } else { process.exitCode = 0 diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index f1cde53a6..fe051ca45 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -28,8 +28,8 @@ describe('Workers', function () { workers.run() - workers.on(event.all.result, status => { - expect(status).equal(false) + workers.on(event.all.result, result => { + expect(result.hasFailed).equal(false) expect(passedCount).equal(5) expect(failedCount).equal(3) done() @@ -63,9 +63,9 @@ describe('Workers', function () { workers.run() - workers.on(event.all.result, status => { + workers.on(event.all.result, result => { expect(workers.getWorkers().length).equal(2) - expect(status).equal(true) + expect(result.hasFailed).equal(false) done() }) }) @@ -100,8 +100,8 @@ describe('Workers', function () { passedCount += 1 }) - workers.on(event.all.result, status => { - expect(status).equal(false) + workers.on(event.all.result, result => { + expect(result.hasFailed).equal(false) expect(passedCount).equal(3) expect(failedCount).equal(2) done() @@ -135,9 +135,9 @@ describe('Workers', function () { workers.run() - workers.on(event.all.result, status => { + workers.on(event.all.result, result => { expect(workers.getWorkers().length).equal(2) - expect(status).equal(true) + expect(result.hasFailed).equal(false) done() }) }) @@ -170,9 +170,9 @@ describe('Workers', function () { workers.run() - workers.on(event.all.result, status => { + workers.on(event.all.result, result => { expect(workers.getWorkers().length).equal(2) - expect(status).equal(true) + expect(result.hasFailed).equal(false) done() }) }) @@ -199,8 +199,8 @@ describe('Workers', function () { workers.run() recorder.add(() => share({ fromMain: true })) - workers.on(event.all.result, status => { - expect(status).equal(true) + workers.on(event.all.result, result => { + expect(result.hasFailed).equal(false) done() }) }) @@ -258,9 +258,9 @@ describe('Workers', function () { workers.run() - workers.on(event.all.result, status => { + workers.on(event.all.result, result => { expect(workers.getWorkers().length).equal(8) - expect(status).equal(true) + expect(result.hasFailed).equal(false) done() }) }) From 851e6902b12d12335cdc0017caf408476fbfe89d Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jan 2025 06:44:46 +0200 Subject: [PATCH 04/18] improved handling information between workers and main process --- lib/command/run-workers.js | 9 ------ lib/command/workers/runTests.js | 4 +-- lib/listener/result.js | 1 - lib/mocha/hooks.js | 12 +++++++ lib/mocha/test.js | 19 +++++++----- lib/plugin/analyze.js | 55 +++++++++++++++++++++------------ lib/plugin/heal.js | 4 +-- lib/plugin/pageInfo.js | 2 -- lib/result.js | 21 ++++++++++++- lib/step/base.js | 13 ++++---- lib/workers.js | 8 ++++- 11 files changed, 97 insertions(+), 51 deletions(-) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index 4d597f7b0..ac956d291 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -51,15 +51,6 @@ module.exports = async function (workerCount, selectedRuns, options) { }) workers.on(event.all.result, result => { - event.dispatcher.emit(event.workers.result, { - ...result?.simplify(), - suites: suiteArr, - tests: { - passed: passedTestArr, - failed: failedTestArr, - skipped: skippedTestArr, - }, - }) workers.printResults() }) diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 09900f075..3df4b1d9a 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -98,8 +98,8 @@ function initializeListeners() { event.dispatcher.on(event.step.passed, step => sendToParentThread({ event: event.step.passed, workerIndex, data: step.simplify() })) event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: step.simplify() })) - event.dispatcher.on(event.hook.failed, (test, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: { ...test.simplify(), err } })) - event.dispatcher.on(event.hook.passed, (test, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: { ...test.simplify(), err } })) + event.dispatcher.on(event.hook.failed, (hook, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: { ...hook.simplify(), err } })) + event.dispatcher.on(event.hook.passed, (hook, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: { ...hook.simplify(), err } })) event.dispatcher.once(event.all.after, () => { sendToParentThread({ event: event.all.after, workerIndex, data: container.result().simplify() }) diff --git a/lib/listener/result.js b/lib/listener/result.js index fe9762017..7df7d771a 100644 --- a/lib/listener/result.js +++ b/lib/listener/result.js @@ -7,7 +7,6 @@ module.exports = function () { }) event.dispatcher.on(event.test.started, test => { - container.result().addStats({ tests: 1 }) container.result().addTest(test) }) } diff --git a/lib/mocha/hooks.js b/lib/mocha/hooks.js index 3c9aa7d4d..8ac00a7fb 100644 --- a/lib/mocha/hooks.js +++ b/lib/mocha/hooks.js @@ -1,4 +1,6 @@ const event = require('../event') +const { serializeError } = require('../utils') +// const { serializeTest } = require('./test') class Hook { constructor(context, error) { @@ -13,6 +15,16 @@ class Hook { return this.constructor.name.replace('Hook', '') } + simplify() { + return { + hookName: this.hookName, + title: this.title, + // test: this.test ? serializeTest(this.test) : null, + // suite: this.suite ? serializeSuite(this.suite) : null, + error: this.error ? serializeError(this.error) : null, + } + } + toString() { return this.hookName } diff --git a/lib/mocha/test.js b/lib/mocha/test.js index bc648c281..c0c3d6f48 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -1,9 +1,9 @@ const Test = require('mocha/lib/test') const Suite = require('mocha/lib/suite') const { test: testWrapper } = require('./asyncWrapper') -const { enhanceMochaSuite } = require('./suite') +const { enhanceMochaSuite, createSuite } = require('./suite') const { genTestId, serializeError, clearString } = require('../utils') - +const Step = require('../step/base') /** * Factory function to create enhanced tests * @param {string} title - Test title @@ -67,15 +67,18 @@ function enhanceMochaTest(test) { } function deserializeTest(test) { - test = Object.assign(new Test(test.title || '', () => {}), test) + test = Object.assign( + createTest(test.title || '', () => {}), + test, + ) test.parent = Object.assign(new Suite(test.parent.title), test.parent) - enhanceMochaTest(test) enhanceMochaSuite(test.parent) + test.steps = test.steps.map(step => Object.assign(new Step(step.title), step)) return test } function serializeTest(test, err = null) { - test = { ...test } + // test = { ...test } if (test.start && !test.duration) { const end = +new Date() @@ -107,14 +110,14 @@ function serializeTest(test, err = null) { uid: test.uid, retries: test._retries, title: test.title, - status: test.status, + state: test.state, notes: test.notes || [], meta: test.meta || {}, - artifacts: test.artifacts || [], + artifacts: test.artifacts || {}, duration: test.duration || 0, err, parent, - steps: [...test.steps].map(step => step.simplify()), + steps: [...test.steps].map(step => (step.simplify ? step.simplify() : step)), } } diff --git a/lib/plugin/analyze.js b/lib/plugin/analyze.js index 710a70e1f..3c3bc6748 100644 --- a/lib/plugin/analyze.js +++ b/lib/plugin/analyze.js @@ -1,5 +1,6 @@ const debug = require('debug')('codeceptjs:analyze') const { isMainThread } = require('node:worker_threads') +const { arrowRight } = require('figures') const container = require('../container') const ai = require('../ai') const colors = require('chalk') @@ -16,10 +17,11 @@ const defaultConfig = { prompts: { clusterize: testsAndErrors => { const serializedFailedTests = testsAndErrors - .map(({ test, error }, index) => { + .map((test, index) => { + if (!test || !test.err) return return ` TEST #${index + 1}: ${serializeTest(test)} - ERROR: ${serializeError(error).slice(0, MAX_DATA_LENGTH / testsAndErrors.length)}` + ERROR: ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / testsAndErrors.length)}` }) .join('\n\n---\n\n') @@ -96,7 +98,7 @@ const defaultConfig = { Pick one of the categories of failures and explain it. - Common causes of failures: + Common causes of failures in order of priority: * Browser connection error / browser crash * Network errors (server error, timeout, etc) @@ -118,9 +120,9 @@ const defaultConfig = { If you have suggestions for the test, write them in SUMMARY section. Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest. Be concise, each section should not take more than one sentence. - + Response format: - + CATEGORY STEPS SUMMARY @@ -141,30 +143,38 @@ const defaultConfig = { module.exports = function (config = {}) { config = Object.assign(defaultConfig, config) - let failedTestsAndErrors = [] + event.dispatcher.on(event.all.result, async result => { + if (!isMainThread) return // run only on main thread + if (!ai.isEnabled) { + console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.') + return + } - event.dispatcher.on(event.test.failed, (test, error) => { - if (!ai.isEnabled) return - failedTestsAndErrors.push({ test, error }) + printReport(result) }) - event.dispatcher.on(event.all.result, async () => { + event.dispatcher.on(event.workers.result, async result => { if (!ai.isEnabled) { console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.') return } + printReport(result) + }) + + async function printReport(result) { + const failedTestsAndErrors = result.tests.filter(t => t.state === 'failed' && t.err) + if (!failedTestsAndErrors.length) return - if (!isMainThread) return // run only on main thread - debug(failedTestsAndErrors.map(e => serializeTest(e.test) + '\n' + serializeError(e.error))) + debug(failedTestsAndErrors.map(t => serializeTest(t) + '\n' + serializeError(t.err))) console.log() console.log(colors.bold.white('🪄 AI REPORT:')) try { if (failedTestsAndErrors.length >= config.clusterize) { - const response = await clusterize() + const response = await clusterize(failedTestsAndErrors) console.log(response) return } @@ -172,7 +182,7 @@ module.exports = function (config = {}) { output.plugin('analyze', `Analyzing first ${config.analyze} failed tests...`) const uniqueErrors = failedTestsAndErrors.filter((item, index, array) => { - return array.findIndex(t => serializeError(t.error) === serializeError(item.error)) === index + return array.findIndex(t => serializeError(t.err) === serializeError(item.err)) === index }) for (let i = 0; i < config.analyze; i++) { @@ -185,19 +195,20 @@ module.exports = function (config = {}) { console.log() console.log('--------------------------------') - console.log(colors.bold.white(uniqueErrors[i].test.fullTitle())) + console.log(arrowRight, colors.bold.white(uniqueErrors[i].fullTitle())) + console.log() console.log(response) } } catch (err) { console.error('Error analyzing failed tests', err) } - if (!container.plugins('pageInfo')) { + if (!Object.keys(container.plugins()).includes('pageInfo')) { console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.') } - }) + } - async function clusterize() { + async function clusterize(failedTestsAndErrors) { const spinner = ora('Clusterizing failures...').start() const prompt = config.prompts.clusterize(failedTestsAndErrors) try { @@ -212,7 +223,7 @@ module.exports = function (config = {}) { async function analyze(failedTestAndError) { const spinner = ora('Analyzing failure...').start() - const prompt = config.prompts.analyze(failedTestAndError.test, failedTestAndError.error) + const prompt = config.prompts.analyze(failedTestAndError, failedTestAndError.err) try { const response = await ai.createCompletion(prompt) spinner.stop() @@ -225,6 +236,12 @@ module.exports = function (config = {}) { } function serializeError(error) { + if (typeof error === 'string') { + return error + } + + if (!error) return + let errorMessage = 'ERROR: ' + error.message if (error.inspect) { diff --git a/lib/plugin/heal.js b/lib/plugin/heal.js index 35644f875..abf788122 100644 --- a/lib/plugin/heal.js +++ b/lib/plugin/heal.js @@ -117,10 +117,10 @@ module.exports = function (config = {}) { } }) - event.dispatcher.on(event.workers.result, ({ tests }) => { + event.dispatcher.on(event.workers.result, result => { const { print } = output - const healedTests = Object.values(tests) + const healedTests = Object.values(result.tests) .flat() .filter(test => test.notes.some(note => note.type === 'heal')) if (!healedTests.length) return diff --git a/lib/plugin/pageInfo.js b/lib/plugin/pageInfo.js index d350f8c68..c04c8efcf 100644 --- a/lib/plugin/pageInfo.js +++ b/lib/plugin/pageInfo.js @@ -125,8 +125,6 @@ function pageStateToMarkdown(pageState) { markdown += '\n\n' } - markdown += `### ${ucfirst(humanizeString(key))}\n\n` - return markdown } diff --git a/lib/result.js b/lib/result.js index a9b6ff447..9fcd5d204 100644 --- a/lib/result.js +++ b/lib/result.js @@ -1,5 +1,6 @@ const fs = require('fs') const path = require('path') +const { serializeTest } = require('./mocha/test') class Result { constructor() { @@ -52,6 +53,12 @@ class Result { } addTest(test) { + const existingTestIndex = this._tests.findIndex(t => !!t.uid && t.uid === test.uid) + if (existingTestIndex >= 0) { + this._tests[existingTestIndex] = test + return + } + this._tests.push(test) } @@ -72,12 +79,24 @@ class Result { return this._endTime ? +this._endTime - +this._startTime : 0 } + get failedTests() { + return this._tests.filter(test => test.state === 'failed') + } + + get passedTests() { + return this._tests.filter(test => test.state === 'passed') + } + + get skippedTests() { + return this._tests.filter(test => test.state === 'skipped' || test.state === 'pending') + } + simplify() { return { hasFailed: this.hasFailed, stats: this.stats, duration: this.duration, - tests: this._tests.map(test => test.simplify()), + tests: this._tests.map(test => serializeTest(test)), failures: this._failures, } } diff --git a/lib/step/base.js b/lib/step/base.js index 85b66626e..08d72e9e5 100644 --- a/lib/step/base.js +++ b/lib/step/base.js @@ -185,12 +185,12 @@ class Step { if (step.args) { for (const arg of step.args) { // check if arg is a JOI object - if (arg && arg.$_root) { - args.push(JSON.stringify(arg).slice(0, 300)) - } else if (arg && typeof arg === 'function') { + if (arg && typeof arg === 'function') { args.push(arg.name) - } else { + } else if (typeof arg == 'string') { args.push(arg) + } else { + args.push(JSON.stringify(arg).slice(0, 300)) } } } @@ -198,9 +198,10 @@ class Step { return { opts: step.opts || {}, title: step.name, - args: JSON.stringify(args), + args: args, status: step.status, - duration: step.duration || 0, + startTime: step.startTime, + endTime: step.endTime, parent, } } diff --git a/lib/workers.js b/lib/workers.js index 74ab182b6..1e96beeee 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -387,8 +387,12 @@ class Workers extends EventEmitter { switch (message.event) { case event.all.result: + // we ensure consistency of result by adding tests in the very end Container.result().addFailures(message.data.failures) Container.result().addStats(message.data.stats) + message.data.tests.forEach(test => { + Container.result().addTest(deserializeTest(test)) + }) break case event.suite.before: this.emit(event.suite.before, deserializeSuite(message.data)) @@ -424,7 +428,7 @@ class Workers extends EventEmitter { this.emit(event.step.passed, message.data) break case event.step.failed: - this.emit(event.step.failed, message.data) + this.emit(event.step.failed, message.data, message.data.error) break } }) @@ -450,6 +454,7 @@ class Workers extends EventEmitter { } this.emit(event.all.result, Container.result()) + event.dispatcher.emit(event.workers.result, Container.result()) this.emit('end') // internal event } @@ -473,6 +478,7 @@ class Workers extends EventEmitter { } output.result(result.stats.passes, result.stats.failures, result.stats.pending, ms(result.duration), result.stats.failedHooks) + process.env.RUNS_WITH_WORKERS = 'false' } } From 7fdff376e173635dec992faf561f72b1b00ca509 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jan 2025 06:48:54 +0200 Subject: [PATCH 05/18] fixed tests --- lib/result.js | 2 +- test/unit/worker_test.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/result.js b/lib/result.js index 9fcd5d204..0a4c3d011 100644 --- a/lib/result.js +++ b/lib/result.js @@ -41,7 +41,7 @@ class Result { } get failures() { - return this._failures + return this._failures.filter(f => f && (!Array.isArray(f) || f.length > 0)) } get stats() { diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index fe051ca45..811eeae87 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -29,7 +29,7 @@ describe('Workers', function () { workers.run() workers.on(event.all.result, result => { - expect(result.hasFailed).equal(false) + expect(result.hasFailed).equal(true) expect(passedCount).equal(5) expect(failedCount).equal(3) done() @@ -101,7 +101,7 @@ describe('Workers', function () { }) workers.on(event.all.result, result => { - expect(result.hasFailed).equal(false) + expect(result.hasFailed).equal(true) expect(passedCount).equal(3) expect(failedCount).equal(2) done() From ee827b2a244f908d9291a250403784ae9b4e9688 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sun, 19 Jan 2025 06:55:27 +0200 Subject: [PATCH 06/18] added result to typings --- typings/jsdoc.conf.js | 1 + 1 file changed, 1 insertion(+) diff --git a/typings/jsdoc.conf.js b/typings/jsdoc.conf.js index ca1d115bf..6b38e805f 100644 --- a/typings/jsdoc.conf.js +++ b/typings/jsdoc.conf.js @@ -5,6 +5,7 @@ module.exports = { './lib/actor.js', './lib/codecept.js', './lib/config.js', + './lib/result.js', './lib/container.js', './lib/data/table.js', './lib/data/dataTableArgument.js', From e583cda6315ce92b740a36e03c7dd0ffba55aede Mon Sep 17 00:00:00 2001 From: DavertMik Date: Mon, 20 Jan 2025 06:02:06 +0200 Subject: [PATCH 07/18] fixed printing steps --- lib/mocha/cli.js | 1 + lib/mocha/test.js | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index 365d618af..fdce025d3 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -118,6 +118,7 @@ class Cli extends Base { } currentMetaStep = metaSteps output.stepShift = 3 + 2 * shift + output.step(step) }) event.dispatcher.on(event.step.finished, () => { diff --git a/lib/mocha/test.js b/lib/mocha/test.js index c0c3d6f48..47f5d11ee 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -127,6 +127,8 @@ function cloneTest(test) { function testToFileName(test) { let fileName = clearString(test.title) + fileName = fileName.replace(/\@\w+/g, '') + fileName = fileName.slice(0, 100) if (fileName.indexOf('{') !== -1) { fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim() } From 82075c8f50434fecc84e16ab993a62a5fae09612 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Tue, 21 Jan 2025 07:00:32 +0200 Subject: [PATCH 08/18] fixed def & failing tests --- docs/plugins.md | 197 ++++-------------- lib/codecept.js | 3 +- lib/command/check.js | 4 + lib/listener/globalTimeout.js | 2 +- lib/listener/steps.js | 1 - lib/listener/store.js | 4 - lib/mocha/cli.js | 18 +- lib/mocha/test.js | 2 +- lib/plugin/analyze.js | 116 +++++++---- lib/recorder.js | 4 +- lib/result.js | 40 ++++ lib/utils.js | 23 ++ lib/workers.js | 1 - .../sandbox/configs/timeouts/suite_test.js | 2 +- test/runner/before_failure_test.js | 3 + test/runner/run_workers_test.js | 2 - 16 files changed, 200 insertions(+), 222 deletions(-) diff --git a/docs/plugins.md b/docs/plugins.md index 155550f7d..9c88deeab 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -7,6 +7,12 @@ title: Plugins +## analyze + +### Parameters + +- `config` **any** (optional, default `{}`) + ## autoDelay Sometimes it takes some time for a page to respond to user's actions. @@ -521,29 +527,13 @@ I.click('=sign-up') // matches => [data-qa=sign-up],[data-test=sign-up] - `config` -## debugErrors - -Prints errors found in HTML code after each failed test. - -It scans HTML and searches for elements with error classes. -If an element found prints a text from it to console and adds as artifact to the test. - -Enable this plugin in config: - -```js -plugins: { - debugErrors: { - enabled: true, -} -``` - -Additional config options: +## customReporter -- `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`) +Sample custom reporter for CodeceptJS. ### Parameters -- `config` (optional, default `{}`) +- `config` ## eachElement @@ -672,6 +662,32 @@ More config options are available: - `config` (optional, default `{}`) +## pageInfo + +Collects information from web page after each failed test and adds it to the test as an artifact. +It is suggested to enable this plugin if you run tests on CI and you need to debug failed tests. +This plugin can be paired with `analyze` plugin to provide more context. + +It collects URL, HTML errors (by classes), and browser logs. + +Enable this plugin in config: + +```js +plugins: { + pageInfo: { + enabled: true, +} +``` + +Additional config options: + +- `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`) +- `browserLogs` - list of types of errors to search for in browser logs (default: `['error']`) + +### Parameters + +- `config` (optional, default `{}`) + ## pauseOnFail Automatically launches [interactive pause][11] when a test fails. @@ -757,79 +773,6 @@ Scenario('scenario tite', () => { - `config` -## retryTo - -Adds global `retryTo` which retries steps a few times before failing. - -Enable this plugin in `codecept.conf.js` (enabled by default for new setups): - -```js -plugins: { - retryTo: { - enabled: true - } -} -``` - -Use it in your tests: - -```js -// retry these steps 5 times before failing -await retryTo(tryNum => { - I.switchTo('#editor frame') - I.click('Open') - I.see('Opened') -}, 5) -``` - -Set polling interval as 3rd argument (200ms by default): - -```js -// retry these steps 5 times before failing -await retryTo( - tryNum => { - I.switchTo('#editor frame') - I.click('Open') - I.see('Opened') - }, - 5, - 100, -) -``` - -Default polling interval can be changed in a config: - -```js -plugins: { - retryTo: { - enabled: true, - pollInterval: 500, - } -} -``` - -Disables retryFailedStep plugin for steps inside a block; - -Use this plugin if: - -- you need repeat a set of actions in flaky tests -- iframe was not rendered and you need to retry switching to it - -#### Configuration - -- `pollInterval` - default interval between retries in ms. 200 by default. -- `registerGlobal` - to register `retryTo` function globally, true by default - -If `registerGlobal` is false you can use retryTo from the plugin: - -```js -const retryTo = codeceptjs.container.plugins('retryTo') -``` - -### Parameters - -- `config` - ## screenshotOnFail Creates screenshot on failure. Screenshot is saved into `output` directory. @@ -1083,76 +1026,6 @@ plugins: { } ``` -## tryTo - -Adds global `tryTo` function in which all failed steps won't fail a test but will return true/false. - -Enable this plugin in `codecept.conf.js` (enabled by default for new setups): - -```js -plugins: { - tryTo: { - enabled: true - } -} -``` - -Use it in your tests: - -```js -const result = await tryTo(() => I.see('Welcome')) - -// if text "Welcome" is on page, result => true -// if text "Welcome" is not on page, result => false -``` - -Disables retryFailedStep plugin for steps inside a block; - -Use this plugin if: - -- you need to perform multiple assertions inside a test -- there is A/B testing on a website you test -- there is "Accept Cookie" banner which may surprisingly appear on a page. - -#### Usage - -#### Multiple Conditional Assertions - -````js - -Add assert requires first: -```js -const assert = require('assert'); -```` - -Then use the assertion: -const result1 = await tryTo(() => I.see('Hello, user')); -const result2 = await tryTo(() => I.seeElement('.welcome')); -assert.ok(result1 && result2, 'Assertions were not succesful'); - -```` - -##### Optional click - -```js -I.amOnPage('/'); -tryTo(() => I.click('Agree', '.cookies')); -```` - -#### Configuration - -- `registerGlobal` - to register `tryTo` function globally, true by default - -If `registerGlobal` is false you can use tryTo from the plugin: - -```js -const tryTo = codeceptjs.container.plugins('tryTo') -``` - -### Parameters - -- `config` - ## wdio Webdriverio services runner. diff --git a/lib/codecept.js b/lib/codecept.js index a3e61744a..c51da9c82 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -9,7 +9,6 @@ const event = require('./event') const runHook = require('./hooks') const output = require('./output') const { emptyFolder } = require('./utils') -const Result = require('./result') /** * CodeceptJS runner @@ -201,7 +200,7 @@ class Codecept { } const done = () => { event.emit(event.all.result, container.result()) - event.emit(event.all.after, container.result()) + event.emit(event.all.after) resolve() } diff --git a/lib/command/check.js b/lib/command/check.js index bfa45cc67..4c32ba359 100644 --- a/lib/command/check.js +++ b/lib/command/check.js @@ -22,6 +22,7 @@ module.exports = async function (options) { config: false, container: false, pageObjects: false, + plugins: false, helpers: false, setup: false, tests: false, @@ -115,6 +116,9 @@ module.exports = async function (options) { } printCheck('page objects', checks['pageObjects'], `Total: ${Object.keys(pageObjects).length} support objects`) + checks.plugins = true // how to check plugins? + printCheck('plugins', checks['plugins'], Object.keys(container.plugins()).join(', ')) + if (Object.keys(helpers).length) { const suite = container.mocha().suite const test = createTest('test', () => {}) diff --git a/lib/listener/globalTimeout.js b/lib/listener/globalTimeout.js index 6bb7dd1fe..182f11428 100644 --- a/lib/listener/globalTimeout.js +++ b/lib/listener/globalTimeout.js @@ -133,7 +133,7 @@ module.exports = function () { debug(`Failing test ${currentTest.title} with timeout ${currentTimeout}s`) recorder.reset() // replace mocha timeout with custom timeout - currentTest.timeout(0) + // currentTest.timeout(0.1) currentTest.callback(new Error(`Timeout ${currentTimeout}s exceeded (with Before hook)`)) currentTest.timedOut = true } diff --git a/lib/listener/steps.js b/lib/listener/steps.js index c7b5beb1c..bcfb1b1ec 100644 --- a/lib/listener/steps.js +++ b/lib/listener/steps.js @@ -13,7 +13,6 @@ let currentHook module.exports = function () { event.dispatcher.on(event.test.before, test => { test.startedAt = +new Date() - test.artifacts = {} }) event.dispatcher.on(event.test.started, test => { diff --git a/lib/listener/store.js b/lib/listener/store.js index 73377e922..763aa1edc 100644 --- a/lib/listener/store.js +++ b/lib/listener/store.js @@ -2,10 +2,6 @@ const event = require('../event') const store = require('../store') module.exports = function () { - event.dispatcher.on(event.all.before, result => { - store.result = result - }) - event.dispatcher.on(event.test.before, test => { store.currentTest = test }) diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index fdce025d3..3c7a4e3f5 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -155,9 +155,10 @@ class Cli extends Base { result() { const container = require('../container') - const stats = this.stats - container.result().addStats(stats) + container.result().addStats(this.stats) container.result().finish() + + const stats = container.result().stats console.log() // passes @@ -181,8 +182,15 @@ class Cli extends Base { err.message = err.inspect() } - // multi-line error messages - err.message = '\n ' + (err.message || '').replace(/^/gm, ' ').trim() + // multi-line error messages (for Playwright) + if (err.message && err.message.includes('\n')) { + const lines = err.message.split('\n') + const truncatedLines = lines.slice(0, 5) + if (lines.length > 5) { + truncatedLines.push('...') + } + err.message = '\n ' + truncatedLines.join('\n').replace(/^/gm, ' ').trim() + } const steps = test.steps || (test.ctx && test.ctx.test.steps) @@ -249,7 +257,7 @@ class Cli extends Base { container.result().addFailures(failuresLog) - output.result(stats.passes, stats.failures, stats.pending, ms(stats.duration)) + output.result(stats.passes, stats.failures, stats.pending, ms(stats.duration), stats.failedHooks) if (stats.failures && output.level() < 3) { output.print(output.styles.debug('Run with --verbose flag to see complete NodeJS stacktrace')) diff --git a/lib/mocha/test.js b/lib/mocha/test.js index 47f5d11ee..55a4341ec 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -71,7 +71,7 @@ function deserializeTest(test) { createTest(test.title || '', () => {}), test, ) - test.parent = Object.assign(new Suite(test.parent.title), test.parent) + test.parent = Object.assign(new Suite(test.parent?.title || 'Suite'), test.parent) enhanceMochaSuite(test.parent) test.steps = test.steps.map(step => Object.assign(new Step(step.title), step)) return test diff --git a/lib/plugin/analyze.js b/lib/plugin/analyze.js index 3c3bc6748..87bf1030a 100644 --- a/lib/plugin/analyze.js +++ b/lib/plugin/analyze.js @@ -7,25 +7,37 @@ const colors = require('chalk') const ora = require('ora-classic') const event = require('../event') const output = require('../output') -const { ansiRegExp } = require('../utils') +const { ansiRegExp, base64EncodeFile, markdownToAnsi } = require('../utils') const MAX_DATA_LENGTH = 5000 const defaultConfig = { - clusterize: 5, + clusterize: 2, analyze: 3, + vision: false, + categories: [ + 'Browser connection error / browser crash', + 'Network errors (server error, timeout, etc)', + 'HTML / page elements (not found, not visible, etc)', + 'Navigation errors (404, etc)', + 'Code errors (syntax error, JS errors, etc)', + 'Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc)', + 'Data errors (password incorrect, no options in select, invalid format, etc)', + 'Assertion failures', + 'Other errors', + ], prompts: { - clusterize: testsAndErrors => { - const serializedFailedTests = testsAndErrors + clusterize: (tests, config) => { + const serializedFailedTests = tests .map((test, index) => { if (!test || !test.err) return return ` TEST #${index + 1}: ${serializeTest(test)} - ERROR: ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / testsAndErrors.length)}` + ERROR: ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / tests.length)}` }) .join('\n\n---\n\n') - return [ + const messages = [ { role: 'user', content: ` @@ -43,16 +55,7 @@ const defaultConfig = { Common categories of failures by order of priority: - * Browser connection error / browser crash - * Network errors (server error, timeout, etc) - * HTML / page elements (not found, not visible, etc) - * Navigation errors (404, etc) - * Code errors (syntax error, JS errors, etc) - * Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc) - * Data errors (password incorrect, no options in select, invalid format, etc) - * Assertion failures - * Other errors - + ${config.categories.join('\n- ')} If there is no groups of tests, say: "No patterns found" Preserve error messages but cut them if they are too long. @@ -67,7 +70,7 @@ const defaultConfig = { _______________________________ - --- GROUP # <(percentage of failed tests)'> + --- GROUP # <(percentage of failed tests)'> CATEGORY ERRORS , , ... SUMMARY @@ -83,38 +86,38 @@ const defaultConfig = { `, }, ] + return messages }, - analyze: (test, error) => { + analyze: (test, config) => { const testMessage = serializeTest(test) - const errorMessage = serializeError(error) - // TODO: if vision is on, attach screenshot to the prompt + const errorMessage = serializeError(test.err) - return [ + const messages = [ { role: 'user', - content: ` + content: [ + { + type: 'text', + text: ` I am qa engineer analyzing failed tests in CodeceptJS testing framework. Please analyze the following failed test and error its error and explain it. Pick one of the categories of failures and explain it. - Common causes of failures in order of priority: - - * Browser connection error / browser crash - * Network errors (server error, timeout, etc) - * HTML / page elements (not found, not visible, etc) - * Navigation errors (404, etc) - * Code errors (syntax error, JS errors, etc) - * Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc) - * Data errors (password incorrect, no options in select, invalid format, etc) - * Assertion failures - * Other errors + Categories of failures in order of priority: + ${config.categories.join('\n- ')} Here is the test and error: + + ------- TEST ------- ${testMessage} + + ------- ERROR ------- ${errorMessage} + ------ INSTRUCTIONS ------ + Do not get to details, be concise. If there is failed step, just write it in STEPS section. If you have suggestions for the test, write them in SUMMARY section. @@ -128,9 +131,24 @@ const defaultConfig = { SUMMARY Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS. + ${config.vision ? 'Also a screenshot of the page is attached to the prompt.' : ''} `, + }, + ], }, ] + + if (config.vision && test.artifacts.screenshot) { + debug('Adding screenshot to prompt') + messages[0].content.push({ + type: 'image_url', + image_url: { + url: 'data:image/png;base64,' + base64EncodeFile(test.artifacts.screenshot), + }, + }) + } + + return messages }, }, } @@ -143,6 +161,11 @@ const defaultConfig = { module.exports = function (config = {}) { config = Object.assign(defaultConfig, config) + event.dispatcher.on(event.workers.before, () => { + if (!ai.isEnabled) return + console.log('Enabled AI analysis') + }) + event.dispatcher.on(event.all.result, async result => { if (!isMainThread) return // run only on main thread if (!ai.isEnabled) { @@ -169,20 +192,19 @@ module.exports = function (config = {}) { debug(failedTestsAndErrors.map(t => serializeTest(t) + '\n' + serializeError(t.err))) - console.log() - console.log(colors.bold.white('🪄 AI REPORT:')) - try { if (failedTestsAndErrors.length >= config.clusterize) { const response = await clusterize(failedTestsAndErrors) + printHeader() console.log(response) return } output.plugin('analyze', `Analyzing first ${config.analyze} failed tests...`) + // we pick only unique errors to not repeat answers const uniqueErrors = failedTestsAndErrors.filter((item, index, array) => { - return array.findIndex(t => serializeError(t.err) === serializeError(item.err)) === index + return array.findIndex(t => t.err?.message === item.err?.message) === index }) for (let i = 0; i < config.analyze; i++) { @@ -193,11 +215,14 @@ module.exports = function (config = {}) { break } + printHeader() console.log() console.log('--------------------------------') - console.log(arrowRight, colors.bold.white(uniqueErrors[i].fullTitle())) + console.log(arrowRight, colors.bold.white(uniqueErrors[i].fullTitle()), config.vision ? '👀' : '') + console.log() console.log() console.log(response) + console.log() } } catch (err) { console.error('Error analyzing failed tests', err) @@ -208,9 +233,19 @@ module.exports = function (config = {}) { } } + let hasPrintedHeader = false + + function printHeader() { + if (!hasPrintedHeader) { + console.log() + console.log(colors.bold.white('🪄 AI REPORT:')) + hasPrintedHeader = true + } + } + async function clusterize(failedTestsAndErrors) { const spinner = ora('Clusterizing failures...').start() - const prompt = config.prompts.clusterize(failedTestsAndErrors) + const prompt = config.prompts.clusterize(failedTestsAndErrors, config) try { const response = await ai.createCompletion(prompt) spinner.stop() @@ -223,7 +258,7 @@ module.exports = function (config = {}) { async function analyze(failedTestAndError) { const spinner = ora('Analyzing failure...').start() - const prompt = config.prompts.analyze(failedTestAndError, failedTestAndError.err) + const prompt = config.prompts.analyze(failedTestAndError, config) try { const response = await ai.createCompletion(prompt) spinner.stop() @@ -303,5 +338,6 @@ function formatResponse(response) { if (line.startsWith('CATEGORY')) return line.replace('CATEGORY', colors.bold.bgGreen('CATEGORY ')) return line }) + .map(line => markdownToAnsi(line)) .join('\n') } diff --git a/lib/recorder.js b/lib/recorder.js index fa60727d5..2c3a3e328 100644 --- a/lib/recorder.js +++ b/lib/recorder.js @@ -191,13 +191,13 @@ module.exports = { .slice(-1) .pop() // no retries or unnamed tasks + debug(`${currentQueue()} Running | ${taskName} | Timeout: ${timeout || 'None'}`) + if (!retryOpts || !taskName || !retry) { const [promise, timer] = getTimeoutPromise(timeout, taskName) return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer)) } - debug(`${currentQueue()} Running | ${taskName}`) - const retryRules = this.retries.slice().reverse() return promiseRetry(Object.assign(defaultRetryOptions, retryOpts), (retry, number) => { if (number > 1) log(`${currentQueue()}Retrying... Attempt #${number}`) diff --git a/lib/result.js b/lib/result.js index 0a4c3d011..7824f2bc8 100644 --- a/lib/result.js +++ b/lib/result.js @@ -2,7 +2,23 @@ const fs = require('fs') const path = require('path') const { serializeTest } = require('./mocha/test') +/** + * Result of the test run + * + * @typedef {Object} Stats + * @property {number} passes + * @property {number} failures + * @property {number} tests + * @property {number} pending + * @property {number} failedHooks + * @property {Date} start + * @property {Date} end + * @property {number} duration + */ class Result { + /** + * Create Result of the test run + */ constructor() { this._stats = { passes: 0, @@ -10,6 +26,9 @@ class Result { tests: 0, pending: 0, failedHooks: 0, + start: null, + end: null, + duration: 0, } this._startTime = new Date() @@ -52,6 +71,11 @@ class Result { return this._startTime } + /** + * Add test to result + * + * @param {CodeceptJS.Test} test + */ addTest(test) { const existingTestIndex = this._tests.findIndex(t => !!t.uid && t.uid === test.uid) if (existingTestIndex >= 0) { @@ -101,17 +125,33 @@ class Result { } } + /** + * Save result to json file + * + * @param {string} fileName + */ save(fileName) { if (!fileName) fileName = 'result.json' fs.writeFileSync(path.join(codeceptjs.outputDir, fileName), JSON.stringify(this.simplify(), null, 2)) } + /** + * Add stats to result + * + * @param {object} newStats + */ addStats(newStats = {}) { this._stats.passes += newStats.passes || 0 this._stats.failures += newStats.failures || 0 this._stats.tests += newStats.tests || 0 this._stats.pending += newStats.pending || 0 this._stats.failedHooks += newStats.failedHooks || 0 + + // do not override start time + this._stats.start = this._stats.start || newStats.start + + this._stats.end = newStats.end || this._stats.end + this._stats.duration = newStats.duration } } diff --git a/lib/utils.js b/lib/utils.js index 9310b3350..25a81634f 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,6 +1,7 @@ const fs = require('fs') const os = require('os') const path = require('path') +const chalk = require('chalk') const getFunctionArguments = require('fn-args') const deepClone = require('lodash.clonedeep') const { convertColorToRGBA, isColorProperty } = require('./colorUtils') @@ -550,3 +551,25 @@ module.exports.serializeError = function (error) { } return null } + +module.exports.base64EncodeFile = function (filePath) { + return Buffer.from(fs.readFileSync(filePath)).toString('base64') +} + +module.exports.markdownToAnsi = function (markdown) { + return ( + markdown + // Headers (# Text) - make blue and bold + .replace(/^(#{1,6})\s+(.+)$/gm, (_, hashes, text) => { + return chalk.bold.blue(`${hashes} ${text}`) + }) + // Bold (**text**) - make bold + .replace(/\*\*(.+?)\*\*/g, (_, text) => { + return chalk.bold(text) + }) + // Italic (*text*) - make italic (dim in terminals) + .replace(/\*(.+?)\*/g, (_, text) => { + return chalk.italic(text) + }) + ) +} diff --git a/lib/workers.js b/lib/workers.js index 1e96beeee..1576263b3 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -11,7 +11,6 @@ const { isFunction, fileExists } = require('./utils') const { replaceValueDeep, deepClone } = require('./utils') const mainConfig = require('./config') const output = require('./output') -const store = require('./store') const event = require('./event') const { deserializeTest } = require('./mocha/test') const { deserializeSuite } = require('./mocha/suite') diff --git a/test/data/sandbox/configs/timeouts/suite_test.js b/test/data/sandbox/configs/timeouts/suite_test.js index f53f2bfb7..07927bb8f 100644 --- a/test/data/sandbox/configs/timeouts/suite_test.js +++ b/test/data/sandbox/configs/timeouts/suite_test.js @@ -10,7 +10,7 @@ Scenario('timeout test in 0.5 #second', { timeout: 0.5 }, ({ I }) => { I.waitForSleep(1000) }) -Scenario('timeout step in 0.5', ({ I }) => { +Scenario('timeout step in 0.5 old syntax', ({ I }) => { I.limitTime(0.2).waitForSleep(100) I.limitTime(0.2).waitForSleep(3000) }) diff --git a/test/runner/before_failure_test.js b/test/runner/before_failure_test.js index 56274a5da..6b6169d79 100644 --- a/test/runner/before_failure_test.js +++ b/test/runner/before_failure_test.js @@ -1,5 +1,6 @@ const path = require('path') const exec = require('child_process').exec +const debug = require('debug')('codeceptjs:test') const runner = path.join(__dirname, '/../../bin/codecept.js') const codecept_dir = path.join(__dirname, '/../data/sandbox') @@ -9,6 +10,7 @@ describe('Failure in before', function () { this.timeout(40000) it('should skip tests that are skipped because of failure in before hook', done => { exec(`${codecept_run}`, (err, stdout) => { + debug(stdout) stdout.should.include('First test will be passed @grep') stdout.should.include('Third test will be skipped @grep') stdout.should.include('Fourth test will be skipped') @@ -20,6 +22,7 @@ describe('Failure in before', function () { it('should skip tests correctly with grep options', done => { exec(`${codecept_run} --grep @grep`, (err, stdout) => { + debug(stdout) stdout.should.include('First test will be passed @grep') stdout.should.include('Third test will be skipped @grep') stdout.should.include('1 passed, 1 failed, 1 failedHooks, 1 skipped') diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 87790c570..ba06d9e54 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -107,8 +107,6 @@ describe('CodeceptJS Workers Runner', function () { exec(`${codecept_run} 2 --grep "Workers Failing"`, (err, stdout) => { expect(stdout).toContain('CodeceptJS') // feature expect(stdout).toContain('Running tests in 2 workers') - // Test Scenario wasn't executed, but we can see it in logs because Before() hook was executed - expect(stdout).not.toContain(' should not be executed ') expect(stdout).toContain('"before each" hook: Before for "should not be executed"') expect(stdout).not.toContain('this is running inside worker') expect(stdout).toContain('failed') From 843fd50a2bfa99b0faca9c09ce3269b732b556af Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 23 Jan 2025 05:08:10 +0200 Subject: [PATCH 09/18] fixing timeout errors --- examples/codecept.config.js | 6 ------ lib/listener/exit.js | 13 +++++-------- lib/listener/globalTimeout.js | 5 +++-- lib/mocha/cli.js | 6 +++++- lib/mocha/suite.js | 1 - lib/mocha/test.js | 7 ++++--- lib/mocha/types.d.ts | 1 + lib/output.js | 2 +- lib/plugin/retryTo.js | 12 ++++++++++-- lib/plugin/tryTo.js | 10 +++++++++- lib/step/base.js | 1 + lib/step/helper.js | 3 +++ lib/step/record.js | 8 ++++---- 13 files changed, 46 insertions(+), 29 deletions(-) diff --git a/examples/codecept.config.js b/examples/codecept.config.js index 9223a3a77..ceac01db0 100644 --- a/examples/codecept.config.js +++ b/examples/codecept.config.js @@ -37,9 +37,6 @@ exports.config = { steps: ['./step_definitions/steps.js'], }, plugins: { - tryTo: { - enabled: true, - }, analyze: { enabled: true, }, @@ -63,9 +60,6 @@ exports.config = { subtitles: { enabled: true, }, - retryTo: { - enabled: true, - }, }, tests: './*_test.js', diff --git a/lib/listener/exit.js b/lib/listener/exit.js index 10cd8cd23..c510a3c16 100644 --- a/lib/listener/exit.js +++ b/lib/listener/exit.js @@ -1,20 +1,17 @@ const event = require('../event') +const debug = require('debug')('codeceptjs:exit') module.exports = function () { let failedTests = [] - event.dispatcher.on(event.test.failed, testOrSuite => { - // NOTE When an error happens in one of the hooks (BeforeAll/BeforeEach...) the event object - // is a suite and not a test - const id = testOrSuite.uid || (testOrSuite.ctx && testOrSuite.ctx.test.uid) || 'empty' + event.dispatcher.on(event.test.failed, test => { + const id = test.uid || (test.ctx && test.ctx.test.uid) || 'empty' failedTests.push(id) }) // if test was successful after retries - event.dispatcher.on(event.test.passed, testOrSuite => { - // NOTE When an error happens in one of the hooks (BeforeAll/BeforeEach...) the event object - // is a suite and not a test - const id = testOrSuite.uid || (testOrSuite.ctx && testOrSuite.ctx.test.uid) || 'empty' + event.dispatcher.on(event.test.passed, test => { + const id = test.uid || (test.ctx && test.ctx.test.uid) || 'empty' failedTests = failedTests.filter(failed => id !== failed) }) diff --git a/lib/listener/globalTimeout.js b/lib/listener/globalTimeout.js index 182f11428..e43258b21 100644 --- a/lib/listener/globalTimeout.js +++ b/lib/listener/globalTimeout.js @@ -125,15 +125,16 @@ module.exports = function () { return } + debug(`step ${step.toCode().trim()}:${step.status} duration`, step.duration) if (typeof timeout === 'number' && !Number.isNaN(timeout)) timeout -= step.duration if (typeof timeout === 'number' && timeout <= 0 && recorder.isRunning()) { debug(`step ${step.toCode().trim()} timed out`) - if (currentTest && currentTest.callback) { + if (currentTest && currentTest.callback && currentTest.type == 'test') { debug(`Failing test ${currentTest.title} with timeout ${currentTimeout}s`) recorder.reset() // replace mocha timeout with custom timeout - // currentTest.timeout(0.1) + currentTest.timeout(0.1) currentTest.callback(new Error(`Timeout ${currentTimeout}s exceeded (with Before hook)`)) currentTest.timedOut = true } diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index 3c7a4e3f5..0690539f2 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -230,11 +230,15 @@ class Cli extends Base { stack.shift() } + if (stack[0].trim() == 'Error:') { + stack.shift() + } + if (output.level() < 3) { stack = stack.slice(0, 3) } - err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}`.trim() + err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}` } catch (e) { console.error(e) } diff --git a/lib/mocha/suite.js b/lib/mocha/suite.js index 7282f930b..ab9e5ec1f 100644 --- a/lib/mocha/suite.js +++ b/lib/mocha/suite.js @@ -1,5 +1,4 @@ const MochaSuite = require('mocha/lib/suite') - /** * @typedef {import('mocha')} Mocha */ diff --git a/lib/mocha/test.js b/lib/mocha/test.js index 55a4341ec..045428be5 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -2,7 +2,7 @@ const Test = require('mocha/lib/test') const Suite = require('mocha/lib/suite') const { test: testWrapper } = require('./asyncWrapper') const { enhanceMochaSuite, createSuite } = require('./suite') -const { genTestId, serializeError, clearString } = require('../utils') +const { genTestId, serializeError, clearString, relativeDir } = require('../utils') const Step = require('../step/base') /** * Factory function to create enhanced tests @@ -46,6 +46,7 @@ function enhanceMochaTest(test) { test.addToSuite = function (suite) { enhanceMochaSuite(suite) suite.addTest(testWrapper(this)) + if (test.file) suite.file = relativeDir(test.file) test.tags = [...(test.tags || []), ...(suite.tags || [])] test.fullTitle = () => `${suite.title}: ${test.title}` test.uid = genTestId(test) @@ -73,7 +74,7 @@ function deserializeTest(test) { ) test.parent = Object.assign(new Suite(test.parent?.title || 'Suite'), test.parent) enhanceMochaSuite(test.parent) - test.steps = test.steps.map(step => Object.assign(new Step(step.title), step)) + if (test.steps) test.steps = test.steps.map(step => Object.assign(new Step(step.title), step)) return test } @@ -117,7 +118,7 @@ function serializeTest(test, err = null) { duration: test.duration || 0, err, parent, - steps: [...test.steps].map(step => (step.simplify ? step.simplify() : step)), + steps: test.steps?.toArray()?.map(step => (step.simplify ? step.simplify() : step)), } } diff --git a/lib/mocha/types.d.ts b/lib/mocha/types.d.ts index 7372ba553..07daf22c0 100644 --- a/lib/mocha/types.d.ts +++ b/lib/mocha/types.d.ts @@ -18,6 +18,7 @@ declare global { opts: Record throws?: Error | string | RegExp | Function totalTimeout?: number + relativeFile?: string addToSuite(suite: Mocha.Suite): void applyOptions(opts: Record): void simplify(): Record diff --git a/lib/output.js b/lib/output.js index fc209d043..1bd32ba03 100644 --- a/lib/output.js +++ b/lib/output.js @@ -135,7 +135,7 @@ module.exports = { */ started: suite => { if (!suite.title) return - print(`${colors.bold(suite.title)} --`) + print(`${colors.bold(suite.title)} --`, colors.underline.grey(suite.file || '')) if (suite.comment) print(suite.comment) }, }, diff --git a/lib/plugin/retryTo.js b/lib/plugin/retryTo.js index ed9c3cdfc..71405d974 100644 --- a/lib/plugin/retryTo.js +++ b/lib/plugin/retryTo.js @@ -1,6 +1,8 @@ -module.exports = function () { +const { retryTo } = require('../effects') + +module.exports = function (config) { console.log(` -Deprecated Warning: 'retryTo' has been moved to the effects module. +Deprecation Warning: 'retryTo' has been moved to the effects module. You should update your tests to use it as follows: \`\`\`javascript @@ -16,4 +18,10 @@ await retryTo((tryNum) => { For more details, refer to the documentation. `) + + if (config.registerGlobal) { + global.retryTo = retryTo + } + + return retryTo } diff --git a/lib/plugin/tryTo.js b/lib/plugin/tryTo.js index 195cea28a..2c6fa13a0 100644 --- a/lib/plugin/tryTo.js +++ b/lib/plugin/tryTo.js @@ -1,4 +1,6 @@ -module.exports = function () { +const { tryTo } = require('../effects') + +module.exports = function (config) { console.log(` Deprecated Warning: 'tryTo' has been moved to the effects module. You should update your tests to use it as follows: @@ -14,4 +16,10 @@ await tryTo(() => { For more details, refer to the documentation. `) + + if (config.registerGlobal) { + global[config.registerGlobal] = tryTo + } + + return tryTo } diff --git a/lib/step/base.js b/lib/step/base.js index 08d72e9e5..1dce7b867 100644 --- a/lib/step/base.js +++ b/lib/step/base.js @@ -163,6 +163,7 @@ class Step { } get duration() { + if (!this.startTime || !this.endTime) return 0 return this.endTime - this.startTime } diff --git a/lib/step/helper.js b/lib/step/helper.js index b52470e3c..ade2a0d3d 100644 --- a/lib/step/helper.js +++ b/lib/step/helper.js @@ -16,6 +16,7 @@ class HelperStep extends Step { */ run() { this.args = Array.prototype.slice.call(arguments) + this.startTime = +Date.now() if (store.dryRun) { this.setStatus('success') @@ -27,7 +28,9 @@ class HelperStep extends Step { result = this.helper[this.helperMethod].apply(this.helper, this.args) } this.setStatus('success') + this.endTime = +Date.now() } catch (err) { + this.endTime = +Date.now() this.setStatus('failed') throw err } diff --git a/lib/step/record.js b/lib/step/record.js index 40922b401..ed7a45699 100644 --- a/lib/step/record.js +++ b/lib/step/record.js @@ -40,7 +40,7 @@ function recordStep(step, args) { if (!step.startTime) { // step can be retries event.emit(event.step.started, step) - step.startTime = Date.now() + step.startTime = +Date.now() } return (val = step.run(...args)) }, @@ -52,15 +52,15 @@ function recordStep(step, args) { event.emit(event.step.after, step) recorder.add('step passed', () => { - step.endTime = Date.now() + step.endTime = +Date.now() event.emit(event.step.passed, step, val) event.emit(event.step.finished, step) }) recorder.catchWithoutStop(err => { step.status = 'failed' - step.endTime = Date.now() - event.emit(event.step.failed, step) + step.endTime = +Date.now() + event.emit(event.step.failed, step, err) event.emit(event.step.finished, step) throw err }) From 1f30058e22a42f6c778e3c39e39413342e9a3f98 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 23 Jan 2025 08:22:52 +0200 Subject: [PATCH 10/18] fixed mocha hooks, analyze plugin, added custom reporter tests --- lib/codecept.js | 2 +- lib/command/run-workers.js | 9 -- lib/command/workers/runTests.js | 3 +- lib/event.js | 35 ++++---- lib/listener/globalTimeout.js | 2 +- lib/listener/result.js | 2 +- lib/mocha/asyncWrapper.js | 14 ++- lib/mocha/cli.js | 2 +- lib/mocha/hooks.js | 25 +++++- lib/mocha/test.js | 21 +++-- lib/mocha/types.d.ts | 2 + lib/output.js | 3 +- lib/plugin/analyze.js | 52 ++++++------ lib/plugin/customReporter.js | 30 +++++-- lib/plugin/tryTo.js | 2 +- lib/rerun.js | 85 ++++++++++--------- lib/result.js | 19 +++-- lib/utils.js | 4 + .../custom-reporter-plugin/codecept.conf.js | 44 ++++++++++ .../custom-reporter-plugin_test.js | 22 +++++ test/runner/custom-reporter-plugin_test.js | 41 +++++++++ test/runner/run_workers_test.js | 2 +- test/unit/plugin/retryFailedStep_test.js | 25 ------ 23 files changed, 292 insertions(+), 154 deletions(-) create mode 100644 test/data/sandbox/configs/custom-reporter-plugin/codecept.conf.js create mode 100644 test/data/sandbox/configs/custom-reporter-plugin/custom-reporter-plugin_test.js create mode 100644 test/runner/custom-reporter-plugin_test.js diff --git a/lib/codecept.js b/lib/codecept.js index c51da9c82..f1a76ca00 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -205,7 +205,7 @@ class Codecept { } try { - event.emit(event.all.before, container.result()) + event.emit(event.all.before) mocha.run(() => done()) } catch (e) { output.error(e.stack) diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index ac956d291..20a26e2c8 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -8,12 +8,6 @@ const Workers = require('../workers') module.exports = async function (workerCount, selectedRuns, options) { process.env.profile = options.profile - const suiteArr = [] - const passedTestArr = [] - const failedTestArr = [] - const skippedTestArr = [] - const stepArr = [] - const { config: testConfig, override = '' } = options const overrideConfigs = tryOrDefault(() => JSON.parse(override), {}) const by = options.suites ? 'suite' : 'test' @@ -36,17 +30,14 @@ module.exports = async function (workerCount, selectedRuns, options) { workers.overrideConfig(overrideConfigs) workers.on(event.test.failed, test => { - failedTestArr.push(test) output.test.failed(test) }) workers.on(event.test.passed, test => { - passedTestArr.push(test) output.test.passed(test) }) workers.on(event.test.skipped, test => { - skippedTestArr.push(test) output.test.skipped(test) }) diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index 3df4b1d9a..d6222575a 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -99,7 +99,8 @@ function initializeListeners() { event.dispatcher.on(event.step.failed, step => sendToParentThread({ event: event.step.failed, workerIndex, data: step.simplify() })) event.dispatcher.on(event.hook.failed, (hook, err) => sendToParentThread({ event: event.hook.failed, workerIndex, data: { ...hook.simplify(), err } })) - event.dispatcher.on(event.hook.passed, (hook, err) => sendToParentThread({ event: event.hook.passed, workerIndex, data: { ...hook.simplify(), err } })) + event.dispatcher.on(event.hook.passed, hook => sendToParentThread({ event: event.hook.passed, workerIndex, data: hook.simplify() })) + event.dispatcher.on(event.hook.finished, hook => sendToParentThread({ event: event.hook.finished, workerIndex, data: hook.simplify() })) event.dispatcher.once(event.all.after, () => { sendToParentThread({ event: event.all.after, workerIndex, data: container.result().simplify() }) diff --git a/lib/event.js b/lib/event.js index d7cc05046..4dd500d8f 100644 --- a/lib/event.js +++ b/lib/event.js @@ -1,10 +1,10 @@ -const debug = require('debug')('codeceptjs:event'); -const events = require('events'); -const { error } = require('./output'); +const debug = require('debug')('codeceptjs:event') +const events = require('events') +const { error } = require('./output') -const dispatcher = new events.EventEmitter(); +const dispatcher = new events.EventEmitter() -dispatcher.setMaxListeners(50); +dispatcher.setMaxListeners(50) /** * @namespace * @alias event @@ -59,6 +59,7 @@ module.exports = { started: 'hook.start', passed: 'hook.passed', failed: 'hook.failed', + finished: 'hook.finished', }, /** @@ -141,33 +142,33 @@ module.exports = { * @param {*} [param] */ emit(event, param) { - let msg = `Emitted | ${event}`; + let msg = `Emitted | ${event}` if (param && param.toString()) { - msg += ` (${param.toString()})`; + msg += ` (${param.toString()})` } - debug(msg); + debug(msg) try { - this.dispatcher.emit.apply(this.dispatcher, arguments); + this.dispatcher.emit.apply(this.dispatcher, arguments) } catch (err) { - error(`Error processing ${event} event:`); - error(err.stack); + error(`Error processing ${event} event:`) + error(err.stack) } }, /** for testing only! */ cleanDispatcher: () => { - let event; + let event for (event in this.test) { - this.dispatcher.removeAllListeners(this.test[event]); + this.dispatcher.removeAllListeners(this.test[event]) } for (event in this.suite) { - this.dispatcher.removeAllListeners(this.test[event]); + this.dispatcher.removeAllListeners(this.test[event]) } for (event in this.step) { - this.dispatcher.removeAllListeners(this.test[event]); + this.dispatcher.removeAllListeners(this.test[event]) } for (event in this.all) { - this.dispatcher.removeAllListeners(this.test[event]); + this.dispatcher.removeAllListeners(this.test[event]) } }, -}; +} diff --git a/lib/listener/globalTimeout.js b/lib/listener/globalTimeout.js index e43258b21..31ea70617 100644 --- a/lib/listener/globalTimeout.js +++ b/lib/listener/globalTimeout.js @@ -130,7 +130,7 @@ module.exports = function () { if (typeof timeout === 'number' && timeout <= 0 && recorder.isRunning()) { debug(`step ${step.toCode().trim()} timed out`) - if (currentTest && currentTest.callback && currentTest.type == 'test') { + if (currentTest && currentTest.callback) { debug(`Failing test ${currentTest.title} with timeout ${currentTimeout}s`) recorder.reset() // replace mocha timeout with custom timeout diff --git a/lib/listener/result.js b/lib/listener/result.js index 7df7d771a..07ca1045c 100644 --- a/lib/listener/result.js +++ b/lib/listener/result.js @@ -6,7 +6,7 @@ module.exports = function () { container.result().addStats({ failedHooks: 1 }) }) - event.dispatcher.on(event.test.started, test => { + event.dispatcher.on(event.test.before, test => { container.result().addTest(test) }) } diff --git a/lib/mocha/asyncWrapper.js b/lib/mocha/asyncWrapper.js index 398ce2260..560776ed6 100644 --- a/lib/mocha/asyncWrapper.js +++ b/lib/mocha/asyncWrapper.js @@ -13,12 +13,19 @@ const injectHook = function (inject, suite) { recorder.throw(err) } recorder.catch(err => { - event.emit(event.test.failed, suite, err) + suiteTestFailedHookError(suite, err) throw err }) return recorder.promise() } +function suiteTestFailedHookError(suite, err) { + suite.eachTest(test => { + test.err = err + event.emit(event.test.failed, test, err) + }) +} + function makeDoneCallableOnce(done) { let called = false return function (err) { @@ -61,6 +68,7 @@ module.exports.test = test => { err = newErr } } + test.err = err event.emit(event.test.failed, test, err) event.emit(event.test.finished, test) recorder.add(() => doneFn(err)) @@ -112,7 +120,7 @@ module.exports.injected = function (fn, suite, hookName) { const errHandler = err => { recorder.session.start('teardown') recorder.cleanAsyncErr() - if (hookName == 'before' || hookName == 'beforeSuite') suite.eachTest(test => event.emit(event.test.failed, test, err)) + if (hookName == 'before' || hookName == 'beforeSuite') suiteTestFailedHookError(suite, err) if (hookName === 'after') event.emit(event.test.after, suite) if (hookName === 'afterSuite') event.emit(event.suite.after, suite) recorder.add(() => doneFn(err)) @@ -156,6 +164,7 @@ module.exports.injected = function (fn, suite, hookName) { ) .then(() => { recorder.add('fire hook.passed', () => fireHook(event.hook.passed, suite)) + recorder.add('fire hook.finished', () => fireHook(event.hook.finished, suite)) recorder.add(`finish ${hookName} hook`, doneFn) recorder.catch() }) @@ -166,6 +175,7 @@ module.exports.injected = function (fn, suite, hookName) { errHandler(err) }) recorder.add('fire hook.failed', () => fireHook(event.hook.failed, suite, e)) + recorder.add('fire hook.finished', () => fireHook(event.hook.finished, suite)) }) } } diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index 0690539f2..3ebbadbac 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -230,7 +230,7 @@ class Cli extends Base { stack.shift() } - if (stack[0].trim() == 'Error:') { + if (stack[0] && stack[0].trim() == 'Error:') { stack.shift() } diff --git a/lib/mocha/hooks.js b/lib/mocha/hooks.js index 8ac00a7fb..0dedc0adf 100644 --- a/lib/mocha/hooks.js +++ b/lib/mocha/hooks.js @@ -2,13 +2,30 @@ const event = require('../event') const { serializeError } = require('../utils') // const { serializeTest } = require('./test') +/** + * Represents a test hook in the testing framework + * @class + * @property {Object} suite - The test suite this hook belongs to + * @property {Object} test - The test object associated with this hook + * @property {Object} runnable - The current test being executed + * @property {Object} ctx - The context object + * @property {Error|null} err - The error that occurred during hook execution, if any + */ class Hook { + /** + * Creates a new Hook instance + * @param {Object} context - The context object containing suite and test information + * @param {Object} context.suite - The test suite + * @param {Object} context.test - The test object + * @param {Object} context.ctx - The context object + * @param {Error} error - The error object if hook execution failed + */ constructor(context, error) { this.suite = context.suite this.test = context.test this.runnable = context?.ctx?.test this.ctx = context.ctx - this.error = error + this.err = error } get hookName() { @@ -21,7 +38,7 @@ class Hook { title: this.title, // test: this.test ? serializeTest(this.test) : null, // suite: this.suite ? serializeSuite(this.suite) : null, - error: this.error ? serializeError(this.error) : null, + error: this.err ? serializeError(this.err) : null, } } @@ -58,13 +75,13 @@ function fireHook(eventType, suite, error) { const hook = suite.ctx?.test?.title?.match(/"([^"]*)"/)[1] switch (hook) { case 'before each': - event.emit(eventType, new BeforeHook(suite)) + event.emit(eventType, new BeforeHook(suite, error)) break case 'after each': event.emit(eventType, new AfterHook(suite, error)) break case 'before all': - event.emit(eventType, new BeforeSuiteHook(suite)) + event.emit(eventType, new BeforeSuiteHook(suite, error)) break case 'after all': event.emit(eventType, new AfterSuiteHook(suite, error)) diff --git a/lib/mocha/test.js b/lib/mocha/test.js index 045428be5..abcd5b0c7 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -46,7 +46,7 @@ function enhanceMochaTest(test) { test.addToSuite = function (suite) { enhanceMochaSuite(suite) suite.addTest(testWrapper(this)) - if (test.file) suite.file = relativeDir(test.file) + if (test.file && !suite.file) suite.file = test.file test.tags = [...(test.tags || []), ...(suite.tags || [])] test.fullTitle = () => `${suite.title}: ${test.title}` test.uid = genTestId(test) @@ -78,7 +78,7 @@ function deserializeTest(test) { return test } -function serializeTest(test, err = null) { +function serializeTest(test, error = null) { // test = { ...test } if (test.start && !test.duration) { @@ -86,12 +86,14 @@ function serializeTest(test, err = null) { test.duration = end - test.start } + let err + if (test.err) { err = serializeError(test.err) - test.status = 'failed' - } else if (err) { - err = serializeError(err) - test.status = 'failed' + test.state = 'failed' + } else if (error) { + err = serializeError(error) + test.state = 'failed' } const parent = {} if (test.parent) { @@ -105,6 +107,11 @@ function serializeTest(test, err = null) { }) } + let steps = undefined + if (Array.isArray(test.steps)) { + steps = test.steps.map(step => (step.simplify ? step.simplify() : step)) + } + return { opts: test.opts || {}, tags: test.tags || [], @@ -118,7 +125,7 @@ function serializeTest(test, err = null) { duration: test.duration || 0, err, parent, - steps: test.steps?.toArray()?.map(step => (step.simplify ? step.simplify() : step)), + steps, } } diff --git a/lib/mocha/types.d.ts b/lib/mocha/types.d.ts index 07daf22c0..9069f72dd 100644 --- a/lib/mocha/types.d.ts +++ b/lib/mocha/types.d.ts @@ -12,6 +12,8 @@ declare global { type: string text: string }> + state: string + err?: Error config: Record artifacts: string[] inject: Record diff --git a/lib/output.js b/lib/output.js index 1bd32ba03..919764635 100644 --- a/lib/output.js +++ b/lib/output.js @@ -135,7 +135,8 @@ module.exports = { */ started: suite => { if (!suite.title) return - print(`${colors.bold(suite.title)} --`, colors.underline.grey(suite.file || '')) + print(`${colors.bold(suite.title)} --`) + if (suite.file && outputLevel >= 1) print(colors.underline.grey(suite.file)) if (suite.comment) print(suite.comment) }, }, diff --git a/lib/plugin/analyze.js b/lib/plugin/analyze.js index 87bf1030a..7f79fa2bb 100644 --- a/lib/plugin/analyze.js +++ b/lib/plugin/analyze.js @@ -44,7 +44,7 @@ const defaultConfig = { I am test analyst analyzing failed tests in CodeceptJS testing framework. Please analyze the following failed tests and classify them into groups by their cause. - If there is no common cause, say: "No common cause found". + If there is no groups detected, say: "No common groups found". Provide a short description of the group and a list of failed tests that belong to this group. Use percent sign to indicate the percentage of failed tests in the group if this percentage is greater than 30%. @@ -70,19 +70,20 @@ const defaultConfig = { _______________________________ - --- GROUP # <(percentage of failed tests)'> - CATEGORY - ERRORS , , ... - SUMMARY - STEPS (in format I.click(), I.see(), etc) - AFFECTED TESTS () , , ... - SUITE , (if all tests in the group have the same suite or suites) - TAG (if all tags in group have the same tag) + ## Group <(percentage of failed tests in group compare to all failed tests)'> + + * CATEGORY + * ERROR , , ... + * SUMMARY + * STEP (in format I.click(), I.see(), etc; if all failures happend on the same step) + * AFFECTED TESTS () + * SUITE , (only if all tests in the group have the same suite or suites) + * TAG (if all tags in group have the same tag) `, }, { role: 'assistant', - content: `--- GROUP' + content: `## ' `, }, ] @@ -126,9 +127,9 @@ const defaultConfig = { Response format: - CATEGORY - STEPS - SUMMARY + * CATEGORY + * STEPS + * SUMMARY Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS. ${config.vision ? 'Also a screenshot of the page is attached to the prompt.' : ''} @@ -148,6 +149,11 @@ const defaultConfig = { }) } + messages.push({ + role: 'assistant', + content: `## `, + }) + return messages }, }, @@ -177,6 +183,11 @@ module.exports = function (config = {}) { }) event.dispatcher.on(event.workers.result, async result => { + if (!result.hasFailed) { + console.log('Everything is fine, skipping AI analysis') + return + } + if (!ai.isEnabled) { console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.') return @@ -186,7 +197,7 @@ module.exports = function (config = {}) { }) async function printReport(result) { - const failedTestsAndErrors = result.tests.filter(t => t.state === 'failed' && t.err) + const failedTestsAndErrors = result.tests.filter(t => t.err) if (!failedTestsAndErrors.length) return @@ -322,22 +333,11 @@ function serializeTest(test) { } function formatResponse(response) { + if (!response.startsWith('##')) response = '## ' + response return response .split('\n') .map(line => line.trim()) .filter(line => !/^[A-Z\s]+$/.test(line)) - .map(line => { - if (line.startsWith('ANALYSIS REPORT')) return line.replace('ANALYSIS REPORT', colors.bold.white('ANALYSIS REPORT ')) - if (line.startsWith('GROUP')) return line.replace('GROUP', colors.bold.bgWhite('GROUP ')) - if (line.startsWith('STEPS')) return line.replace('STEPS', colors.bold.bgBlue('STEP ')) - if (line.startsWith('AFFECTED TESTS')) return line.replace('AFFECTED TESTS', colors.bold.bgWhite('AFFECTED TESTS ')) - if (line.startsWith('ERRORS')) return line.replace('ERRORS', colors.bold.bgRed('ERRORS ')) - if (line.startsWith('TAG')) return line.replace('TAG', colors.bold.bgGray('TAG ')) - if (line.startsWith('SUITE')) return line.replace('SUITE', colors.bold.bgGray('SUITE ')) - if (line.startsWith('SUMMARY')) return line.replace('SUMMARY', colors.bold.bgYellow('SUMMARY ')) - if (line.startsWith('CATEGORY')) return line.replace('CATEGORY', colors.bold.bgGreen('CATEGORY ')) - return line - }) .map(line => markdownToAnsi(line)) .join('\n') } diff --git a/lib/plugin/customReporter.js b/lib/plugin/customReporter.js index 9780636b8..6b2cc0c05 100644 --- a/lib/plugin/customReporter.js +++ b/lib/plugin/customReporter.js @@ -4,9 +4,9 @@ const event = require('../event') * Sample custom reporter for CodeceptJS. */ module.exports = function (config) { - event.dispatcher.on(event.hook.before, hook => { - if (config.onHookBefore) { - config.onHookBefore(hook) + event.dispatcher.on(event.hook.finished, hook => { + if (config.onHookFinished) { + config.onHookFinished(hook) } }) @@ -16,14 +16,32 @@ module.exports = function (config) { } }) - event.dispatcher.on(event.test.failed, test => { + event.dispatcher.on(event.test.failed, (test, err) => { if (config.onTestFailed) { - config.onTestFailed(test) + config.onTestFailed(test, err) + } + }) + + event.dispatcher.on(event.test.passed, test => { + if (config.onTestPassed) { + config.onTestPassed(test) + } + }) + + event.dispatcher.on(event.test.skipped, test => { + if (config.onTestSkipped) { + config.onTestSkipped(test) + } + }) + + event.dispatcher.on(event.test.finished, test => { + if (config.onTestFinished) { + config.onTestFinished(test) } }) event.dispatcher.on(event.all.result, result => { - if (config.onAllResult) { + if (config.onResult) { config.onResult(result) } diff --git a/lib/plugin/tryTo.js b/lib/plugin/tryTo.js index 2c6fa13a0..2eb77245c 100644 --- a/lib/plugin/tryTo.js +++ b/lib/plugin/tryTo.js @@ -18,7 +18,7 @@ For more details, refer to the documentation. `) if (config.registerGlobal) { - global[config.registerGlobal] = tryTo + global.tryTo = tryTo } return tryTo diff --git a/lib/rerun.js b/lib/rerun.js index df4549209..a4f700ffe 100644 --- a/lib/rerun.js +++ b/lib/rerun.js @@ -1,81 +1,82 @@ -const fsPath = require('path'); -const container = require('./container'); -const event = require('./event'); -const BaseCodecept = require('./codecept'); -const output = require('./output'); +const fsPath = require('path') +const container = require('./container') +const event = require('./event') +const BaseCodecept = require('./codecept') +const output = require('./output') class CodeceptRerunner extends BaseCodecept { runOnce(test) { return new Promise((resolve, reject) => { // @ts-ignore - container.createMocha(); - const mocha = container.mocha(); - this.testFiles.forEach((file) => { - delete require.cache[file]; - }); - mocha.files = this.testFiles; + container.createMocha() + const mocha = container.mocha() + this.testFiles.forEach(file => { + delete require.cache[file] + }) + mocha.files = this.testFiles if (test) { if (!fsPath.isAbsolute(test)) { - test = fsPath.join(global.codecept_dir, test); + test = fsPath.join(global.codecept_dir, test) } - mocha.files = mocha.files.filter(t => fsPath.basename(t, '.js') === test || t === test); + mocha.files = mocha.files.filter(t => fsPath.basename(t, '.js') === test || t === test) } try { - mocha.run((failures) => { + mocha.run(failures => { if (failures === 0) { - resolve(); + resolve() } else { - reject(new Error(`${failures} tests fail`)); + reject(new Error(`${failures} tests fail`)) } - }); + }) } catch (e) { - reject(e); + reject(e) } - }); + }) } async runTests(test) { - const configRerun = this.config.rerun || {}; - const minSuccess = configRerun.minSuccess || 1; - const maxReruns = configRerun.maxReruns || 1; + const configRerun = this.config.rerun || {} + const minSuccess = configRerun.minSuccess || 1 + const maxReruns = configRerun.maxReruns || 1 if (minSuccess > maxReruns) { - process.exitCode = 1; - throw new Error(`run-rerun Configuration Error: minSuccess must be less than maxReruns. Current values: minSuccess=${minSuccess} maxReruns=${maxReruns}`); + process.exitCode = 1 + throw new Error(`run-rerun Configuration Error: minSuccess must be less than maxReruns. Current values: minSuccess=${minSuccess} maxReruns=${maxReruns}`) } if (maxReruns === 1) { - await this.runOnce(test); - return; + await this.runOnce(test) + return } - let successCounter = 0; - let rerunsCounter = 0; + let successCounter = 0 + let rerunsCounter = 0 while (rerunsCounter < maxReruns && successCounter < minSuccess) { - rerunsCounter++; + container.result().reset() // reset result + rerunsCounter++ try { - await this.runOnce(test); - successCounter++; - output.success(`\nProcess run ${rerunsCounter} of max ${maxReruns}, success runs ${successCounter}/${minSuccess}\n`); + await this.runOnce(test) + successCounter++ + output.success(`\nProcess run ${rerunsCounter} of max ${maxReruns}, success runs ${successCounter}/${minSuccess}\n`) } catch (e) { - output.error(`\nFail run ${rerunsCounter} of max ${maxReruns}, success runs ${successCounter}/${minSuccess} \n`); - console.error(e); + output.error(`\nFail run ${rerunsCounter} of max ${maxReruns}, success runs ${successCounter}/${minSuccess} \n`) + console.error(e) } } if (successCounter < minSuccess) { - throw new Error(`Flaky tests detected! ${successCounter} success runs achieved instead of ${minSuccess} success runs expected`); + throw new Error(`Flaky tests detected! ${successCounter} success runs achieved instead of ${minSuccess} success runs expected`) } } async run(test) { - event.emit(event.all.before, this); + event.emit(event.all.before, this) try { - await this.runTests(test); + await this.runTests(test) } catch (e) { - output.error(e.stack); - throw e; + output.error(e.stack) + throw e } finally { - event.emit(event.all.result, this); - event.emit(event.all.after, this); + event.emit(event.all.result, this) + event.emit(event.all.after, this) } } } -module.exports = CodeceptRerunner; +module.exports = CodeceptRerunner diff --git a/lib/result.js b/lib/result.js index 7824f2bc8..9e562d8fc 100644 --- a/lib/result.js +++ b/lib/result.js @@ -20,6 +20,14 @@ class Result { * Create Result of the test run */ constructor() { + this._startTime = new Date() + this._endTime = null + + this.reset() + this.start() + } + + reset() { this._stats = { passes: 0, failures: 0, @@ -31,16 +39,11 @@ class Result { duration: 0, } - this._startTime = new Date() - this._endTime = null - /** @type {CodeceptJS.Test[]} */ this._tests = [] /** @type {String[]} */ this._failures = [] - - this.start() } start() { @@ -52,7 +55,7 @@ class Result { } get hasFailed() { - return this.tests.some(test => test.state === 'failed') + return this._stats.failures > 0 } get tests() { @@ -96,7 +99,7 @@ class Result { } get hasFailures() { - return this._failures.length > 0 + return this.stats.failures > 0 } get duration() { @@ -132,7 +135,7 @@ class Result { */ save(fileName) { if (!fileName) fileName = 'result.json' - fs.writeFileSync(path.join(codeceptjs.outputDir, fileName), JSON.stringify(this.simplify(), null, 2)) + fs.writeFileSync(path.join(global.output_dir, fileName), JSON.stringify(this.simplify(), null, 2)) } /** diff --git a/lib/utils.js b/lib/utils.js index 25a81634f..3696678d9 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -563,6 +563,10 @@ module.exports.markdownToAnsi = function (markdown) { .replace(/^(#{1,6})\s+(.+)$/gm, (_, hashes, text) => { return chalk.bold.blue(`${hashes} ${text}`) }) + // Bullet points - replace with yellow bullet character + .replace(/^[-*]\s+(.+)$/gm, (_, text) => { + return `${chalk.yellow('•')} ${text}` + }) // Bold (**text**) - make bold .replace(/\*\*(.+?)\*\*/g, (_, text) => { return chalk.bold(text) diff --git a/test/data/sandbox/configs/custom-reporter-plugin/codecept.conf.js b/test/data/sandbox/configs/custom-reporter-plugin/codecept.conf.js new file mode 100644 index 000000000..536db958b --- /dev/null +++ b/test/data/sandbox/configs/custom-reporter-plugin/codecept.conf.js @@ -0,0 +1,44 @@ +exports.config = { + tests: './*_test.js', + output: './output', + helpers: { + FileSystem: {}, + }, + include: {}, + bootstrap: false, + plugins: { + customReporter: { + enabled: true, + onHookFinished: hook => { + console.log(`Hook Finished: ${hook.title}`) + }, + onTestBefore: test => { + console.log(`Test Started: ${test.title}`) + }, + onTestPassed: test => { + console.log(`Test Passed: ${test.title}`) + }, + onTestFailed: (test, err) => { + console.log(`Test Failed: ${test.title}`) + console.log(`Error: ${err.message}`) + }, + onTestSkipped: test => { + console.log(`Test Skipped: ${test.title}`) + }, + onTestFinished: test => { + console.log(`Test Finished: ${test.title}`) + console.log(`Test Status: ${test.state}`) + console.log(`Test Error: ${test.err}`) + }, + onResult: result => { + console.log('All tests completed') + console.log(`Total: ${result.stats.tests}`) + console.log(`Passed: ${result.stats.passes}`) + console.log(`Failed: ${result.stats.failures}`) + }, + save: true, + }, + }, + mocha: {}, + name: 'custom-reporter-plugin tests', +} diff --git a/test/data/sandbox/configs/custom-reporter-plugin/custom-reporter-plugin_test.js b/test/data/sandbox/configs/custom-reporter-plugin/custom-reporter-plugin_test.js new file mode 100644 index 000000000..902c9786a --- /dev/null +++ b/test/data/sandbox/configs/custom-reporter-plugin/custom-reporter-plugin_test.js @@ -0,0 +1,22 @@ +Feature('custom-reporter-plugin') + +BeforeSuite(({ I }) => { + I.say('I print before suite hook') +}) + +Before(({ I }) => { + I.say('I print before hook') +}) + +Scenario('test custom-reporter-plugin', ({ I }) => { + I.amInPath('.') + I.seeFile('this-file-should-not-exist.txt') +}) + +After(({ I }) => { + I.say('I print after hook') +}) + +AfterSuite(({ I }) => { + I.say('I print after suite hook') +}) diff --git a/test/runner/custom-reporter-plugin_test.js b/test/runner/custom-reporter-plugin_test.js new file mode 100644 index 000000000..af475c1b3 --- /dev/null +++ b/test/runner/custom-reporter-plugin_test.js @@ -0,0 +1,41 @@ +const { expect } = require('expect') +const exec = require('child_process').exec +const { codecept_dir, codecept_run } = require('./consts') +const debug = require('debug')('codeceptjs:tests') +const fs = require('fs') +const path = require('path') + +const config_run_config = (config, grep, verbose = false) => `${codecept_run} ${verbose ? '--verbose' : ''} --config ${codecept_dir}/configs/custom-reporter-plugin/${config} ${grep ? `--grep "${grep}"` : ''}` + +describe('CodeceptJS custom-reporter-plugin', function () { + this.timeout(10000) + + it('should run custom-reporter-plugin test', done => { + exec(config_run_config('codecept.conf.js'), (err, stdout) => { + debug(stdout) + + // Check for custom reporter output messages + expect(stdout).toContain('Hook Finished:') + expect(stdout).toContain('Test Started:') + expect(stdout).toContain('Test Failed:') + expect(stdout).toContain('Test Finished:') + expect(stdout).toContain('All tests completed') + expect(stdout).toContain('Total:') + expect(stdout).toContain('Passed:') + + // Check if result file exists and has content + const resultFile = path.join(`${codecept_dir}/configs/custom-reporter-plugin`, 'output', 'result.json') + expect(fs.existsSync(resultFile)).toBe(true) + + const resultContent = JSON.parse(fs.readFileSync(resultFile, 'utf8')) + expect(resultContent).toBeTruthy() + expect(resultContent).toHaveProperty('stats') + expect(resultContent.stats).toHaveProperty('tests') + expect(resultContent.stats).toHaveProperty('passes') + expect(resultContent.stats).toHaveProperty('failures') + + expect(err).toBeTruthy() + done() + }) + }) +}) diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index ba06d9e54..e8490fc1f 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -113,7 +113,7 @@ describe('CodeceptJS Workers Runner', function () { expect(stdout).toContain('FAILURES') expect(stdout).toContain('Workers Failing') // Only 1 test is executed - Before hook in Workers Failing - expect(stdout).toContain('✖ Workers Failing') + expect(stdout).toContain('✖ should not be executed') expect(stdout).toContain('FAIL | 0 passed, 1 failed') expect(err.code).toEqual(1) done() diff --git a/test/unit/plugin/retryFailedStep_test.js b/test/unit/plugin/retryFailedStep_test.js index 2a13e2212..e8012868e 100644 --- a/test/unit/plugin/retryFailedStep_test.js +++ b/test/unit/plugin/retryFailedStep_test.js @@ -46,31 +46,6 @@ describe('retryFailedStep', () => { return recorder.promise() }) - it('should not retry failed step when tryTo plugin is enabled', async () => { - tryTo() - retryFailedStep({ retries: 2, minTimeout: 1 }) - event.dispatcher.emit(event.test.before, {}) - event.dispatcher.emit(event.step.started, { name: 'click' }) - - try { - let counter = 0 - await recorder.add( - () => { - counter++ - if (counter < 3) { - throw new Error('Retry failed step is disabled when tryTo plugin is enabled') - } - }, - undefined, - undefined, - true, - ) - return recorder.promise() - } catch (e) { - expect(e.message).equal('Retry failed step is disabled when tryTo plugin is enabled') - } - }) - it('should not retry within', async () => { retryFailedStep({ retries: 1, minTimeout: 1 }) event.dispatcher.emit(event.test.before, {}) From 29e1be115dc4dcff2ed3e0ecb421b5be3061bf9b Mon Sep 17 00:00:00 2001 From: DavertMik Date: Thu, 23 Jan 2025 09:14:03 +0200 Subject: [PATCH 11/18] improved prompt for analyze plugin --- lib/plugin/analyze.js | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/plugin/analyze.js b/lib/plugin/analyze.js index 7f79fa2bb..e92e38a3a 100644 --- a/lib/plugin/analyze.js +++ b/lib/plugin/analyze.js @@ -32,10 +32,10 @@ const defaultConfig = { .map((test, index) => { if (!test || !test.err) return return ` - TEST #${index + 1}: ${serializeTest(test)} - ERROR: ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / tests.length)}` + #${index + 1}: ${serializeTest(test)} + ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / tests.length)}`.trim() }) - .join('\n\n---\n\n') + .join('\n\n--------\n\n') const messages = [ { @@ -60,25 +60,29 @@ const defaultConfig = { If there is no groups of tests, say: "No patterns found" Preserve error messages but cut them if they are too long. Respond clearly and directly, without introductory words or phrases like ‘Of course,’ ‘Here is the answer,’ etc. - Do not list more than 3 tests in the group. Do not list more than 3 errors in the group. If you identify that all tests in the group have the same tag, add this tag to the group report, otherwise ignore TAG section. If you identify that all tests in the group have the same suite, add this suite to the group report, otherwise ignore SUITE section. Pick different emojis for each group. + Do not include group into report if it has only one test in affected tests section. Provide list of groups in following format: _______________________________ - ## Group <(percentage of failed tests in group compare to all failed tests)'> + ## Group * CATEGORY * ERROR , , ... * SUMMARY - * STEP (in format I.click(), I.see(), etc; if all failures happend on the same step) - * AFFECTED TESTS () - * SUITE , (only if all tests in the group have the same suite or suites) - * TAG (if all tags in group have the same tag) + * STEP (use CodeceptJS format I.click(), I.see(), etc; if all failures happend on the same step) + * SUITE , (if SUITE is present, and if all tests in the group have the same suite or suites) + * TAG (if TAG is present, and if all tests in the group have the same tag) + * AFFECTED TESTS (): + x + x + x + x ... `, }, { @@ -318,9 +322,12 @@ function serializeTest(test) { if (test.suite) { testMessage += '\n SUITE: ' + test.suite.title } + if (test.parent) { + testMessage += '\n SUITE: ' + test.parent.title + } if (test.steps?.length) { - const failedSteps = test.steps.filter(s => s.status === 'failed') + const failedSteps = test.steps if (failedSteps.length) testMessage += '\n STEP: ' + failedSteps.map(s => s.toCode()).join('; ') } @@ -339,5 +346,6 @@ function formatResponse(response) { .map(line => line.trim()) .filter(line => !/^[A-Z\s]+$/.test(line)) .map(line => markdownToAnsi(line)) + .map(line => line.replace(/^x /gm, ` ${colors.red.bold('x')} `)) .join('\n') } From a0ae921f18a540592dc30b56046b9663acc89e4e Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 24 Jan 2025 04:19:55 +0200 Subject: [PATCH 12/18] refactored timeouts to fix tests --- lib/actor.js | 3 +-- lib/listener/globalTimeout.js | 26 +++++++++++++++++--------- lib/plugin/stepTimeout.js | 2 +- lib/recorder.js | 4 ++-- lib/step/base.js | 2 +- lib/step/record.js | 2 +- lib/{step => }/timeout.js | 16 ++++++++++++++++ 7 files changed, 39 insertions(+), 16 deletions(-) rename lib/{step => }/timeout.js (82%) diff --git a/lib/actor.js b/lib/actor.js index ff3a54050..081dae75e 100644 --- a/lib/actor.js +++ b/lib/actor.js @@ -3,8 +3,7 @@ const MetaStep = require('./step/meta') const recordStep = require('./step/record') const container = require('./container') const { methodsOfObject } = require('./utils') -const { TIMEOUT_ORDER } = require('./step/timeout') -const recorder = require('./recorder') +const { TIMEOUT_ORDER } = require('./timeout') const event = require('./event') const store = require('./store') const output = require('./output') diff --git a/lib/listener/globalTimeout.js b/lib/listener/globalTimeout.js index 31ea70617..3cfc4de9d 100644 --- a/lib/listener/globalTimeout.js +++ b/lib/listener/globalTimeout.js @@ -4,7 +4,7 @@ const recorder = require('../recorder') const Config = require('../config') const store = require('../store') const debug = require('debug')('codeceptjs:timeout') -const { TIMEOUT_ORDER } = require('../step/timeout') +const { TIMEOUT_ORDER, TimeoutError, TestTimeoutError } = require('../timeout') const { BeforeSuiteHook, AfterSuiteHook } = require('../mocha/hooks') module.exports = function () { @@ -119,25 +119,33 @@ module.exports = function () { } }) + event.dispatcher.on(event.step.after, step => { + if (typeof timeout !== 'number') return + if (!store.timeouts) return + + recorder.catchWithoutStop(err => { + if (timeout && err instanceof TimeoutError && +Date.now() - step.startTime >= timeout) { + debug('Step failed due to global test or suite timeout') + throw new TestTimeoutError(currentTimeout) + } + throw err + }) + }) + event.dispatcher.on(event.step.finished, step => { if (!store.timeouts) { debug('step', step.toCode().trim(), 'timeout disabled') return } + if (typeof timeout === 'number') debug('Timeout', timeout) + debug(`step ${step.toCode().trim()}:${step.status} duration`, step.duration) if (typeof timeout === 'number' && !Number.isNaN(timeout)) timeout -= step.duration if (typeof timeout === 'number' && timeout <= 0 && recorder.isRunning()) { debug(`step ${step.toCode().trim()} timed out`) - if (currentTest && currentTest.callback) { - debug(`Failing test ${currentTest.title} with timeout ${currentTimeout}s`) - recorder.reset() - // replace mocha timeout with custom timeout - currentTest.timeout(0.1) - currentTest.callback(new Error(`Timeout ${currentTimeout}s exceeded (with Before hook)`)) - currentTest.timedOut = true - } + recorder.throw(new TestTimeoutError(currentTimeout)) } }) } diff --git a/lib/plugin/stepTimeout.js b/lib/plugin/stepTimeout.js index 36f06d5c1..d512c6b0a 100644 --- a/lib/plugin/stepTimeout.js +++ b/lib/plugin/stepTimeout.js @@ -1,5 +1,5 @@ const event = require('../event') -const { TIMEOUT_ORDER } = require('../step/timeout') +const { TIMEOUT_ORDER } = require('../timeout') const defaultConfig = { timeout: 150, diff --git a/lib/recorder.js b/lib/recorder.js index 2c3a3e328..97bbd9cb8 100644 --- a/lib/recorder.js +++ b/lib/recorder.js @@ -3,7 +3,7 @@ const promiseRetry = require('promise-retry') const chalk = require('chalk') const { printObjectProperties } = require('./utils') const { log } = require('./output') - +const { TimeoutError } = require('./timeout') const MAX_TASKS = 100 let promise @@ -386,7 +386,7 @@ function getTimeoutPromise(timeoutMs, taskName) { return [ new Promise((done, reject) => { timer = setTimeout(() => { - reject(new Error(`Action ${taskName} was interrupted on step timeout ${timeoutMs}ms`)) + reject(new TimeoutError(`Action ${taskName} was interrupted on step timeout ${timeoutMs}ms`)) }, timeoutMs || 2e9) }), timer, diff --git a/lib/step/base.js b/lib/step/base.js index 1dce7b867..cf5825afe 100644 --- a/lib/step/base.js +++ b/lib/step/base.js @@ -1,6 +1,6 @@ const color = require('chalk') const Secret = require('../secret') -const { getCurrentTimeout } = require('./timeout') +const { getCurrentTimeout } = require('../timeout') const { ucfirst, humanizeString, serializeError } = require('../utils') const STACK_LINE = 5 diff --git a/lib/step/record.js b/lib/step/record.js index ed7a45699..c29908adf 100644 --- a/lib/step/record.js +++ b/lib/step/record.js @@ -3,7 +3,7 @@ const recorder = require('../recorder') const StepConfig = require('./config') const { debug } = require('../output') const store = require('../store') -const { TIMEOUT_ORDER } = require('./timeout') +const { TIMEOUT_ORDER } = require('../timeout') const retryStep = require('./retry') function recordStep(step, args) { step.status = 'queued' diff --git a/lib/step/timeout.js b/lib/timeout.js similarity index 82% rename from lib/step/timeout.js rename to lib/timeout.js index 876644d41..1260587ba 100644 --- a/lib/step/timeout.js +++ b/lib/timeout.js @@ -36,7 +36,23 @@ function getCurrentTimeout(timeouts) { return totalTimeout } +class TimeoutError extends Error { + constructor(message) { + super(message) + this.name = 'TimeoutError' + } +} + +class TestTimeoutError extends TimeoutError { + constructor(timeout) { + super(`Timeout ${timeout}s exceeded (with Before hook)`) + this.name = 'TestTimeoutError' + } +} + module.exports = { TIMEOUT_ORDER, getCurrentTimeout, + TimeoutError, + TestTimeoutError, } From 0d4b43299b90fb6d79f588c98e0fdcb0d5439fb2 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 24 Jan 2025 18:33:46 +0200 Subject: [PATCH 13/18] fixed saving screenshot file --- lib/mocha/test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/mocha/test.js b/lib/mocha/test.js index abcd5b0c7..a10472f6b 100644 --- a/lib/mocha/test.js +++ b/lib/mocha/test.js @@ -135,7 +135,8 @@ function cloneTest(test) { function testToFileName(test) { let fileName = clearString(test.title) - fileName = fileName.replace(/\@\w+/g, '') + // remove tags with empty string (disable for now) + // fileName = fileName.replace(/\@\w+/g, '') fileName = fileName.slice(0, 100) if (fileName.indexOf('{') !== -1) { fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim() From eb139fe1415897b6f67faad0fa23f6904b24a4dc Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 24 Jan 2025 18:37:06 +0200 Subject: [PATCH 14/18] fixed step sections --- lib/mocha/cli.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index 3ebbadbac..bf54ad346 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -101,9 +101,11 @@ class Cli extends Base { event.dispatcher.on(event.step.started, step => { let processingStep = step const metaSteps = [] + let isHidden = false while (processingStep.metaStep) { metaSteps.unshift(processingStep.metaStep) processingStep = processingStep.metaStep + if (processingStep.collapsed) isHidden = true } const shift = metaSteps.length @@ -117,6 +119,7 @@ class Cli extends Base { } } currentMetaStep = metaSteps + if (isHidden) return output.stepShift = 3 + 2 * shift output.step(step) }) From 6e5d01761d90f0e6819cd11200680d1f8e2ab4ee Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 24 Jan 2025 22:25:47 +0200 Subject: [PATCH 15/18] fixed tests for timeout errors --- lib/listener/globalTimeout.js | 14 +++++++++++--- lib/mocha/cli.js | 5 ++++- lib/recorder.js | 2 +- lib/timeout.js | 8 ++++++++ test/runner/timeout_test.js | 7 ++++--- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/listener/globalTimeout.js b/lib/listener/globalTimeout.js index 3cfc4de9d..07bed2807 100644 --- a/lib/listener/globalTimeout.js +++ b/lib/listener/globalTimeout.js @@ -4,7 +4,7 @@ const recorder = require('../recorder') const Config = require('../config') const store = require('../store') const debug = require('debug')('codeceptjs:timeout') -const { TIMEOUT_ORDER, TimeoutError, TestTimeoutError } = require('../timeout') +const { TIMEOUT_ORDER, TimeoutError, TestTimeoutError, StepTimeoutError } = require('../timeout') const { BeforeSuiteHook, AfterSuiteHook } = require('../mocha/hooks') module.exports = function () { @@ -124,9 +124,17 @@ module.exports = function () { if (!store.timeouts) return recorder.catchWithoutStop(err => { - if (timeout && err instanceof TimeoutError && +Date.now() - step.startTime >= timeout) { + // we wrap timeout errors in a StepTimeoutError + // but only if global timeout is set + // should we wrap all timeout errors? + if (err instanceof TimeoutError) { + const testTimeoutExceeded = timeout && +Date.now() - step.startTime >= timeout debug('Step failed due to global test or suite timeout') - throw new TestTimeoutError(currentTimeout) + if (testTimeoutExceeded) { + debug('Test failed due to global test or suite timeout') + throw new TestTimeoutError(currentTimeout) + } + throw new StepTimeoutError(currentTimeout, step) } throw err }) diff --git a/lib/mocha/cli.js b/lib/mocha/cli.js index bf54ad346..313bb8834 100644 --- a/lib/mocha/cli.js +++ b/lib/mocha/cli.js @@ -192,9 +192,12 @@ class Cli extends Base { if (lines.length > 5) { truncatedLines.push('...') } - err.message = '\n ' + truncatedLines.join('\n').replace(/^/gm, ' ').trim() + err.message = truncatedLines.join('\n').replace(/^/gm, ' ').trim() } + // add new line before the message + err.message = '\n ' + err.message + const steps = test.steps || (test.ctx && test.ctx.test.steps) if (steps && steps.length) { diff --git a/lib/recorder.js b/lib/recorder.js index 97bbd9cb8..5f7dbd59b 100644 --- a/lib/recorder.js +++ b/lib/recorder.js @@ -386,7 +386,7 @@ function getTimeoutPromise(timeoutMs, taskName) { return [ new Promise((done, reject) => { timer = setTimeout(() => { - reject(new TimeoutError(`Action ${taskName} was interrupted on step timeout ${timeoutMs}ms`)) + reject(new TimeoutError(`Action ${taskName} was interrupted on timeout ${timeoutMs}ms`)) }, timeoutMs || 2e9) }), timer, diff --git a/lib/timeout.js b/lib/timeout.js index 1260587ba..ba9ba43b8 100644 --- a/lib/timeout.js +++ b/lib/timeout.js @@ -50,9 +50,17 @@ class TestTimeoutError extends TimeoutError { } } +class StepTimeoutError extends TimeoutError { + constructor(timeout, step) { + super(`Step ${step.toCode().trim()} timed out after ${timeout}s`) + this.name = 'StepTimeoutError' + } +} + module.exports = { TIMEOUT_ORDER, getCurrentTimeout, TimeoutError, TestTimeoutError, + StepTimeoutError, } diff --git a/test/runner/timeout_test.js b/test/runner/timeout_test.js index a3c1ffefe..ef34048bd 100644 --- a/test/runner/timeout_test.js +++ b/test/runner/timeout_test.js @@ -33,7 +33,8 @@ describe('CodeceptJS Timeouts', function () { it('should ignore timeouts if no timeout', done => { exec(config_run_config('codecept.conf.js', 'no timeout test'), (err, stdout) => { debug_this_test && console.log(stdout) - expect(stdout).not.toContain('Timeout') + expect(stdout).not.toContain('TimeoutError') + expect(stdout).not.toContain('was interrupted on') expect(err).toBeFalsy() done() }) @@ -52,7 +53,7 @@ describe('CodeceptJS Timeouts', function () { it('should prefer step timeout', done => { exec(config_run_config('codecept.conf.js', 'timeout step', true), (err, stdout) => { debug_this_test && console.log(stdout) - expect(stdout).toContain('was interrupted on step timeout 200ms') + expect(stdout).toContain('was interrupted on timeout 200ms') expect(err).toBeTruthy() done() }) @@ -61,7 +62,7 @@ describe('CodeceptJS Timeouts', function () { it('should keep timeout with steps', done => { exec(config_run_config('codecept.timeout.conf.js', 'timeout step', true), (err, stdout) => { debug_this_test && console.log(stdout) - expect(stdout).toContain('was interrupted on step timeout 100ms') + expect(stdout).toContain('was interrupted on timeout 100ms') expect(err).toBeTruthy() done() }) From 481086792917c4e9d736a82dbcfc833547aa0a13 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 24 Jan 2025 22:59:57 +0200 Subject: [PATCH 16/18] added change fot pageInfo plugin --- lib/plugin/pageInfo.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/plugin/pageInfo.js b/lib/plugin/pageInfo.js index c04c8efcf..950b500aa 100644 --- a/lib/plugin/pageInfo.js +++ b/lib/plugin/pageInfo.js @@ -129,15 +129,15 @@ function pageStateToMarkdown(pageState) { } function getBrowserErrors(logs, type = ['error']) { - // Playwright console messages + // Playwright & WebDriver console messages let errors = logs - .filter(log => log.type) - .map(l => ({ type: l.type(), text: l.text() })) - .filter(l => type.includes(l.type)) - .map(l => l.text) - - // If console log coming from other helpers they may need other analysis - // TODO: Add other helpers analysis + .map(log => { + if (typeof log === 'string') return log + if (!log.type) return null + return { type: log.type(), text: log.text() } + }) + .filter(l => l && (typeof l === 'string' || type.includes(l.type))) + .map(l => (typeof l === 'string' ? l : l.text)) return errors } From a26679f1805dd34d650a7f2487cdf540ee4505dc Mon Sep 17 00:00:00 2001 From: DavertMik Date: Fri, 24 Jan 2025 23:42:24 +0200 Subject: [PATCH 17/18] fixed timeout tests --- test/runner/step-enhancements_test.js | 2 +- test/runner/step_timeout_test.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/runner/step-enhancements_test.js b/test/runner/step-enhancements_test.js index b988562e4..c68990f13 100644 --- a/test/runner/step-enhancements_test.js +++ b/test/runner/step-enhancements_test.js @@ -26,7 +26,7 @@ describe('CodeceptJS step-enhancements', function () { debug(stdout) expect(err).toBeTruthy() expect(stdout).not.toContain('OK') - expect(stdout).toContain('was interrupted on step timeout 100ms') + expect(stdout).toContain('was interrupted on timeout 100ms') done() }) }) diff --git a/test/runner/step_timeout_test.js b/test/runner/step_timeout_test.js index e055c7ffc..7cd216aca 100644 --- a/test/runner/step_timeout_test.js +++ b/test/runner/step_timeout_test.js @@ -11,7 +11,7 @@ describe('CodeceptJS Steps', function () { it('should stop test, when step timeout exceeded', done => { exec(config_run_config('codecept-1000.conf.js', 'Default command timeout'), (err, stdout) => { - expect(stdout).toContain('Action exceededByTimeout: 1500 was interrupted on step timeout 1000ms') + expect(stdout).toContain('Action exceededByTimeout: 1500 was interrupted on timeout 1000ms') expect(stdout).toContain('0 passed, 1 failed') expect(stdout).toContain(figures.cross + ' I.exceededByTimeout(1500)') expect(err).toBeTruthy() @@ -21,7 +21,7 @@ describe('CodeceptJS Steps', function () { it('should respect custom timeout with regex', done => { exec(config_run_config('codecept-1000.conf.js', 'Wait with longer timeout', debug_this_test), (err, stdout) => { - expect(stdout).not.toContain('was interrupted on step timeout') + expect(stdout).not.toContain('was interrupted on timeout') expect(stdout).toContain('1 passed') expect(err).toBeFalsy() done() @@ -30,7 +30,7 @@ describe('CodeceptJS Steps', function () { it('should respect custom timeout with full step name', done => { exec(config_run_config('codecept-1000.conf.js', 'Wait with shorter timeout', debug_this_test), (err, stdout) => { - expect(stdout).toContain('Action waitTadShorter: 750 was interrupted on step timeout 500ms') + expect(stdout).toContain('Action waitTadShorter: 750 was interrupted on timeout 500ms') expect(stdout).toContain('0 passed, 1 failed') expect(err).toBeTruthy() done() @@ -39,7 +39,7 @@ describe('CodeceptJS Steps', function () { it('should not stop test, when step not exceeded', done => { exec(config_run_config('codecept-2000.conf.js', 'Default command timeout'), (err, stdout) => { - expect(stdout).not.toContain('was interrupted on step timeout') + expect(stdout).not.toContain('was interrupted on timeout') expect(stdout).toContain('1 passed') expect(err).toBeFalsy() done() @@ -48,7 +48,7 @@ describe('CodeceptJS Steps', function () { it('should ignore timeout for steps with `wait*` prefix', done => { exec(config_run_config('codecept-1000.conf.js', 'Wait command timeout'), (err, stdout) => { - expect(stdout).not.toContain('was interrupted on step timeout') + expect(stdout).not.toContain('was interrupted on timeout') expect(stdout).toContain('1 passed') expect(err).toBeFalsy() done() @@ -57,7 +57,7 @@ describe('CodeceptJS Steps', function () { it('step timeout should work nicely with step retries', done => { exec(config_run_config('codecept-1000.conf.js', 'Rerun sleep'), (err, stdout) => { - expect(stdout).not.toContain('was interrupted on step timeout') + expect(stdout).not.toContain('was interrupted on timeout') expect(stdout).toContain('1 passed') expect(err).toBeFalsy() done() From bb4b1c924733581523a1068b185a6e329db94c11 Mon Sep 17 00:00:00 2001 From: DavertMik Date: Sat, 25 Jan 2025 00:14:15 +0200 Subject: [PATCH 18/18] fixed timeout tests with retries --- test/runner/timeout_test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/runner/timeout_test.js b/test/runner/timeout_test.js index ef34048bd..fc4fb3768 100644 --- a/test/runner/timeout_test.js +++ b/test/runner/timeout_test.js @@ -9,6 +9,9 @@ const config_run_config = (config, grep, verbose = false) => `${codecept_run} ${ describe('CodeceptJS Timeouts', function () { this.timeout(10000) + // some times messages are different + this.retries(2); + it('should stop test when timeout exceeded', done => { exec(config_run_config('codecept.conf.js', 'timed out'), (err, stdout) => { debug_this_test && console.log(stdout)