diff --git a/.changeset/fix-matcher-threshold-save.md b/.changeset/fix-matcher-threshold-save.md new file mode 100644 index 00000000..bfbf5db3 --- /dev/null +++ b/.changeset/fix-matcher-threshold-save.md @@ -0,0 +1,18 @@ +--- +"@wdio/visual-service": patch +--- + +# 🐛 Bugfixes + +## #1111 Pass matcher threshold to core as saveAboveTolerance + +When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold. + +The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected. + +This fix ensures the matcher passes the expected threshold to the core as `saveAboveTolerance`, so images are only saved when the mismatch actually exceeds the user's acceptable threshold. + + +# Committers: 1 + +- Wim Selles ([@wswebcreation](https://github.com/wswebcreation)) diff --git a/packages/visual-service/src/matcher.ts b/packages/visual-service/src/matcher.ts index abb7b173..92acf3b7 100644 --- a/packages/visual-service/src/matcher.ts +++ b/packages/visual-service/src/matcher.ts @@ -119,6 +119,25 @@ function parseMatcherParams ( */ options.returnAllCompareData = true + /** + * Pass the expected threshold to the core as `saveAboveTolerance` so it knows + * when to save actual images (only when mismatch exceeds the threshold). + * This ensures that when `alwaysSaveActualImage: false`, images are not saved + * if the comparison passes within the user's acceptable threshold. + * Only set if user hasn't explicitly set saveAboveTolerance. + * For numeric thresholds, use that value; otherwise default to 0 (same as comparison default). + * @see https://github.com/webdriverio/visual-testing/issues/1111 + */ + if (options.saveAboveTolerance === undefined) { + // Only set saveAboveTolerance for numeric thresholds (including undefined which defaults to 0) + // Asymmetric matchers can't be converted to a numeric tolerance + if (typeof expectedResult === 'number') { + options.saveAboveTolerance = expectedResult + } else if (expectedResult === undefined) { + options.saveAboveTolerance = DEFAULT_EXPECTED_RESULT + } + } + return { expectedResult, options } } diff --git a/packages/visual-service/tests/matcher.test.ts b/packages/visual-service/tests/matcher.test.ts index ec17f39a..7f9cde99 100644 --- a/packages/visual-service/tests/matcher.test.ts +++ b/packages/visual-service/tests/matcher.test.ts @@ -145,4 +145,72 @@ describe('custom visual matcher', () => { expect(results.pass).toBe(false) expect(results.message()).toMatchSnapshot() }) + + describe('saveAboveTolerance threshold passthrough (#1111)', () => { + it('should pass numeric expectedResult as saveAboveTolerance to checkScreen', async () => { + await toMatchScreenSnapshot(browser, 'foo', 0.9, {}) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0.9 + }) + }) + + it('should pass numeric expectedResult as saveAboveTolerance to checkFullPageScreen', async () => { + await toMatchFullPageSnapshot(browser, 'foo', 0.5, {}) + expect(browser.checkFullPageScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0.5 + }) + }) + + it('should pass numeric expectedResult as saveAboveTolerance to checkElement', async () => { + await toMatchElementSnapshot(browser as any as WebdriverIO.Element, 'foo', 1.5, {}) + expect(browser.checkElement).toHaveBeenCalledWith(browser, 'foo', { + returnAllCompareData: true, + saveAboveTolerance: 1.5 + }) + }) + + it('should pass numeric expectedResult as saveAboveTolerance to checkTabbablePage', async () => { + await toMatchTabbablePageSnapshot(browser, 'foo', 2.0, {}) + expect(browser.checkTabbablePage).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 2.0 + }) + }) + + it('should use default saveAboveTolerance of 0 when no threshold is provided', async () => { + await toMatchScreenSnapshot(browser, 'foo') + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0 + }) + }) + + it('should not override user-provided saveAboveTolerance', async () => { + await toMatchScreenSnapshot(browser, 'foo', 0.9, { saveAboveTolerance: 0.1 }) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0.1 // User's explicit value is preserved + }) + }) + + it('should not set saveAboveTolerance for asymmetric matchers', async () => { + await toMatchScreenSnapshot(browser, 'foo', expect.any(Number)) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true + // No saveAboveTolerance - can't convert asymmetric matcher to number + }) + }) + + it('should set saveAboveTolerance to 0 when options object is passed without threshold', async () => { + // When only options are passed (no expectedResult), threshold defaults to 0 + await toMatchScreenSnapshot(browser, 'foo', { hideScrollBars: true }) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + hideScrollBars: true, + returnAllCompareData: true, + saveAboveTolerance: 0 + }) + }) + }) }) diff --git a/tests/specs/matcher.spec.ts b/tests/specs/matcher.spec.ts index d33a6659..b63aaa0e 100644 --- a/tests/specs/matcher.spec.ts +++ b/tests/specs/matcher.spec.ts @@ -1,4 +1,5 @@ -/// +import { readdirSync } from 'node:fs' +import { join } from 'node:path' import { browser, expect } from '@wdio/globals' describe('@wdio/visual-service matcher', () => { @@ -36,4 +37,36 @@ describe('@wdio/visual-service matcher', () => { ] }) }) + + it(`should NOT save actual image when mismatch is within threshold (#1111) for '${browserName}'`, async () => { + const tag = 'threshold-test-1111' + const actualFolder = join(process.cwd(), '.tmp/actual') + const subtitle = await $('.hero__subtitle') + const getActualImageCount = () => { + try { + return readdirSync(actualFolder).filter(f => f.includes(tag)).length + } catch { + return 0 + } + } + + // 1. Save the original subtitle as baseline + await browser.saveElement(subtitle, tag) + + // 2. Manipulate the subtitle to create a small text difference + await browser.execute( + 'arguments[0].innerHTML = "Test Demo Page";', + subtitle + ) + + const beforeCount = getActualImageCount() + + // 3. Run the matcher with a threshold (90%) higher than the expected mismatch + await expect(subtitle).toMatchElementSnapshot(tag, 90) + + const afterCount = getActualImageCount() + + // 4. With the fix: no new actual image should be saved when mismatch is within threshold + expect(afterCount).toBe(beforeCount) + }) })