From 70c473e38572b63e151e0cb0b7bc1d29a9336c1d Mon Sep 17 00:00:00 2001 From: wswebcreation Date: Sat, 31 Jan 2026 07:11:05 +0100 Subject: [PATCH 1/3] fix: pass matcher threshold to core as saveAboveTolerance When using visual matchers with a threshold (e.g., toMatchScreenSnapshot('tag', 0.9)) and alwaysSaveActualImage: false, images were still saved even when the comparison passed within the threshold. The matcher now passes the expected threshold to the core as saveAboveTolerance, ensuring images are only saved when mismatch exceeds the user's acceptable threshold. Fixes #1111 Co-authored-by: Cursor --- .changeset/fix-matcher-threshold-save.md | 13 ++++ packages/visual-service/src/matcher.ts | 19 ++++++ packages/visual-service/tests/matcher.test.ts | 68 +++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 .changeset/fix-matcher-threshold-save.md diff --git a/.changeset/fix-matcher-threshold-save.md b/.changeset/fix-matcher-threshold-save.md new file mode 100644 index 00000000..02b5191f --- /dev/null +++ b/.changeset/fix-matcher-threshold-save.md @@ -0,0 +1,13 @@ +--- +"@wdio/visual-service": patch +--- + +# 🐛 Bugfixes + +## #1111 Pass matcher threshold to core as saveAboveTolerance + +When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold. + +The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected. + +This fix ensures the matcher passes the expected threshold to the core as `saveAboveTolerance`, so images are only saved when the mismatch actually exceeds the user's acceptable threshold. diff --git a/packages/visual-service/src/matcher.ts b/packages/visual-service/src/matcher.ts index abb7b173..92acf3b7 100644 --- a/packages/visual-service/src/matcher.ts +++ b/packages/visual-service/src/matcher.ts @@ -119,6 +119,25 @@ function parseMatcherParams ( */ options.returnAllCompareData = true + /** + * Pass the expected threshold to the core as `saveAboveTolerance` so it knows + * when to save actual images (only when mismatch exceeds the threshold). + * This ensures that when `alwaysSaveActualImage: false`, images are not saved + * if the comparison passes within the user's acceptable threshold. + * Only set if user hasn't explicitly set saveAboveTolerance. + * For numeric thresholds, use that value; otherwise default to 0 (same as comparison default). + * @see https://github.com/webdriverio/visual-testing/issues/1111 + */ + if (options.saveAboveTolerance === undefined) { + // Only set saveAboveTolerance for numeric thresholds (including undefined which defaults to 0) + // Asymmetric matchers can't be converted to a numeric tolerance + if (typeof expectedResult === 'number') { + options.saveAboveTolerance = expectedResult + } else if (expectedResult === undefined) { + options.saveAboveTolerance = DEFAULT_EXPECTED_RESULT + } + } + return { expectedResult, options } } diff --git a/packages/visual-service/tests/matcher.test.ts b/packages/visual-service/tests/matcher.test.ts index ec17f39a..7f9cde99 100644 --- a/packages/visual-service/tests/matcher.test.ts +++ b/packages/visual-service/tests/matcher.test.ts @@ -145,4 +145,72 @@ describe('custom visual matcher', () => { expect(results.pass).toBe(false) expect(results.message()).toMatchSnapshot() }) + + describe('saveAboveTolerance threshold passthrough (#1111)', () => { + it('should pass numeric expectedResult as saveAboveTolerance to checkScreen', async () => { + await toMatchScreenSnapshot(browser, 'foo', 0.9, {}) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0.9 + }) + }) + + it('should pass numeric expectedResult as saveAboveTolerance to checkFullPageScreen', async () => { + await toMatchFullPageSnapshot(browser, 'foo', 0.5, {}) + expect(browser.checkFullPageScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0.5 + }) + }) + + it('should pass numeric expectedResult as saveAboveTolerance to checkElement', async () => { + await toMatchElementSnapshot(browser as any as WebdriverIO.Element, 'foo', 1.5, {}) + expect(browser.checkElement).toHaveBeenCalledWith(browser, 'foo', { + returnAllCompareData: true, + saveAboveTolerance: 1.5 + }) + }) + + it('should pass numeric expectedResult as saveAboveTolerance to checkTabbablePage', async () => { + await toMatchTabbablePageSnapshot(browser, 'foo', 2.0, {}) + expect(browser.checkTabbablePage).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 2.0 + }) + }) + + it('should use default saveAboveTolerance of 0 when no threshold is provided', async () => { + await toMatchScreenSnapshot(browser, 'foo') + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0 + }) + }) + + it('should not override user-provided saveAboveTolerance', async () => { + await toMatchScreenSnapshot(browser, 'foo', 0.9, { saveAboveTolerance: 0.1 }) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true, + saveAboveTolerance: 0.1 // User's explicit value is preserved + }) + }) + + it('should not set saveAboveTolerance for asymmetric matchers', async () => { + await toMatchScreenSnapshot(browser, 'foo', expect.any(Number)) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + returnAllCompareData: true + // No saveAboveTolerance - can't convert asymmetric matcher to number + }) + }) + + it('should set saveAboveTolerance to 0 when options object is passed without threshold', async () => { + // When only options are passed (no expectedResult), threshold defaults to 0 + await toMatchScreenSnapshot(browser, 'foo', { hideScrollBars: true }) + expect(browser.checkScreen).toHaveBeenCalledWith('foo', { + hideScrollBars: true, + returnAllCompareData: true, + saveAboveTolerance: 0 + }) + }) + }) }) From c488c6aa7d4226a5b9d95162ef36ed0404acbc68 Mon Sep 17 00:00:00 2001 From: wswebcreation Date: Sat, 31 Jan 2026 07:25:13 +0100 Subject: [PATCH 2/3] chore: add extra UI test --- tests/specs/matcher.spec.ts | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/tests/specs/matcher.spec.ts b/tests/specs/matcher.spec.ts index d33a6659..b63aaa0e 100644 --- a/tests/specs/matcher.spec.ts +++ b/tests/specs/matcher.spec.ts @@ -1,4 +1,5 @@ -/// +import { readdirSync } from 'node:fs' +import { join } from 'node:path' import { browser, expect } from '@wdio/globals' describe('@wdio/visual-service matcher', () => { @@ -36,4 +37,36 @@ describe('@wdio/visual-service matcher', () => { ] }) }) + + it(`should NOT save actual image when mismatch is within threshold (#1111) for '${browserName}'`, async () => { + const tag = 'threshold-test-1111' + const actualFolder = join(process.cwd(), '.tmp/actual') + const subtitle = await $('.hero__subtitle') + const getActualImageCount = () => { + try { + return readdirSync(actualFolder).filter(f => f.includes(tag)).length + } catch { + return 0 + } + } + + // 1. Save the original subtitle as baseline + await browser.saveElement(subtitle, tag) + + // 2. Manipulate the subtitle to create a small text difference + await browser.execute( + 'arguments[0].innerHTML = "Test Demo Page";', + subtitle + ) + + const beforeCount = getActualImageCount() + + // 3. Run the matcher with a threshold (90%) higher than the expected mismatch + await expect(subtitle).toMatchElementSnapshot(tag, 90) + + const afterCount = getActualImageCount() + + // 4. With the fix: no new actual image should be saved when mismatch is within threshold + expect(afterCount).toBe(beforeCount) + }) }) From 95b3b79a410a03085d99c8459924d32d8e7fb6ac Mon Sep 17 00:00:00 2001 From: wswebcreation Date: Sat, 31 Jan 2026 07:26:25 +0100 Subject: [PATCH 3/3] chore: update changelog --- .changeset/fix-matcher-threshold-save.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.changeset/fix-matcher-threshold-save.md b/.changeset/fix-matcher-threshold-save.md index 02b5191f..bfbf5db3 100644 --- a/.changeset/fix-matcher-threshold-save.md +++ b/.changeset/fix-matcher-threshold-save.md @@ -11,3 +11,8 @@ When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `always The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected. This fix ensures the matcher passes the expected threshold to the core as `saveAboveTolerance`, so images are only saved when the mismatch actually exceeds the user's acceptable threshold. + + +# Committers: 1 + +- Wim Selles ([@wswebcreation](https://github.com/wswebcreation))