fix: pass matcher threshold to core as saveAboveTolerance

.changeset/fix-matcher-threshold-save.md

@@ -0,0 +1,18 @@
+---
+"@wdio/visual-service": patch
+---
+
+# 🐛 Bugfixes
+
+## #1111 Pass matcher threshold to core as saveAboveTolerance
+
+When using visual matchers like `toMatchScreenSnapshot('tag', 0.9)` with `alwaysSaveActualImage: false`, the actual image was still being saved even when the comparison passed within the threshold.
+
+The root cause was that the matcher's expected threshold was not being passed to the core comparison logic. The core used `saveAboveTolerance` (defaulting to 0) to decide whether to save images, while the matcher used the user-provided threshold to determine pass/fail - these were disconnected.
+
+This fix ensures the matcher passes the expected threshold to the core as `saveAboveTolerance`, so images are only saved when the mismatch actually exceeds the user's acceptable threshold.
+
+
+# Committers: 1
+
+- Wim Selles ([@wswebcreation](https://github.com/wswebcreation))

packages/visual-service/src/matcher.ts

@@ -119,6 +119,25 @@ function parseMatcherParams (
      */
     options.returnAllCompareData = true
 
+    /**
+     * Pass the expected threshold to the core as `saveAboveTolerance` so it knows
+     * when to save actual images (only when mismatch exceeds the threshold).
+     * This ensures that when `alwaysSaveActualImage: false`, images are not saved
+     * if the comparison passes within the user's acceptable threshold.
+     * Only set if user hasn't explicitly set saveAboveTolerance.
+     * For numeric thresholds, use that value; otherwise default to 0 (same as comparison default).
+     * @see https://github.com/webdriverio/visual-testing/issues/1111
+     */
+    if (options.saveAboveTolerance === undefined) {
+        // Only set saveAboveTolerance for numeric thresholds (including undefined which defaults to 0)
+        // Asymmetric matchers can't be converted to a numeric tolerance
+        if (typeof expectedResult === 'number') {
+            options.saveAboveTolerance = expectedResult
+        } else if (expectedResult === undefined) {
+            options.saveAboveTolerance = DEFAULT_EXPECTED_RESULT
+        }
+    }
+
     return { expectedResult, options }
 }
 

packages/visual-service/tests/matcher.test.ts

@@ -145,4 +145,72 @@ describe('custom visual matcher', () => {
         expect(results.pass).toBe(false)
         expect(results.message()).toMatchSnapshot()
     })
+
+    describe('saveAboveTolerance threshold passthrough (#1111)', () => {
+        it('should pass numeric expectedResult as saveAboveTolerance to checkScreen', async () => {
+            await toMatchScreenSnapshot(browser, 'foo', 0.9, {})
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0.9
+            })
+        })
+
+        it('should pass numeric expectedResult as saveAboveTolerance to checkFullPageScreen', async () => {
+            await toMatchFullPageSnapshot(browser, 'foo', 0.5, {})
+            expect(browser.checkFullPageScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0.5
+            })
+        })
+
+        it('should pass numeric expectedResult as saveAboveTolerance to checkElement', async () => {
+            await toMatchElementSnapshot(browser as any as WebdriverIO.Element, 'foo', 1.5, {})
+            expect(browser.checkElement).toHaveBeenCalledWith(browser, 'foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 1.5
+            })
+        })
+
+        it('should pass numeric expectedResult as saveAboveTolerance to checkTabbablePage', async () => {
+            await toMatchTabbablePageSnapshot(browser, 'foo', 2.0, {})
+            expect(browser.checkTabbablePage).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 2.0
+            })
+        })
+
+        it('should use default saveAboveTolerance of 0 when no threshold is provided', async () => {
+            await toMatchScreenSnapshot(browser, 'foo')
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0
+            })
+        })
+
+        it('should not override user-provided saveAboveTolerance', async () => {
+            await toMatchScreenSnapshot(browser, 'foo', 0.9, { saveAboveTolerance: 0.1 })
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true,
+                saveAboveTolerance: 0.1  // User's explicit value is preserved
+            })
+        })
+
+        it('should not set saveAboveTolerance for asymmetric matchers', async () => {
+            await toMatchScreenSnapshot(browser, 'foo', expect.any(Number))
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                returnAllCompareData: true
+                // No saveAboveTolerance - can't convert asymmetric matcher to number
+            })
+        })
+
+        it('should set saveAboveTolerance to 0 when options object is passed without threshold', async () => {
+            // When only options are passed (no expectedResult), threshold defaults to 0
+            await toMatchScreenSnapshot(browser, 'foo', { hideScrollBars: true })
+            expect(browser.checkScreen).toHaveBeenCalledWith('foo', {
+                hideScrollBars: true,
+                returnAllCompareData: true,
+                saveAboveTolerance: 0
+            })
+        })
+    })
 })

tests/specs/matcher.spec.ts

@@ -1,4 +1,5 @@
-/// <reference types="@wdio/visual-service" />
+import { readdirSync } from 'node:fs'
+import { join } from 'node:path'
 import { browser, expect } from '@wdio/globals'
 
 describe('@wdio/visual-service matcher', () => {
@@ -36,4 +37,36 @@ describe('@wdio/visual-service matcher', () => {
             ]
         })
     })
+
+    it(`should NOT save actual image when mismatch is within threshold (#1111) for '${browserName}'`, async () => {
+        const tag = 'threshold-test-1111'
+        const actualFolder = join(process.cwd(), '.tmp/actual')
+        const subtitle = await $('.hero__subtitle')
+        const getActualImageCount = () => {
+            try {
+                return readdirSync(actualFolder).filter(f => f.includes(tag)).length
+            } catch {
+                return 0
+            }
+        }
+
+        // 1. Save the original subtitle as baseline
+        await browser.saveElement(subtitle, tag)
+
+        // 2. Manipulate the subtitle to create a small text difference
+        await browser.execute(
+            'arguments[0].innerHTML = "Test Demo Page";',
+            subtitle
+        )
+
+        const beforeCount = getActualImageCount()
+
+        // 3. Run the matcher with a threshold (90%) higher than the expected mismatch
+        await expect(subtitle).toMatchElementSnapshot(tag, 90)
+
+        const afterCount = getActualImageCount()
+
+        // 4. With the fix: no new actual image should be saved when mismatch is within threshold
+        expect(afterCount).toBe(beforeCount)
+    })
 })