From 5baa8bdd7baba927871ed20859405e14312297b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Fri, 5 Jun 2026 13:42:47 +0200 Subject: [PATCH] fix: update handleCloudflareChallenge for new Cloudflare challenge markup (#3717) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What `handleCloudflareChallenge` stopped detecting the Cloudflare challenge. Cloudflare inserted a `.footer-wrapper` element between `.footer-inner` and `.diagnostic-wrapper` on the challenge page, which broke the direct-child detection selector — so the helper returned immediately and the request handler ran against an unsolved "Just a moment..." page (exactly the report in #3629). - Detection now matches by **descendants** (`.footer .footer-inner .diagnostic-wrapper .ray-id`) so intervening wrappers don't break it. - The click position now targets the **actual challenge widget** (the box holding the Turnstile checkbox input), falling back to the old `.main-content div` for older layouts. ## e2e alignment The `camoufox-cloudflare` e2e was also a false green: its own `isBlocked` check used the same broken selector, so the assertion always passed. Fixed that selector, and aligned the browser stack — `camoufox-js@^0.11.0` installs the Camoufox **150** (Firefox 150) build, so the actor now uses **Playwright 1.60** (also Firefox 150). A mismatched Firefox crashes playwright-core on Cloudflare's uncaught page errors (`pageError.location` is undefined), so the versions must line up. Depends on camoufox-js ≥ 0.11.0 (apify/camoufox-js#283). Closes #3629 --- package.json | 2 +- .../src/internals/utils/playwright-utils.ts | 10 ++++++++-- test/e2e/camoufox-cloudflare/actor/main.js | 2 +- .../camoufox-cloudflare/actor/package.json | 4 ++-- yarn.lock | 20 +++++++++---------- 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index 5473ea4197f4..240b2ec97710 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,7 @@ "apify-node-curl-impersonate": "^1.0.15", "basic-auth-parser": "^0.0.2", "body-parser": "^2.0.0", - "camoufox-js": "^0.10.0", + "camoufox-js": "^0.11.0", "commitlint": "^21.0.0", "cross-env": "^10.0.0", "deep-equal": "^2.0.5", diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts index 6bd4f6873cf2..6c6d34bf0b81 100644 --- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts +++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts @@ -753,7 +753,9 @@ async function handleCloudflareChallenge( options.isChallengeCallback ??= async () => { return await page.evaluate(async () => { - return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id'); + // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants + // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between). + return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id'); }); }; @@ -782,7 +784,11 @@ async function handleCloudflareChallenge( const bb = await page .evaluate(() => { - const div = document.querySelector('.main-content div'); + // Prefer the actual challenge widget (the box holding the Turnstile checkbox input); + // fall back to the first content div for older challenge layouts. + const div = + document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ?? + document.querySelector('.main-content div'); return div?.getBoundingClientRect(); }) .catch(() => undefined); diff --git a/test/e2e/camoufox-cloudflare/actor/main.js b/test/e2e/camoufox-cloudflare/actor/main.js index f26bb84d257b..b0756d73c293 100644 --- a/test/e2e/camoufox-cloudflare/actor/main.js +++ b/test/e2e/camoufox-cloudflare/actor/main.js @@ -28,7 +28,7 @@ await Actor.main(async () => { async requestHandler({ page, parseWithCheerio }) { const isBlocked = await page .evaluate(async () => { - return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id'); + return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id'); }) .catch(() => false); const $ = await parseWithCheerio(); diff --git a/test/e2e/camoufox-cloudflare/actor/package.json b/test/e2e/camoufox-cloudflare/actor/package.json index 9dc5dbde4852..67bd357d90e6 100644 --- a/test/e2e/camoufox-cloudflare/actor/package.json +++ b/test/e2e/camoufox-cloudflare/actor/package.json @@ -13,8 +13,8 @@ "@crawlee/playwright": "file:./packages/playwright-crawler", "@crawlee/types": "file:./packages/types", "@crawlee/utils": "file:./packages/utils", - "camoufox-js": "^0.9.3", - "playwright": "1.58.2" + "camoufox-js": "^0.11.0", + "playwright": "1.60.0" }, "overrides": { "apify": { diff --git a/yarn.lock b/yarn.lock index 36015348ac5a..fbf2d13a3c76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1141,7 +1141,7 @@ __metadata: apify-node-curl-impersonate: "npm:^1.0.15" basic-auth-parser: "npm:^0.0.2" body-parser: "npm:^2.0.0" - camoufox-js: "npm:^0.10.0" + camoufox-js: "npm:^0.11.0" commitlint: "npm:^21.0.0" cross-env: "npm:^10.0.0" deep-equal: "npm:^2.0.5" @@ -4401,14 +4401,14 @@ __metadata: languageName: node linkType: hard -"better-sqlite3@npm:^12.2.0": - version: 12.4.1 - resolution: "better-sqlite3@npm:12.4.1" +"better-sqlite3@npm:^12.10.0": + version: 12.10.0 + resolution: "better-sqlite3@npm:12.10.0" dependencies: bindings: "npm:^1.5.0" node-gyp: "npm:latest" prebuild-install: "npm:^7.1.1" - checksum: 10c0/88773a75d996b4171e5690a38459b05dc814a792701b224bd9909ee084dc0b4c64aaffbdbcf4bbbc6d4e247faf19e91b2a56cf4175d746d3bd9ff14764eb05aa + checksum: 10c0/b269792c1e456201b16fb9abd1e3fa9284d7710929f991899cebbb61e56dc3703673be29dadcb2f4a2d02961c49145f53d7d7b13b7b47298c033ec9de524f34f languageName: node linkType: hard @@ -4764,12 +4764,12 @@ __metadata: languageName: node linkType: hard -"camoufox-js@npm:^0.10.0": - version: 0.10.2 - resolution: "camoufox-js@npm:0.10.2" +"camoufox-js@npm:^0.11.0": + version: 0.11.0 + resolution: "camoufox-js@npm:0.11.0" dependencies: adm-zip: "npm:^0.5.16" - better-sqlite3: "npm:^12.2.0" + better-sqlite3: "npm:^12.10.0" cli-progress: "npm:^3.12.0" commander: "npm:^14.0.0" fingerprint-generator: "npm:^2.1.66" @@ -4784,7 +4784,7 @@ __metadata: playwright-core: "*" bin: camoufox-js: dist/__main__.js - checksum: 10c0/f04cd3b46f507649d7d2c882ad5f9e1a3a4420480d5a196b40d26ca635d2d2193764625fa6518baa17a2283445a689194aa2139c67825f1326604870859f7890 + checksum: 10c0/ac17d937edd690133638568e34726fe5b60ce763d6252edf163e6326571de7263d6103facd5fb3a7ca6979d7339aad454c588eef945676e96a8083d06c884165 languageName: node linkType: hard