Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added
- **DuckDuckGo anomaly challenge detection.** Detect DuckDuckGo anomaly challenge walls (`//duckduckgo.com/anomaly.js`, `anomaly-modal`) in Tier 1 and the MITM proxy, escalating requests to the browser tiers rather than treating the challenge page as successful content (#119).

## [1.5.0] - 2026-09-04

### Changed
Expand Down
19 changes: 19 additions & 0 deletions apps/api/src/proxy/__tests__/directForward.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ const fetchFixture = (req: Request): Response => {
return new Response('<html><div id="sec-if-cpt-container" class="behavioral-content"></div></html>', {
headers: { "Content-Type": "text/html; charset=utf-8" },
})
if (pathname === "/duckduckgo-challenge")
return new Response(
'<form id="challenge-form" action="//duckduckgo.com/anomaly.js?sv=html"><div data-testid="anomaly-modal"></div></form>',
{ status: 202, headers: { "Content-Type": "text/html; charset=utf-8" } },
)
if (pathname === "/video")
return new Response(chunked(Buffer.from([0, 1, 2, 3]), Buffer.from([4, 5, 6, 7])), {
headers: { "Content-Type": "video/mp4" },
Expand Down Expand Up @@ -297,6 +302,20 @@ describe("directForwardHttp — buffered by default", () => {
expect(result.body.toString()).toContain("Just a moment")
})

test("detects a 202 DuckDuckGo anomaly challenge", async () => {
const result = await directForwardHttp({
url: `${baseUrl}/duckduckgo-challenge`,
method: "POST",
headers: {},
})

expect(result.mode).toBe("buffer")
if (result.mode !== "buffer") return
expect(result.status).toBe(202)
expect(result.challengeDetected).toBe(true)
expect(result.body.toString()).toContain("anomaly-modal")
})

test("detects a challenge in a compressed HTML response", async () => {
const result = await directForwardHttp({
url: `${baseUrl}/gzip-challenge`,
Expand Down
2 changes: 1 addition & 1 deletion apps/api/src/proxy/directForward.ts
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ async function readHttpResponse(

// Challenge detection on the buffered body. Bounded preview keeps this cheap.
const previewText = decodeForInspection(body.subarray(0, offset), headers["content-encoding"])
const challengeType = detectChallengeType(previewText, headers)
const challengeType = detectChallengeType(previewText, headers, status)
const challengeDetected = !skipChallengeDetection && isChallengeWall(status, body.length, challengeType)

return {
Expand Down
1 change: 1 addition & 0 deletions packages/tiers/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ export {
hasDataDomeCaptcha,
hasDataDomeChallenge,
hasDdosGuardChallenge,
hasDuckDuckGoChallenge,
hasHcaptcha,
hasImpervaChallenge,
hasRecaptcha,
Expand Down
15 changes: 15 additions & 0 deletions packages/tiers/src/tiers/1.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
hasAkamaiChallenge,
hasAwsWafCaptcha,
hasAwsWafChallenge,
hasDuckDuckGoChallenge,
hasHcaptcha,
hasRecaptcha,
hasTurnstile,
Expand Down Expand Up @@ -142,6 +143,20 @@ export async function runTier1(
}
}

if (hasDuckDuckGoChallenge(previewText, responseHeaders)) {
return {
tier: 1,
status: "needs-js",
durationMs: Date.now() - start,
reason: "duckduckgo-anomaly-challenge",
challenge: "duckduckgo",
responseHeaders,
contentType,
body: rawBytes,
statusCode: res.status,
}
}

// JS-only challenges: the page's static HTML is just a shell that loads the
// captcha widget via <script src="...api.js">. Plain fetch sees the shell and
// would otherwise report success — but the real content (including the widget)
Expand Down
9 changes: 9 additions & 0 deletions packages/tiers/src/tiers/3.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
hasAkamaiChallenge,
hasDataDomeChallenge,
hasDdosGuardChallenge,
hasDuckDuckGoChallenge,
hasImpervaChallenge,
isBlocked,
isBrowserErrorPage,
Expand All @@ -35,6 +36,7 @@ const DATACENTER_BLOCKED_REASONS: Partial<Record<ChallengeType, string>> = {
"aws-waf": "datacenter-ip-blocked (AWS WAF token obtained but challenge persisted — needs residential proxy)",
datadome:
"datadome-persistent (a datadome cookie was issued but the wall held — check BROWSER_HEADFUL_POOL_SIZE, then try a residential proxy)",
duckduckgo: "datacenter-ip-blocked (DuckDuckGo anomaly challenge persisted — needs residential proxy)",
}

const DEFAULT_DATACENTER_BLOCKED_REASON =
Expand Down Expand Up @@ -236,6 +238,13 @@ export async function runTier3(
}
}

if (hasDuckDuckGoChallenge(html)) {
const pageTitle = await page.title().catch(() => "?")
const pageUrl = page.url()
console.log(`[tier3] duckduckgo-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`)
return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: "duckduckgo-persistent" }
}

if (isBlocked(mainResponse.status, html)) {
return { tier: 3, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` }
}
Expand Down
13 changes: 13 additions & 0 deletions packages/tiers/src/tiers/4.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
hasAkamaiChallenge,
hasDataDomeChallenge,
hasDdosGuardChallenge,
hasDuckDuckGoChallenge,
hasImpervaChallenge,
isBlocked,
isBrowserErrorPage,
Expand Down Expand Up @@ -208,6 +209,18 @@ export async function runTier4(
}
}

if (hasDuckDuckGoChallenge(html)) {
const pageTitle = await page.title().catch(() => "?")
const pageUrl = page.url()
console.log(`[tier4] duckduckgo-persistent: url="${pageUrl}" title="${pageTitle}" html=${html.length}b`)
return {
tier: 4,
status: "blocked",
durationMs: Date.now() - start,
reason: "duckduckgo-persistent",
}
}

if (isBlocked(mainResponse.status, html)) {
return { tier: 4, status: "blocked", durationMs: Date.now() - start, reason: `http-${mainResponse.status}` }
}
Expand Down
3 changes: 2 additions & 1 deletion packages/tiers/src/utils/challengeWait.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// Navigates manually if cf_clearance is set but redirect hasn't fired after 5s.

import type { Frame, Page } from "patchright"
import { hasTurnstile, isCloudflarePage } from "./detect"
import { hasDuckDuckGoChallenge, hasTurnstile, isCloudflarePage } from "./detect"

export const CF_CHALLENGE_TITLE = /just a moment|verify you are human|please wait|one more step|attention required/i

Expand Down Expand Up @@ -42,6 +42,7 @@ export async function waitForChallengeResolution(
const active =
CF_CHALLENGE_TITLE.test(title) ||
isCloudflarePage(html, responseHeaders()) ||
hasDuckDuckGoChallenge(html) ||
/\/cdn-cgi\/challenge-platform|\/cdn-cgi\/challenge\//i.test(url) ||
hasChallengeFrame

Expand Down
27 changes: 24 additions & 3 deletions packages/tiers/src/utils/detect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export type ChallengeType =
| "ddos-guard"
| "aws-waf"
| "datadome"
| "duckduckgo"
| "none"

export function hasCloudflareChallengeHeader(headers: Record<string, string> = {}): boolean {
Expand All @@ -35,6 +36,7 @@ export function getAwsWafAction(
export function isCloudflarePage(html: string, headers: Record<string, string>): boolean {
if (hasCloudflareChallengeHeader(headers)) return true
if (hasDdosGuardChallenge(html)) return false
if (hasDuckDuckGoChallenge(html)) return false
if (/<title>[^<]*(just a moment|please wait|checking|attention required)[^<]*<\/title>/i.test(html)) return true
if (/checking your browser/i.test(html)) return true
if (/enable javascript and cookies to continue/i.test(html)) return true
Expand Down Expand Up @@ -139,6 +141,16 @@ export function hasDdosGuardChallenge(html: string, _headers: Record<string, str
return false
}

// DuckDuckGo anti-bot anomaly challenge markers.
// Ordinary DuckDuckGo search results do NOT contain anomaly.js or anomaly-modal elements.
export function hasDuckDuckGoChallenge(html: string, _headers: Record<string, string> = {}): boolean {
if (/action=["'][^"']*\/anomaly\.js/i.test(html)) return true
if (/src=["'][^"']*\/anomaly\.js/i.test(html)) return true
if (/data-testid=["']anomaly-modal["']/i.test(html)) return true
if (/class=["'][^"']*anomaly-modal/i.test(html) && /challenge-form/i.test(html)) return true
return false
}

// AWS WAF JavaScript challenge — the interstitial page that loads challenge.js to
// issue an aws-waf-token cookie before redirecting to the protected resource.
export function hasAwsWafChallenge(html: string, headers: Record<string, string> = {}, status?: number): boolean {
Expand Down Expand Up @@ -220,6 +232,7 @@ export function detectChallengeType(
if (hasDataDomeChallenge(html, headers, status)) return "datadome"
if (hasTurnstile(html)) return "cloudflare-turnstile"
if (hasDdosGuardChallenge(html, headers)) return "ddos-guard"
if (hasDuckDuckGoChallenge(html, headers)) return "duckduckgo"
if (isCloudflarePage(html, headers)) return "cloudflare-interstitial"
if (hasImpervaChallenge(html, headers)) return "imperva"
if (hasAkamaiChallenge(html, headers)) return "akamai"
Expand All @@ -236,6 +249,7 @@ export function isBlocked(status: number, html: string): boolean {
if (hasAkamaiChallenge(html)) return true
if (hasDdosGuardChallenge(html)) return true
if (hasDataDomeChallenge(html)) return true
if (hasDuckDuckGoChallenge(html)) return true
return false
}

Expand All @@ -245,7 +259,8 @@ export function needsJs(html: string, headers: Record<string, string>): boolean
hasImpervaChallenge(html, headers) ||
hasAkamaiChallenge(html, headers) ||
hasDdosGuardChallenge(html, headers) ||
hasDataDomeChallenge(html, headers)
hasDataDomeChallenge(html, headers) ||
hasDuckDuckGoChallenge(html, headers)
)
}

Expand All @@ -266,9 +281,15 @@ const LEAN_BODY_THRESHOLDS: Partial<Record<ChallengeType, number>> = {
export function isChallengeWall(status: number, bodyLength: number, challengeType: ChallengeType): boolean {
if (challengeType === "none") return false
if (status === 403 || status === 503) return true
// These three never serve real content alongside their wall, so the type alone settles
// These four never serve real content alongside their wall, so the type alone settles
// it. For datadome that leans on the header invariant documented in getDataDomeAction().
if (challengeType === "akamai" || challengeType === "aws-waf" || challengeType === "datadome") return true
if (
challengeType === "akamai" ||
challengeType === "aws-waf" ||
challengeType === "datadome" ||
challengeType === "duckduckgo"
)
return true
const threshold = LEAN_BODY_THRESHOLDS[challengeType]
if (threshold !== undefined && bodyLength < threshold) return true
return false
Expand Down
65 changes: 65 additions & 0 deletions packages/tiers/tests/duckduckgoDetection.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import { describe, expect, test } from "bun:test"
import { runTier1 } from "../src/tiers/1"
import {
detectChallengeType,
hasDuckDuckGoChallenge,
isBlocked,
isChallengeWall,
isCloudflarePage,
needsJs,
} from "../src/utils/detect"
import { DUCKDUCKGO_ANOMALY_CHALLENGE, DUCKDUCKGO_SEARCH_PAGE } from "./fixtures/duckduckgo"

async function withFetch(response: Response, run: () => Promise<void>) {
const original = globalThis.fetch
;(globalThis as { fetch: typeof fetch }).fetch = (async () => response) as typeof fetch
try {
await run()
} finally {
;(globalThis as { fetch: typeof fetch }).fetch = original
}
}

const htmlResponse = (body: string, status: number, headers: Record<string, string> = {}) =>
new Response(body, { status, headers: { "content-type": "text/html", ...headers } })

describe("DuckDuckGo anomaly challenge detection", () => {
test("classifies a real anomaly challenge independently from Cloudflare", () => {
expect(hasDuckDuckGoChallenge(DUCKDUCKGO_ANOMALY_CHALLENGE)).toBe(true)
expect(detectChallengeType(DUCKDUCKGO_ANOMALY_CHALLENGE)).toBe("duckduckgo")
expect(isCloudflarePage(DUCKDUCKGO_ANOMALY_CHALLENGE, {})).toBe(false)
expect(needsJs(DUCKDUCKGO_ANOMALY_CHALLENGE, {})).toBe(true)
expect(isBlocked(202, DUCKDUCKGO_ANOMALY_CHALLENGE)).toBe(true)
expect(isChallengeWall(202, DUCKDUCKGO_ANOMALY_CHALLENGE.length, "duckduckgo")).toBe(true)
expect(isChallengeWall(200, DUCKDUCKGO_ANOMALY_CHALLENGE.length, "duckduckgo")).toBe(true)
})

test("does not classify an ordinary DuckDuckGo search result page", () => {
expect(hasDuckDuckGoChallenge(DUCKDUCKGO_SEARCH_PAGE)).toBe(false)
expect(detectChallengeType(DUCKDUCKGO_SEARCH_PAGE)).toBe("none")
expect(isChallengeWall(200, DUCKDUCKGO_SEARCH_PAGE.length, "none")).toBe(false)
})

test("does not classify a bare provider mention or text search query mentioning anomaly", () => {
const html = "<p>DuckDuckGo anomaly detection system documentation</p>"
expect(hasDuckDuckGoChallenge(html)).toBe(false)
expect(detectChallengeType(html)).toBe("none")
})

test("lets authoritative Cloudflare headers win", () => {
const headers = { "CF-Mitigated": "Challenge" }
expect(detectChallengeType(DUCKDUCKGO_ANOMALY_CHALLENGE, headers)).toBe("cloudflare-interstitial")
expect(isCloudflarePage(DUCKDUCKGO_ANOMALY_CHALLENGE, headers)).toBe(true)
})

test("Tier 1 escalates DuckDuckGo anomaly challenge to needs-js", async () => {
await withFetch(htmlResponse(DUCKDUCKGO_ANOMALY_CHALLENGE, 202, { "x-test": "forwarded" }), async () => {
const result = await runTier1("https://html.duckduckgo.com/html/")
expect(result.status).toBe("needs-js")
expect(result.reason).toBe("duckduckgo-anomaly-challenge")
expect(result.challenge).toBe("duckduckgo")
expect(result.statusCode).toBe(202)
expect(result.responseHeaders?.["x-test"]).toBe("forwarded")
})
})
})
37 changes: 37 additions & 0 deletions packages/tiers/tests/fixtures/duckduckgo.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
export const DUCKDUCKGO_ANOMALY_CHALLENGE = `<!DOCTYPE html>
<html lang="en">
<head>
<title>DuckDuckGo</title>
</head>
<body>
<center id="lite_wrapper">
<iframe name="ifr" width="0" height="0" border="0" class="hidden"></iframe>
<form id="challenge-form" action="//duckduckgo.com/anomaly.js?sv=html&cc=sre&st=1788811733&gk=d4cd0dabcf4caa22ad92fab40844c786" method="POST">
<div class="anomaly-modal__mask">
<div class="anomaly-modal__modal is-ie" data-testid="anomaly-modal">
<div class="anomaly-modal__controls">
<button name="challenge-submit" class="btn btn--primary anomaly-modal__submit js-anomaly-modal-submit" form="challenge-form" value="d4cd0dabcf4caa22ad92fab40844c786">Submit</button>
</div>
</div>
</div>
</form>
</center>
</body>
</html>`

export const DUCKDUCKGO_SEARCH_PAGE = `<!DOCTYPE html>
<html lang="en">
<head>
<title>test at DuckDuckGo</title>
</head>
<body>
<div id="links" class="results">
<div class="result results_links results_links_deep highlight_result">
<a class="result__url" href="https://example.com">example.com</a>
<h2 class="result__title">
<a class="result__a" href="https://example.com">Example Domain</a>
</h2>
</div>
</div>
</body>
</html>`