diff --git a/README.md b/README.md index d5450b5..806902e 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,15 @@ Measured against real traffic shapes: serve retrieval allow ChatGPT-User 403 training block ClaudeBot from an unpublished IP 402 training charge ClaudeBot from a real Anthropic IP -serve search allow Googlebot +serve search allow Googlebot, PerplexityBot +serve preview allow Slackbot, facebookexternalhit ``` +`preview` is its own intent rather than a flavour of `retrieval`: a link unfurl +is a person pasting your URL into a conversation, not an assistant answering a +question. Folding the two together would inflate the retrieval number, which is +the one figure the split exists to measure. + Settlement is never ours. `mppxGateway` wraps Stripe's MPP SDK; `x402Gateway` calls a facilitator you supply. The library emits challenges and reads credentials — holding money would drag PCI scope into edge middleware. @@ -279,6 +285,26 @@ its `evidence`, a `risk` rating, and a `caveat` where over-blocking is plausible — the datacenter-ASN rule is marked `high` risk because corporate VPNs and privacy relays egress from hosting networks. +### Pools that no per-address threshold catches + +A rotating proxy pool is built so that no single address looks abusive. Measured +on one production site: 34 addresses across 11 countries, one user agent each, +the heaviest doing 100 requests in a day — every one invisible to a per-address +threshold, while collectively sweeping the site. + +Volume cannot separate that from real readers, so the burst rule keys on *rate*, +which needs `spanSeconds` on your observations: + +```ts +{ ip: '104.28.233.73', requests: 31, distinctPaths: 16, spanSeconds: 1 } +// → 1,860 requests/min. Not a person. +``` + +The rule is only safe because its condition excludes static assets. The WAF sees +every request; the middleware that produced your observations probably does not, +so a naive limit on `Mozilla` throttles a real visitor on their first page view. +Override `assetExclusions` if your app does not serve assets from `/_next/`. + See [`docs/TESTING-PAYMENTS.md`](./docs/TESTING-PAYMENTS.md) for testing the payment path end to end. diff --git a/package-lock.json b/package-lock.json index b36dd7f..8f9cac0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@apideck/agent-analytics", - "version": "0.16.0", + "version": "0.17.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@apideck/agent-analytics", - "version": "0.16.0", + "version": "0.17.0", "license": "MIT", "devDependencies": { "@types/node": "^20.14.0", diff --git a/package.json b/package.json index 7c55180..d9f156e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@apideck/agent-analytics", - "version": "0.16.0", + "version": "0.17.0", "description": "Track AI agent and bot traffic to your Next.js / Vercel app — PostHog, webhooks, or any custom analytics backend. Detects Claude, ChatGPT, Perplexity, Google-Extended, and more.", "keywords": [ "ai", diff --git a/size-budget.json b/size-budget.json index f29c36e..f100233 100644 --- a/size-budget.json +++ b/size-budget.json @@ -14,8 +14,8 @@ "note": "Opt-in payment surface." }, "dist/firewall.js": { - "gzipBudget": 3200, - "note": "Offline analysis tool; never runs in middleware." + "gzipBudget": 4300, + "note": "Offline analysis tool; never runs in middleware, so size buys correctness cheaply here. Raised from 3200 for the distributed-pool rule class and the link-unfurler bypass." }, "dist/markdown.js": { "gzipBudget": 1300, diff --git a/src/firewall.ts b/src/firewall.ts index 80eb1a8..2a70b90 100644 --- a/src/firewall.ts +++ b/src/firewall.ts @@ -88,6 +88,14 @@ export interface TrafficObservation { asn?: number /** Distinct paths this slice touched — a scraper sweeps, a reader does not. */ distinctPaths?: number + /** + * Seconds between this slice's first and last request. Combined with + * `requests` this gives a rate, which is the signal that separates a scraper + * from a reader when volume alone does not: on one production site the + * heaviest single address managed only 100 requests a day — far under any + * sane abuse threshold — but fetched 16 distinct pages in one second. + */ + spanSeconds?: number /** Verification verdict, if you ran one. */ verification?: 'verified' | 'spoofed' | 'unverifiable' | 'not-claimed' country?: string @@ -103,8 +111,32 @@ export interface RecommendOptions { trainingBudget?: { window: number; requests: number } /** Skip the protective bypass rule. Rarely a good idea. */ omitProtectiveBypass?: boolean + /** + * Burst budget for the distributed-scraper rule, in *page* requests. Defaults + * to 30 per 60s, which no human reaches once static assets are excluded. + */ + burstBudget?: { window: number; requests: number } + /** + * How many distinct low-volume addresses must look alike before they are + * treated as one coordinated pool. Defaults to 5. + */ + minPoolSize?: number + /** + * Path prefixes and extensions the burst rule must not count, because the WAF + * sees every asset request while your middleware probably does not. Defaults + * to Next.js internals and the usual static extensions. + */ + assetExclusions?: { prefixes: readonly string[]; extensions: readonly string[] } } +const DEFAULT_ASSET_EXCLUSIONS = { + prefixes: ['/_next/'], + extensions: [ + 'js', 'css', 'png', 'jpg', 'jpeg', 'gif', 'svg', 'webp', 'ico', + 'woff', 'woff2', 'ttf', 'otf', 'map' + ] +} as const + /* -------------------------------------------------------------------------- */ function shellQuote(json: unknown): string { @@ -194,6 +226,7 @@ export function recommendFirewallRules( 'Claude-User', 'Claude-SearchBot', 'Perplexity-User', + 'PerplexityBot', 'Googlebot', 'bingbot', 'DuckDuckBot', @@ -211,6 +244,50 @@ export function recommendFirewallRules( ) } + /* 1b. Link unfurlers, protected separately from the agents above. ---------- + Kept out of the rule above deliberately. Those tokens belong to vendors who + publish IP ranges and increasingly sign their requests; `facebookexternalhit` + and friends do neither and are among the most-forged strings on the web. A + `bypass` skips every managed ruleset too, so folding them into a rule + labelled `low` risk would understate what it hands out. Same protection, + honest label, and an operator can decline this one on its own. */ + const preview = observations.filter((o) => o.intent === 'preview') + if (!opts.omitProtectiveBypass && preview.length) { + const requests = preview.reduce((n, o) => n + o.requests, 0) + const names = [...new Set(preview.map((o) => o.botName))] + out.push( + finish({ + name: 'Allow link unfurlers', + rationale: + 'Someone pasted your URL into a conversation and the platform fetched it to render a card. Blocking it makes your links look broken wherever they are shared.', + evidence: `${requests.toLocaleString('en-US')} requests across ${names.length} platform${names.length === 1 ? '' : 's'} (${names.slice(0, 6).join(', ')})`, + groups: [ + [ + { + type: 'user_agent', + op: 'inc', + value: [ + 'facebookexternalhit', + 'Twitterbot', + 'LinkedInBot', + 'Slackbot', + 'Discordbot', + 'TelegramBot', + 'WhatsApp', + 'redditbot' + ] + } + ] + ], + action: 'bypass', + eventual: 'bypass', + risk: 'medium', + caveat: + 'These user agents are trivially spoofed and none of these platforms publish verifiable IP ranges, so this rule hands a bypass to anyone who sets the header. Scope it to your public content paths and never to anything authenticated or expensive.' + }) + ) + } + /* 2. Failed verification — the only class that earns a denial. ------------- */ const spoofed = observations.filter((o) => o.verification === 'spoofed') const spoofedIps = [...new Set(spoofed.map((o) => o.ip).filter((v): v is string => !!v))] @@ -264,6 +341,78 @@ export function recommendFirewallRules( ) } + /* 3b. Many small addresses behaving as one. -------------------------------- + Rule 3 asks "is any single address abusive?" and a rotating proxy pool is + built precisely so the answer is no. Observed on one production site: 34 + addresses across 11 countries, one user agent each, none above 100 requests + a day — every one of them invisible to a per-address threshold, while + collectively sweeping the site and inflating its analytics. + + Volume cannot separate that from real readers, so this keys on rate. It is + safe to do so only because the condition excludes static assets: a person + loading one page fires dozens of asset requests that the WAF counts and + your middleware does not, and a naive limit on `Mozilla` would throttle + real users on their first page view. */ + const pool = observations.filter( + (o) => + o.ip !== undefined && + !heavy.some((h) => h.ip === o.ip) && + o.verification !== 'verified' && + (o.intent === 'unknown' || o.intent === 'tooling') && + /Mozilla|Chrome|Safari/i.test(o.userAgent) + ) + const minPool = opts.minPoolSize ?? 5 + const poolIps = [...new Set(pool.map((o) => o.ip!))] + if (poolIps.length >= minPool) { + const burst = opts.burstBudget ?? { window: 60, requests: 30 } + const requests = pool.reduce((n, o) => n + o.requests, 0) + const countries = [...new Set(pool.map((o) => o.country).filter(Boolean))] + + // Only slices that actually carry timing can be shown to exceed the budget. + // Say how many were measurable rather than implying the rest were clean. + const timed = pool.filter((o) => o.spanSeconds !== undefined && o.spanSeconds > 0) + const overBudget = timed.filter( + (o) => o.requests / o.spanSeconds! > burst.requests / burst.window + ) + const peak = timed.length + ? Math.max(...timed.map((o) => (o.requests / o.spanSeconds!) * 60)) + : 0 + + const assets = opts.assetExclusions ?? DEFAULT_ASSET_EXCLUSIONS + const conditions: FirewallCondition[] = [ + { type: 'user_agent', op: 'sub', value: 'Mozilla' }, + ...assets.prefixes.map( + (p): FirewallCondition => ({ type: 'path', op: 'pre', value: p, neg: true }) + ), + { + type: 'path', + op: 're', + value: `\\.(?:${assets.extensions.join('|')})$`, + neg: true + } + ] + + out.push( + finish({ + name: 'Burst limit page navigations', + rationale: + 'A pool of addresses each too small to trip a per-address threshold, together behaving like one scraper. Rate is the only signal that separates them from real readers.', + evidence: + `${poolIps.length} addresses across ${countries.length} countries, ${requests.toLocaleString('en-US')} requests, none individually above the abuse threshold` + + (timed.length + ? `; ${overBudget.length} of ${timed.length} measurable slices exceeded ${burst.requests} requests/${burst.window}s, peaking at ${Math.round(peak).toLocaleString('en-US')}/min` + : '; no timing supplied, so the burst budget is a default rather than a measurement'), + groups: [conditions], + action: 'log', + eventual: 'rate_limit', + rateLimit: { window: burst.window, requests: burst.requests, action: 'rate_limit', keys: ['ip'] }, + risk: 'medium', + caveat: + 'The asset exclusions are what make this safe — the WAF counts every request, including the dozens of assets behind a single page view, so verify they match how your app actually serves static files before enforcing. Vercel counters are per region, so N regions can collectively exceed the limit by ~Nx.' + }) + ) + } + /* 4. Training crawlers: bound the cost, do not disappear from corpora. ----- */ const training = observations.filter((o) => o.intent === 'training') if (training.length) { @@ -357,11 +506,16 @@ export function firewallScript(recommendations: readonly FirewallRecommendation[ lines.push(r.cli) lines.push('') }) + const protective = recommendations.filter((r) => r.action === 'bypass') + if (protective.length) { + lines.push('# Keep the protective allow rules at the top of the evaluation order.') + // Reversed: each --first pushes to the top, so applying them back-to-front + // leaves the array's own order intact once all of them have run. + for (const r of [...protective].reverse()) { + lines.push(`vercel firewall rules reorder ${JSON.stringify(r.name)} --first --yes`) + } + } if (recommendations.length) { - lines.push('# Keep the protective allow rule at the top of the evaluation order.') - lines.push( - `vercel firewall rules reorder ${JSON.stringify(recommendations[0]!.name)} --first --yes` - ) lines.push('') lines.push('vercel firewall diff') lines.push('echo "Review above, then: vercel firewall publish --yes"') diff --git a/src/policy.ts b/src/policy.ts index 9d95e7b..733b84c 100644 --- a/src/policy.ts +++ b/src/policy.ts @@ -21,12 +21,16 @@ export type AgentAction = 'allow' | 'meter' | 'charge' | 'block' * and charging for it is charging for your own marketing. * - `'training'` — bulk corpus collection for model training. You get nothing * back per fetch, which is where a price makes sense. - * - `'search'` — classic index crawlers. Blocking these costs you SEO. + * - `'search'` — index crawlers, traditional and AI-native. Blocking these + * costs you organic traffic or citations in an assistant's answer. + * - `'preview'` — link unfurlers. Someone pasted your URL into Slack, iMessage + * or a tweet and the platform fetched it to render a card. No model involved, + * but blocking it means your links look broken wherever they get shared. * - `'tooling'` — coding agents and HTTP clients. Usually developers using * your docs; treat like retrieval unless you see abuse. * - `'unknown'` — everything else, including real browsers. */ -export type AgentIntent = 'retrieval' | 'training' | 'search' | 'tooling' | 'unknown' +export type AgentIntent = 'retrieval' | 'training' | 'search' | 'preview' | 'tooling' | 'unknown' /** * User agents where a human is waiting on the answer. Deliberately explicit @@ -39,8 +43,33 @@ const RETRIEVAL = /ChatGPT-User|OAI-SearchBot|Claude-User|Claude-SearchBot|Perpl /** Bulk crawlers that collect corpora. No human is waiting on these. */ const TRAINING = /GPTBot|ClaudeBot|Claude-Web|CCBot|Bytespider|Amazonbot|Amzn-SearchBot|Meta-ExternalAgent|meta-externalfetcher|meta-webindexer|FacebookBot|Google-Extended|Applebot-Extended|AI2Bot|Diffbot|omgili|Webzio-Extended|Timpibot|PanguBot|cohere|DeepSeek|Grok|quillbot|MyCentralAIScraperBot|NovaAct|AzureAI-SearchBot|Google-CloudVertexBot/i -/** Classic search indexers — blocking these costs you organic traffic. */ -const SEARCH = /bingbot|Googlebot|DuckDuckBot|YandexBot|Baiduspider|PetalBot|Sogou|Applebot(?!-Extended)/i +/** + * Index crawlers — blocking these costs you organic traffic. + * + * `PerplexityBot` sits here rather than in TRAINING despite the `Bot` suffix: + * Perplexity documents it as the crawler behind their *search results* and + * states it does not feed foundation-model training. Blocking it costs you + * citations, which is the same shape of loss as blocking Googlebot. It was + * previously in no list at all, so it classified as `unknown` and fell through + * both the protective bypass and the training rate limit — a live gap found in + * production traffic, not in review. + */ +const SEARCH = /bingbot|Googlebot|DuckDuckBot|YandexBot|Baiduspider|PetalBot|Sogou|PerplexityBot|Bravebot|Applebot(?!-Extended)/i + +/** + * Link unfurlers. A human shared the URL and a platform fetched it to build a + * preview card — one request, no crawl, and the payoff is a rendered link in a + * conversation. They get their own intent rather than being folded into + * `retrieval` because retrieval is the library's demand signal: counting + * Slackbot as "an assistant went to read this for someone" would inflate the + * one number the split exists to measure. + * + * These tokens are trivially spoofable — `facebookexternalhit` is among the + * most-forged strings on the web. Treat this as a routing hint, never as + * identity, and note that {@link recommendFirewallRules} proposes them as a + * separate, higher-risk rule for exactly that reason. + */ +const PREVIEW = /facebookexternalhit|Twitterbot|LinkedInBot|Slackbot|Discordbot|TelegramBot|WhatsApp|redditbot|Pinterest|SkypeUriPreview|Iframely|Embedly|vkShare|Mastodon|Bluesky/i export interface AgentDecision { action: AgentAction @@ -81,6 +110,11 @@ export interface AgentPolicyOptions { onRetrieval?: AgentAction /** What to do with search indexers. Defaults to `'allow'`. */ onSearch?: AgentAction + /** + * What to do with link unfurlers. Defaults to `'allow'` — gating these does + * not earn you anything, it just makes your links render as bare URLs. + */ + onPreview?: AgentAction /** What to do with coding agents and HTTP clients. Defaults to `'allow'`. */ onTooling?: AgentAction /** Vendor labels or UA substrings always allowed, whatever the intent. */ @@ -105,6 +139,9 @@ export function agentIntent(userAgent: string | null | undefined): AgentIntent { if (RETRIEVAL.test(ua)) return 'retrieval' if (TRAINING.test(ua)) return 'training' if (SEARCH.test(ua)) return 'search' + // After SEARCH so Applebot stays a search crawler rather than an iMessage + // unfurler — Apple uses the same token for both. + if (PREVIEW.test(ua)) return 'preview' // An HTTP-library UA that matched no vendor is a coding agent or a script. if (isHttpClient(ua)) return 'tooling' return 'unknown' @@ -165,14 +202,17 @@ export function agentPolicy(req: Request, opts: AgentPolicyOptions = {}): AgentD ? (opts.onRetrieval ?? 'allow') : intent === 'search' ? (opts.onSearch ?? 'allow') - : intent === 'tooling' - ? (opts.onTooling ?? 'allow') - : 'allow' + : intent === 'preview' + ? (opts.onPreview ?? 'allow') + : intent === 'tooling' + ? (opts.onTooling ?? 'allow') + : 'allow' const REASONS: Record = { retrieval: 'a person is waiting on this answer', training: 'bulk corpus collection', search: 'search index crawler', + preview: 'link unfurler building a preview card', tooling: 'coding agent or HTTP client', unknown: 'not a recognised agent' } diff --git a/test/firewall.test.ts b/test/firewall.test.ts index 38bc96b..6bc4069 100644 --- a/test/firewall.test.ts +++ b/test/firewall.test.ts @@ -134,6 +134,129 @@ describe('recommendFirewallRules', () => { }) }) +describe('distributed pools', () => { + // Modelled on real traffic: 34 addresses, 11 countries, one UA each, the + // heaviest doing 100 requests a day. Every one of them sits far below any + // per-address threshold, which is the entire design of a rotating proxy pool. + const POOL: TrafficObservation[] = [ + { userAgent: 'Mozilla/5.0 (Macintosh) AppleWebKit/605.1.15 Version/26.5 Safari/605.1.15', botName: 'Headless', intent: 'unknown', requests: 100, distinctPaths: 76, ip: '172.225.240.217', country: 'Germany', spanSeconds: 577 }, + { userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/150.0.0.0 Safari/537.36', botName: 'Headless', intent: 'unknown', requests: 71, distinctPaths: 40, ip: '93.156.192.71', country: 'Spain', spanSeconds: 139 }, + { userAgent: 'Mozilla/5.0 (Macintosh) Chrome/150.0.0.0 Safari/537.36', botName: 'Headless', intent: 'unknown', requests: 68, distinctPaths: 50, ip: '148.252.147.43', country: 'United Kingdom', spanSeconds: 50 }, + { userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/151.0.0.0 Safari/537.36', botName: 'Headless', intent: 'unknown', requests: 34, distinctPaths: 28, ip: '41.239.255.216', country: 'Egypt', spanSeconds: 329 }, + { userAgent: 'Mozilla/5.0 (X11; Linux x86_64) Chrome/150.0.0.0 Safari/537.36', botName: 'Headless', intent: 'unknown', requests: 31, distinctPaths: 16, ip: '104.28.233.73', country: 'United States', spanSeconds: 1 }, + { userAgent: 'Mozilla/5.0 (Macintosh) Chrome/150.0.0.0 Safari/537.36', botName: 'Headless', intent: 'unknown', requests: 13, distinctPaths: 12, ip: '109.206.198.23', country: 'Poland', spanSeconds: 11 } + ] + + const burst = () => + recommendFirewallRules(POOL).find((r) => r.name === 'Burst limit page navigations') + + it('catches a pool that no per-address threshold would', () => { + // Sanity-check the premise first: if any of these tripped rule 3 on its + // own, this rule would be redundant and the test would prove nothing. + const perAddress = recommendFirewallRules(POOL).find((r) => r.name.match(/high-volume/)) + expect(perAddress).toBeUndefined() + expect(burst()).toBeDefined() + }) + + it('excludes static assets, which is the only reason it is safe', () => { + // The WAF sees every asset request; the middleware that produced these + // observations does not. Without these exclusions a 30/60s limit throttles + // a real person on their first page view. + const conditions = burst()!.groups[0]! + expect(conditions.some((c) => c.type === 'path' && c.op === 'pre' && c.neg === true)).toBe(true) + const ext = conditions.find((c) => c.type === 'path' && c.op === 're')! + expect(ext.neg).toBe(true) + expect(String(ext.value)).toMatch(/css/) + expect(String(ext.value)).toMatch(/woff2/) + }) + + it('reports the peak rate it actually measured', () => { + // 31 requests in 1 second — the slice that made the case. + expect(burst()!.evidence).toMatch(/1,860\/min/) + expect(burst()!.evidence).toMatch(/6 addresses across 6 countries/) + }) + + it('says so when it had no timing rather than implying it measured one', () => { + const untimed = POOL.map(({ spanSeconds: _drop, ...o }) => o) + expect( + recommendFirewallRules(untimed).find((r) => r.name === 'Burst limit page navigations')!.evidence + ).toMatch(/no timing supplied/) + }) + + it('does not re-cover addresses the per-address rule already caught', () => { + const heavy: TrafficObservation[] = [ + ...POOL, + { userAgent: 'Mozilla/5.0 (X11; Linux x86_64) Chrome/150 Safari/537.36', botName: 'Headless', intent: 'unknown', requests: 90_000, distinctPaths: 40_000, ip: '164.92.65.128', country: 'United States' } + ] + const rules = recommendFirewallRules(heavy) + const perAddress = rules.find((r) => r.name.match(/high-volume/))! + expect(perAddress.groups[0]![0]!.value).toContain('164.92.65.128') + // The burst rule still fires for the pool, but the heavy address is not + // double-counted into its evidence. + expect(rules.find((r) => r.name === 'Burst limit page navigations')!.evidence).toMatch( + /6 addresses/ + ) + }) + + it('stays quiet below the pool size', () => { + expect( + recommendFirewallRules(POOL.slice(0, 3)).some((r) => r.name === 'Burst limit page navigations') + ).toBe(false) + expect( + recommendFirewallRules(POOL.slice(0, 3), { minPoolSize: 3 }).some( + (r) => r.name === 'Burst limit page navigations' + ) + ).toBe(true) + }) + + it('leaves verified crawlers out of the pool', () => { + const verified = POOL.map((o) => ({ ...o, verification: 'verified' as const })) + expect( + recommendFirewallRules(verified).some((r) => r.name === 'Burst limit page navigations') + ).toBe(false) + }) +}) + +describe('link unfurlers', () => { + const WITH_PREVIEW: TrafficObservation[] = [ + ...OBS, + { userAgent: 'Slackbot-LinkExpanding 1.0', botName: 'Slack', intent: 'preview', requests: 420 }, + { userAgent: 'facebookexternalhit/1.1', botName: 'Facebook', intent: 'preview', requests: 180 } + ] + + it('gets its own rule rather than riding the low-risk bypass', () => { + const rules = recommendFirewallRules(WITH_PREVIEW) + const unfurl = rules.find((r) => r.name === 'Allow link unfurlers')! + const agents = rules.find((r) => r.name.match(/retrieval and search/i))! + expect(unfurl.action).toBe('bypass') + // A bypass skips managed rulesets too, and these tokens are unverifiable. + // Labelling that `low` alongside vendors who publish IP ranges would + // understate what the rule hands out. + expect(unfurl.risk).toBe('medium') + expect(agents.risk).toBe('low') + expect(agents.groups[0]![0]!.value).not.toContain('facebookexternalhit') + }) + + it('sits immediately behind the agent bypass in evaluation order', () => { + const rules = recommendFirewallRules(WITH_PREVIEW) + expect(rules[0]!.name).toMatch(/retrieval and search/i) + expect(rules[1]!.name).toBe('Allow link unfurlers') + }) + + it('is not emitted when nothing unfurled', () => { + expect(recommendFirewallRules(OBS).some((r) => r.name === 'Allow link unfurlers')).toBe(false) + }) + + it('keeps every bypass rule on top of the published order', () => { + const script = firewallScript(recommendFirewallRules(WITH_PREVIEW)) + const reorders = script.split('\n').filter((l) => l.includes('--first --yes')) + expect(reorders).toHaveLength(2) + // Applied back-to-front, so the agent rule lands on top. + expect(reorders[0]).toContain('Allow link unfurlers') + expect(reorders[1]).toContain('Allow retrieval and search agents') + }) +}) + describe('firewallScript', () => { it('is a runnable script that publishes nothing', () => { const script = firewallScript(recommendFirewallRules(OBS)) diff --git a/test/policy.test.ts b/test/policy.test.ts index a783405..aa66697 100644 --- a/test/policy.test.ts +++ b/test/policy.test.ts @@ -142,6 +142,10 @@ describe('agentIntent and agentPolicy never disagree', () => { 'Mozilla/5.0 (compatible; Googlebot/2.1)', 'Mozilla/5.0 (compatible; Applebot/0.1)', 'Mozilla/5.0 (compatible; Applebot-Extended/0.1)', + 'Mozilla/5.0 (compatible; PerplexityBot/1.0)', + 'Mozilla/5.0 (compatible; Perplexity-User/1.0)', + 'facebookexternalhit/1.1', + 'Slackbot-LinkExpanding 1.0', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36', '', 'SomethingCompletelyUnknown/9' @@ -161,3 +165,83 @@ describe('agentIntent and agentPolicy never disagree', () => { expect(agentIntent('axios/1.8.4')).toBe('tooling') }) }) + +describe('vendors that fell through every list', () => { + // Found in production traffic, not in review: 42 requests a day from + // PerplexityBot classified as `unknown`, which put it in no firewall rule at + // all — neither protected by the retrieval/search bypass nor bounded by the + // training rate limit. The `Bot` suffix reads like a corpus crawler, but + // Perplexity documents it as the crawler behind their search results. + it('classifies PerplexityBot as search, not unknown', () => { + expect(agentIntent('Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)')).toBe('search') + }) + + it('keeps Perplexity-User on retrieval — the tokens must not collide', () => { + expect(agentIntent('Mozilla/5.0 (compatible; Perplexity-User/1.0)')).toBe('retrieval') + }) + + it('never leaves a known agent in the intent gap', () => { + // Anything here that returns `unknown` is invisible to every generated + // rule. That is the actual failure mode, so assert the absence directly. + const KNOWN = [ + 'Mozilla/5.0 (compatible; PerplexityBot/1.0)', + 'Mozilla/5.0 (compatible; Bravebot/1.0)', + 'facebookexternalhit/1.1', + 'Twitterbot/1.0', + 'LinkedInBot/1.0', + 'Slackbot-LinkExpanding 1.0', + 'Discordbot/2.0', + 'TelegramBot (like TwitterBot)', + 'WhatsApp/2.23', + 'redditbot/1.0' + ] + for (const ua of KNOWN) { + expect(agentIntent(ua), `${ua} is in no intent bucket`).not.toBe('unknown') + } + }) +}) + +describe('link unfurlers', () => { + const UNFURLERS = [ + ['facebookexternalhit/1.1', 'facebook'], + ['Twitterbot/1.0', 'twitter'], + ['LinkedInBot/1.0 (compatible; Mozilla/5.0)', 'linkedin'], + ['Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)', 'slack'], + ['Discordbot/2.0 (+https://discordapp.com)', 'discord'], + ['WhatsApp/2.23.20.0', 'whatsapp'] + ] as const + + it.each(UNFURLERS)('%s is preview', (ua) => { + expect(agentIntent(ua)).toBe('preview') + }) + + it('is allowed by default', () => { + const d = agentPolicy( + new Request('https://example.com/', { headers: { 'user-agent': 'Slackbot-LinkExpanding 1.0' } }) + ) + expect(d.action).toBe('allow') + expect(d.intent).toBe('preview') + }) + + it('honours onPreview', () => { + const d = agentPolicy( + new Request('https://example.com/', { headers: { 'user-agent': 'Discordbot/2.0' } }), + { onPreview: 'block' } + ) + expect(d.action).toBe('block') + }) + + it('does not steal Applebot from search', () => { + // Apple uses one token for the search crawler and for iMessage previews. + // PREVIEW is tested after SEARCH so the crawler classification wins; if the + // order is ever flipped this catches it. + expect(agentIntent('Mozilla/5.0 (compatible; Applebot/0.1)')).toBe('search') + }) + + it('keeps preview out of the retrieval demand signal', () => { + // The whole point of a separate bucket: retrieval is what the library sells + // as demand. A Slack unfurl is not someone asking an assistant a question, + // and counting it as one inflates the headline number. + expect(agentIntent('Slackbot-LinkExpanding 1.0')).not.toBe('retrieval') + }) +})