Close the intent gap and catch distributed scrapers (0.17.0)

README.md

@@ -156,9 +156,15 @@ Measured against real traffic shapes:
 serve  retrieval  allow   ChatGPT-User
 403    training   block   ClaudeBot from an unpublished IP
 402    training   charge  ClaudeBot from a real Anthropic IP
-serve  search     allow   Googlebot
+serve  search     allow   Googlebot, PerplexityBot
+serve  preview    allow   Slackbot, facebookexternalhit
 ```
 
+`preview` is its own intent rather than a flavour of `retrieval`: a link unfurl
+is a person pasting your URL into a conversation, not an assistant answering a
+question. Folding the two together would inflate the retrieval number, which is
+the one figure the split exists to measure.
+
 Settlement is never ours. `mppxGateway` wraps Stripe's MPP SDK; `x402Gateway`
 calls a facilitator you supply. The library emits challenges and reads
 credentials — holding money would drag PCI scope into edge middleware.
@@ -279,6 +285,26 @@ its `evidence`, a `risk` rating, and a `caveat` where over-blocking is plausible
 — the datacenter-ASN rule is marked `high` risk because corporate VPNs and
 privacy relays egress from hosting networks.
 
+### Pools that no per-address threshold catches
+
+A rotating proxy pool is built so that no single address looks abusive. Measured
+on one production site: 34 addresses across 11 countries, one user agent each,
+the heaviest doing 100 requests in a day — every one invisible to a per-address
+threshold, while collectively sweeping the site.
+
+Volume cannot separate that from real readers, so the burst rule keys on *rate*,
+which needs `spanSeconds` on your observations:
+
+```ts
+{ ip: '104.28.233.73', requests: 31, distinctPaths: 16, spanSeconds: 1 }
+// → 1,860 requests/min. Not a person.
+```
+
+The rule is only safe because its condition excludes static assets. The WAF sees
+every request; the middleware that produced your observations probably does not,
+so a naive limit on `Mozilla` throttles a real visitor on their first page view.
+Override `assetExclusions` if your app does not serve assets from `/_next/`.
+
 See [`docs/TESTING-PAYMENTS.md`](./docs/TESTING-PAYMENTS.md) for testing the
 payment path end to end.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apideck/agent-analytics",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Track AI agent and bot traffic to your Next.js / Vercel app — PostHog, webhooks, or any custom analytics backend. Detects Claude, ChatGPT, Perplexity, Google-Extended, and more.",
   "keywords": [
     "ai",

size-budget.json

@@ -14,8 +14,8 @@
       "note": "Opt-in payment surface."
     },
     "dist/firewall.js": {
-      "gzipBudget": 3200,
-      "note": "Offline analysis tool; never runs in middleware."
+      "gzipBudget": 4300,
+      "note": "Offline analysis tool; never runs in middleware, so size buys correctness cheaply here. Raised from 3200 for the distributed-pool rule class and the link-unfurler bypass."
     },
     "dist/markdown.js": {
       "gzipBudget": 1300,

src/firewall.ts

@@ -88,6 +88,14 @@ export interface TrafficObservation {
   asn?: number
   /** Distinct paths this slice touched — a scraper sweeps, a reader does not. */
   distinctPaths?: number
+  /**
+   * Seconds between this slice's first and last request. Combined with
+   * `requests` this gives a rate, which is the signal that separates a scraper
+   * from a reader when volume alone does not: on one production site the
+   * heaviest single address managed only 100 requests a day — far under any
+   * sane abuse threshold — but fetched 16 distinct pages in one second.
+   */
+  spanSeconds?: number
   /** Verification verdict, if you ran one. */
   verification?: 'verified' | 'spoofed' | 'unverifiable' | 'not-claimed'
   country?: string
@@ -103,8 +111,32 @@ export interface RecommendOptions {
   trainingBudget?: { window: number; requests: number }
   /** Skip the protective bypass rule. Rarely a good idea. */
   omitProtectiveBypass?: boolean
+  /**
+   * Burst budget for the distributed-scraper rule, in *page* requests. Defaults
+   * to 30 per 60s, which no human reaches once static assets are excluded.
+   */
+  burstBudget?: { window: number; requests: number }
+  /**
+   * How many distinct low-volume addresses must look alike before they are
+   * treated as one coordinated pool. Defaults to 5.
+   */
+  minPoolSize?: number
+  /**
+   * Path prefixes and extensions the burst rule must not count, because the WAF
+   * sees every asset request while your middleware probably does not. Defaults
+   * to Next.js internals and the usual static extensions.
+   */
+  assetExclusions?: { prefixes: readonly string[]; extensions: readonly string[] }
 }
 
+const DEFAULT_ASSET_EXCLUSIONS = {
+  prefixes: ['/_next/'],
+  extensions: [
+    'js', 'css', 'png', 'jpg', 'jpeg', 'gif', 'svg', 'webp', 'ico',
+    'woff', 'woff2', 'ttf', 'otf', 'map'
+  ]
+} as const
+
 /* -------------------------------------------------------------------------- */
 
 function shellQuote(json: unknown): string {
@@ -194,6 +226,7 @@ export function recommendFirewallRules(
                 'Claude-User',
                 'Claude-SearchBot',
                 'Perplexity-User',
+                'PerplexityBot',
                 'Googlebot',
                 'bingbot',
                 'DuckDuckBot',
@@ -211,6 +244,50 @@ export function recommendFirewallRules(
     )
   }
 
+  /* 1b. Link unfurlers, protected separately from the agents above. ----------
+     Kept out of the rule above deliberately. Those tokens belong to vendors who
+     publish IP ranges and increasingly sign their requests; `facebookexternalhit`
+     and friends do neither and are among the most-forged strings on the web. A
+     `bypass` skips every managed ruleset too, so folding them into a rule
+     labelled `low` risk would understate what it hands out. Same protection,
+     honest label, and an operator can decline this one on its own. */
+  const preview = observations.filter((o) => o.intent === 'preview')
+  if (!opts.omitProtectiveBypass && preview.length) {
+    const requests = preview.reduce((n, o) => n + o.requests, 0)
+    const names = [...new Set(preview.map((o) => o.botName))]
+    out.push(
+      finish({
+        name: 'Allow link unfurlers',
+        rationale:
+          'Someone pasted your URL into a conversation and the platform fetched it to render a card. Blocking it makes your links look broken wherever they are shared.',
+        evidence: `${requests.toLocaleString('en-US')} requests across ${names.length} platform${names.length === 1 ? '' : 's'} (${names.slice(0, 6).join(', ')})`,
+        groups: [
+          [
+            {
+              type: 'user_agent',
+              op: 'inc',
+              value: [
+                'facebookexternalhit',
+                'Twitterbot',
+                'LinkedInBot',
+                'Slackbot',
+                'Discordbot',
+                'TelegramBot',
+                'WhatsApp',
+                'redditbot'
+              ]
+            }
+          ]
+        ],
+        action: 'bypass',
+        eventual: 'bypass',
+        risk: 'medium',
+        caveat:
+          'These user agents are trivially spoofed and none of these platforms publish verifiable IP ranges, so this rule hands a bypass to anyone who sets the header. Scope it to your public content paths and never to anything authenticated or expensive.'
+      })
+    )
+  }
+
   /* 2. Failed verification — the only class that earns a denial. ------------- */
   const spoofed = observations.filter((o) => o.verification === 'spoofed')
   const spoofedIps = [...new Set(spoofed.map((o) => o.ip).filter((v): v is string => !!v))]
@@ -264,6 +341,78 @@ export function recommendFirewallRules(
     )
   }
 
+  /* 3b. Many small addresses behaving as one. --------------------------------
+     Rule 3 asks "is any single address abusive?" and a rotating proxy pool is
+     built precisely so the answer is no. Observed on one production site: 34
+     addresses across 11 countries, one user agent each, none above 100 requests
+     a day — every one of them invisible to a per-address threshold, while
+     collectively sweeping the site and inflating its analytics.
+
+     Volume cannot separate that from real readers, so this keys on rate. It is
+     safe to do so only because the condition excludes static assets: a person
+     loading one page fires dozens of asset requests that the WAF counts and
+     your middleware does not, and a naive limit on `Mozilla` would throttle
+     real users on their first page view. */
+  const pool = observations.filter(
+    (o) =>
+      o.ip !== undefined &&
+      !heavy.some((h) => h.ip === o.ip) &&
+      o.verification !== 'verified' &&
+      (o.intent === 'unknown' || o.intent === 'tooling') &&
+      /Mozilla|Chrome|Safari/i.test(o.userAgent)
+  )
+  const minPool = opts.minPoolSize ?? 5
+  const poolIps = [...new Set(pool.map((o) => o.ip!))]
+  if (poolIps.length >= minPool) {
+    const burst = opts.burstBudget ?? { window: 60, requests: 30 }
+    const requests = pool.reduce((n, o) => n + o.requests, 0)
+    const countries = [...new Set(pool.map((o) => o.country).filter(Boolean))]
+
+    // Only slices that actually carry timing can be shown to exceed the budget.
+    // Say how many were measurable rather than implying the rest were clean.
+    const timed = pool.filter((o) => o.spanSeconds !== undefined && o.spanSeconds > 0)
+    const overBudget = timed.filter(
+      (o) => o.requests / o.spanSeconds! > burst.requests / burst.window
+    )
+    const peak = timed.length
+      ? Math.max(...timed.map((o) => (o.requests / o.spanSeconds!) * 60))
+      : 0
+
+    const assets = opts.assetExclusions ?? DEFAULT_ASSET_EXCLUSIONS
+    const conditions: FirewallCondition[] = [
+      { type: 'user_agent', op: 'sub', value: 'Mozilla' },
+      ...assets.prefixes.map(
+        (p): FirewallCondition => ({ type: 'path', op: 'pre', value: p, neg: true })
+      ),
+      {
+        type: 'path',
+        op: 're',
+        value: `\\.(?:${assets.extensions.join('|')})$`,
+        neg: true
+      }
+    ]
+
+    out.push(
+      finish({
+        name: 'Burst limit page navigations',
+        rationale:
+          'A pool of addresses each too small to trip a per-address threshold, together behaving like one scraper. Rate is the only signal that separates them from real readers.',
+        evidence:
+          `${poolIps.length} addresses across ${countries.length} countries, ${requests.toLocaleString('en-US')} requests, none individually above the abuse threshold` +
+          (timed.length
+            ? `; ${overBudget.length} of ${timed.length} measurable slices exceeded ${burst.requests} requests/${burst.window}s, peaking at ${Math.round(peak).toLocaleString('en-US')}/min`
+            : '; no timing supplied, so the burst budget is a default rather than a measurement'),
+        groups: [conditions],
+        action: 'log',
+        eventual: 'rate_limit',
+        rateLimit: { window: burst.window, requests: burst.requests, action: 'rate_limit', keys: ['ip'] },
+        risk: 'medium',
+        caveat:
+          'The asset exclusions are what make this safe — the WAF counts every request, including the dozens of assets behind a single page view, so verify they match how your app actually serves static files before enforcing. Vercel counters are per region, so N regions can collectively exceed the limit by ~Nx.'
+      })
+    )
+  }
+
   /* 4. Training crawlers: bound the cost, do not disappear from corpora. ----- */
   const training = observations.filter((o) => o.intent === 'training')
   if (training.length) {
@@ -357,11 +506,16 @@ export function firewallScript(recommendations: readonly FirewallRecommendation[
     lines.push(r.cli)
     lines.push('')
   })
+  const protective = recommendations.filter((r) => r.action === 'bypass')
+  if (protective.length) {
+    lines.push('# Keep the protective allow rules at the top of the evaluation order.')
+    // Reversed: each --first pushes to the top, so applying them back-to-front
+    // leaves the array's own order intact once all of them have run.
+    for (const r of [...protective].reverse()) {
+      lines.push(`vercel firewall rules reorder ${JSON.stringify(r.name)} --first --yes`)
+    }
+  }
   if (recommendations.length) {
-    lines.push('# Keep the protective allow rule at the top of the evaluation order.')
-    lines.push(
-      `vercel firewall rules reorder ${JSON.stringify(recommendations[0]!.name)} --first --yes`
-    )
     lines.push('')
     lines.push('vercel firewall diff')
     lines.push('echo "Review above, then: vercel firewall publish --yes"')

src/policy.ts

@@ -21,12 +21,16 @@ export type AgentAction = 'allow' | 'meter' | 'charge' | 'block'
  *   and charging for it is charging for your own marketing.
  * - `'training'`  — bulk corpus collection for model training. You get nothing
  *   back per fetch, which is where a price makes sense.
- * - `'search'`    — classic index crawlers. Blocking these costs you SEO.
+ * - `'search'`    — index crawlers, traditional and AI-native. Blocking these
+ *   costs you organic traffic or citations in an assistant's answer.
+ * - `'preview'`   — link unfurlers. Someone pasted your URL into Slack, iMessage
+ *   or a tweet and the platform fetched it to render a card. No model involved,
+ *   but blocking it means your links look broken wherever they get shared.
  * - `'tooling'`   — coding agents and HTTP clients. Usually developers using
  *   your docs; treat like retrieval unless you see abuse.
  * - `'unknown'`   — everything else, including real browsers.
  */
-export type AgentIntent = 'retrieval' | 'training' | 'search' | 'tooling' | 'unknown'
+export type AgentIntent = 'retrieval' | 'training' | 'search' | 'preview' | 'tooling' | 'unknown'
 
 /**
  * User agents where a human is waiting on the answer. Deliberately explicit
@@ -39,8 +43,33 @@ const RETRIEVAL = /ChatGPT-User|OAI-SearchBot|Claude-User|Claude-SearchBot|Perpl
 /** Bulk crawlers that collect corpora. No human is waiting on these. */
 const TRAINING = /GPTBot|ClaudeBot|Claude-Web|CCBot|Bytespider|Amazonbot|Amzn-SearchBot|Meta-ExternalAgent|meta-externalfetcher|meta-webindexer|FacebookBot|Google-Extended|Applebot-Extended|AI2Bot|Diffbot|omgili|Webzio-Extended|Timpibot|PanguBot|cohere|DeepSeek|Grok|quillbot|MyCentralAIScraperBot|NovaAct|AzureAI-SearchBot|Google-CloudVertexBot/i
 
-/** Classic search indexers — blocking these costs you organic traffic. */
-const SEARCH = /bingbot|Googlebot|DuckDuckBot|YandexBot|Baiduspider|PetalBot|Sogou|Applebot(?!-Extended)/i
+/**
+ * Index crawlers — blocking these costs you organic traffic.
+ *
+ * `PerplexityBot` sits here rather than in TRAINING despite the `Bot` suffix:
+ * Perplexity documents it as the crawler behind their *search results* and
+ * states it does not feed foundation-model training. Blocking it costs you
+ * citations, which is the same shape of loss as blocking Googlebot. It was
+ * previously in no list at all, so it classified as `unknown` and fell through
+ * both the protective bypass and the training rate limit — a live gap found in
+ * production traffic, not in review.
+ */
+const SEARCH = /bingbot|Googlebot|DuckDuckBot|YandexBot|Baiduspider|PetalBot|Sogou|PerplexityBot|Bravebot|Applebot(?!-Extended)/i
+
+/**
+ * Link unfurlers. A human shared the URL and a platform fetched it to build a
+ * preview card — one request, no crawl, and the payoff is a rendered link in a
+ * conversation. They get their own intent rather than being folded into
+ * `retrieval` because retrieval is the library's demand signal: counting
+ * Slackbot as "an assistant went to read this for someone" would inflate the
+ * one number the split exists to measure.
+ *
+ * These tokens are trivially spoofable — `facebookexternalhit` is among the
+ * most-forged strings on the web. Treat this as a routing hint, never as
+ * identity, and note that {@link recommendFirewallRules} proposes them as a
+ * separate, higher-risk rule for exactly that reason.
+ */
+const PREVIEW = /facebookexternalhit|Twitterbot|LinkedInBot|Slackbot|Discordbot|TelegramBot|WhatsApp|redditbot|Pinterest|SkypeUriPreview|Iframely|Embedly|vkShare|Mastodon|Bluesky/i
 
 export interface AgentDecision {
   action: AgentAction
@@ -81,6 +110,11 @@ export interface AgentPolicyOptions {
   onRetrieval?: AgentAction
   /** What to do with search indexers. Defaults to `'allow'`. */
   onSearch?: AgentAction
+  /**
+   * What to do with link unfurlers. Defaults to `'allow'` — gating these does
+   * not earn you anything, it just makes your links render as bare URLs.
+   */
+  onPreview?: AgentAction
   /** What to do with coding agents and HTTP clients. Defaults to `'allow'`. */
   onTooling?: AgentAction
   /** Vendor labels or UA substrings always allowed, whatever the intent. */
@@ -105,6 +139,9 @@ export function agentIntent(userAgent: string | null | undefined): AgentIntent {
   if (RETRIEVAL.test(ua)) return 'retrieval'
   if (TRAINING.test(ua)) return 'training'
   if (SEARCH.test(ua)) return 'search'
+  // After SEARCH so Applebot stays a search crawler rather than an iMessage
+  // unfurler — Apple uses the same token for both.
+  if (PREVIEW.test(ua)) return 'preview'
   // An HTTP-library UA that matched no vendor is a coding agent or a script.
   if (isHttpClient(ua)) return 'tooling'
   return 'unknown'
@@ -165,14 +202,17 @@ export function agentPolicy(req: Request, opts: AgentPolicyOptions = {}): AgentD
         ? (opts.onRetrieval ?? 'allow')
         : intent === 'search'
           ? (opts.onSearch ?? 'allow')
-          : intent === 'tooling'
-            ? (opts.onTooling ?? 'allow')
-            : 'allow'
+          : intent === 'preview'
+            ? (opts.onPreview ?? 'allow')
+            : intent === 'tooling'
+              ? (opts.onTooling ?? 'allow')
+              : 'allow'
 
   const REASONS: Record<AgentIntent, string> = {
     retrieval: 'a person is waiting on this answer',
     training: 'bulk corpus collection',
     search: 'search index crawler',
+    preview: 'link unfurler building a preview card',
     tooling: 'coding agent or HTTP client',
     unknown: 'not a recognised agent'
   }