From ba9bafa0947e1b20dceee5d3ef47308a75d7d667 Mon Sep 17 00:00:00 2001 From: Chris Portscheller Date: Thu, 12 Nov 2026 21:14:42 -0600 Subject: [PATCH] Web Bot Auth: follow draft-ietf-webbotauth-httpsig-protocol-01 A key verifies only for the Signature-Agent directory that published it. Signature-Agent values are key-set URLs (a bare origin means its well-known directory), parameterized members are ignored, the covered member must be the one relied on, and directory fetches refuse redirects. Resolution is checked against the Go library's shared vectors. Vercel validator bundle regenerated. --- CHANGELOG.md | 6 ++ packages/webdecoy/src/agent/directory.ts | 41 ++++---- packages/webdecoy/src/agent/identity.ts | 45 +++++++++ packages/webdecoy/src/agent/sfv.ts | 10 ++ .../src/agent/signature-agent-resolution.json | 72 ++++++++++++++ packages/webdecoy/src/agent/signature.ts | 53 ++++++++++- .../webdecoy/src/agent/web-bot-auth.test.ts | 93 ++++++++++++++++++- .../webdecoy/src/clearance/gate.generated.js | 18 ++-- packages/webdecoy/src/edge-runtime.test.ts | 5 +- 9 files changed, 308 insertions(+), 35 deletions(-) create mode 100644 packages/webdecoy/src/agent/identity.ts create mode 100644 packages/webdecoy/src/agent/signature-agent-resolution.json diff --git a/CHANGELOG.md b/CHANGELOG.md index aa49274..89799e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Web Bot Auth verification follows draft-ietf-webbotauth-httpsig-protocol-01.** A `Signature-Agent` value is the URL of the agent's key set, and a key now verifies only for the directory that published it, so a key reused by another directory, or published at a different URL on the same host, no longer borrows the trusted agent's name. A bare origin, the form agents send today, still means that origin's well-known directory. Members carrying parameters are ignored, a signature must cover the `Signature-Agent` member it relies on, and directory fetches no longer follow redirects. Signatures that cannot be attributed report `claimed`, never `verified`. +- **Breaking for direct `DirectoryCache` users:** `resolveKey(keyId)` is now `resolveKey(keyId, agent)`, where `agent` is the resolved Signature-Agent identifier. `AgentVerifier` and `detectBot()` callers are unaffected. +- The Vercel validator's bundled clearance decision carries the same Web Bot Auth change. + ## [0.22.3] - 2026-11-11 ### Changed diff --git a/packages/webdecoy/src/agent/directory.ts b/packages/webdecoy/src/agent/directory.ts index aec8794..29d54ab 100644 --- a/packages/webdecoy/src/agent/directory.ts +++ b/packages/webdecoy/src/agent/directory.ts @@ -22,8 +22,7 @@ import { jwkThumbprint, type JWK } from './thumbprint'; import type { VerificationKey, KeyResolver } from './signature'; import type { SignedAgentDirectory } from './types'; - -const WELL_KNOWN = '/.well-known/http-message-signatures-directory'; +import { resolveSignatureAgent } from './identity'; const COLD_RETRY_BACKOFF_MS = 30_000; /** @@ -62,10 +61,14 @@ export class DirectoryCache { this.opts = opts; } - /** Resolve a keyid to a verification key, fetching/refreshing as needed. */ - resolveKey: KeyResolver = async (keyId: string) => { + /** + * Resolve a keyid published by one agent identity (a resolved + * Signature-Agent URL) to a verification key, fetching/refreshing as + * needed. A key is only ever attributed to the directory that published it. + */ + resolveKey: KeyResolver = async (keyId: string, agent: string) => { await this.ensureFresh(); - return this.keys.get(keyId); + return this.keys.get(cacheKey(agent, keyId)); }; /** Kick off a fetch without blocking — call at startup to warm the cache. */ @@ -110,7 +113,7 @@ export class DirectoryCache { for (const keys of results) { if (keys === null) continue; fetchedAny = true; - for (const k of keys) next.set(k.thumbprint, k.key); + for (const k of keys) next.set(cacheKey(k.identifier, k.thumbprint), k.key); } if (fetchedAny) { @@ -127,8 +130,15 @@ export class DirectoryCache { private async fetchDirectory( src: SignedAgentDirectory, - ): Promise<{ thumbprint: string; key: VerificationKey }[] | null> { - const url = directoryUrl(src.directory); + ): Promise<{ identifier: string; thumbprint: string; key: VerificationKey }[] | null> { + let url: string; + let identifier: string; + try { + ({ fetch: url, identifier } = resolveSignatureAgent(src.directory)); + } catch { + this.debug(`${src.name}: directory is not an https URL`); + return null; + } const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), this.opts.fetchTimeoutMs); try { @@ -138,7 +148,8 @@ export class DirectoryCache { Accept: 'application/http-message-signatures-directory+json, application/jwk-set+json, application/json', }, - redirect: 'follow', + // Web Bot Auth draft 01 §5.5: redirects are a discovery failure. + redirect: 'error', }); if (!res.ok) { this.debug(`${src.name}: HTTP ${res.status}`); @@ -149,10 +160,10 @@ export class DirectoryCache { this.debug(`${src.name}: no keys in directory`); return null; } - const out: { thumbprint: string; key: VerificationKey }[] = []; + const out: { identifier: string; thumbprint: string; key: VerificationKey }[] = []; for (const jwk of body.keys) { const resolved = await importKey(jwk, src); - if (resolved) out.push(resolved); + if (resolved) out.push({ identifier, ...resolved }); } this.debug(`${src.name}: ${out.length} key(s)`); return out; @@ -169,12 +180,8 @@ export class DirectoryCache { } } -/** Append the well-known path unless the directory string already carries one. */ -function directoryUrl(directory: string): string { - const u = new URL(directory); - if (u.pathname && u.pathname !== '/') return u.toString(); - u.pathname = WELL_KNOWN; - return u.toString(); +function cacheKey(agent: string, keyId: string): string { + return `${agent} ${keyId}`; } /** diff --git a/packages/webdecoy/src/agent/identity.ts b/packages/webdecoy/src/agent/identity.ts new file mode 100644 index 0000000..15c10d6 --- /dev/null +++ b/packages/webdecoy/src/agent/identity.ts @@ -0,0 +1,45 @@ +/** + * Signature-Agent identity (draft-ietf-webbotauth-httpsig-protocol-01 §5.5). + * + * A Signature-Agent value is a URL that serves a JWK Set, and that URL (minus + * its query) is the agent's identity. A bare origin is the form signers sent + * under earlier drafts and still send today; it means the origin's well-known + * directory, because fetching the origin itself would return a web page. + * + * Mirrors `ResolveSignatureAgent` in github.com/WebDecoy/web-bot-auth. The + * shared vectors in `signature-agent-resolution.json` keep the two in step. + */ + +export const WELL_KNOWN_DIRECTORY = '/.well-known/http-message-signatures-directory'; + +export interface ResolvedAgent { + /** The URL a verifier fetches the JWK Set from. */ + fetch: string; + /** The identity keys are attributed to: the fetched URL without its query. */ + identifier: string; +} + +/** Resolve a Signature-Agent value. Throws when it is not a usable https URL. */ +export function resolveSignatureAgent(value: string): ResolvedAgent { + // WHATWG URL parsing forgives "https:///host"; the draft does not. + if (!/^https:\/\/[^/\\]/i.test(value)) throw new Error('signature-agent must be an https URL'); + const url = new URL(value); + if (url.protocol !== 'https:' || !url.hostname || url.username || url.password) { + throw new Error('signature-agent must be an https URL'); + } + url.hash = ''; + const hasQuery = url.href.includes('?'); + const authority = canonicalAuthority(url); + if (url.pathname === '/' && !hasQuery) { + const wellKnown = `https://${authority}${WELL_KNOWN_DIRECTORY}`; + return { fetch: wellKnown, identifier: wellKnown }; + } + const path = url.pathname.replace(/%[0-9a-f]{2}/gi, (m) => m.toUpperCase()); + return { fetch: url.href, identifier: `https://${authority}${path}` }; +} + +function canonicalAuthority(url: URL): string { + const host = url.hostname.toLowerCase(); + if (!url.port || url.port === '443') return host; + return `${host}:${url.port}`; +} diff --git a/packages/webdecoy/src/agent/sfv.ts b/packages/webdecoy/src/agent/sfv.ts index ce1c760..04f8c97 100644 --- a/packages/webdecoy/src/agent/sfv.ts +++ b/packages/webdecoy/src/agent/sfv.ts @@ -51,6 +51,16 @@ class Cursor { } /** Parse an RFC 8941 dictionary header value. Throws on malformed input. */ +/** Parse a header value that is exactly one sf-string (RFC 8941 §3.3.3). */ +export function parseBareString(input: string): string { + const p = new Cursor(input); + p.skipSP(); + const value = parseString(p); + p.skipSP(); + if (!p.eof()) throw new Error('trailing data after string'); + return value; +} + export function parseDictionary(input: string): DictEntry[] { const p = new Cursor(input); const out: DictEntry[] = []; diff --git a/packages/webdecoy/src/agent/signature-agent-resolution.json b/packages/webdecoy/src/agent/signature-agent-resolution.json new file mode 100644 index 0000000..260f8ce --- /dev/null +++ b/packages/webdecoy/src/agent/signature-agent-resolution.json @@ -0,0 +1,72 @@ +{ + "_comment": "Signature-Agent member value -> (fetch URL, draft-01 identifier). Generated from and checked by ResolveSignatureAgent; the edge and SDK TypeScript ports test against a copy. Ports may differ on the fetch URL's spelling (case, default port) but must agree on the identifier.", + "valid": [ + { + "value": "https://agent.example", + "fetch": "https://agent.example/.well-known/http-message-signatures-directory", + "identifier": "https://agent.example/.well-known/http-message-signatures-directory" + }, + { + "value": "https://agent.example/", + "fetch": "https://agent.example/.well-known/http-message-signatures-directory", + "identifier": "https://agent.example/.well-known/http-message-signatures-directory" + }, + { + "value": "https://Agent.Example:443", + "fetch": "https://Agent.Example:443/.well-known/http-message-signatures-directory", + "identifier": "https://agent.example/.well-known/http-message-signatures-directory" + }, + { + "value": "https://agent.example/.well-known/http-message-signatures-directory", + "fetch": "https://agent.example/.well-known/http-message-signatures-directory", + "identifier": "https://agent.example/.well-known/http-message-signatures-directory" + }, + { + "value": "https://agent.example/jwks.json", + "fetch": "https://agent.example/jwks.json", + "identifier": "https://agent.example/jwks.json" + }, + { + "value": "https://agent.example/keys?rev=3", + "fetch": "https://agent.example/keys?rev=3", + "identifier": "https://agent.example/keys" + }, + { + "value": "https://agent.example/?rev=3", + "fetch": "https://agent.example/?rev=3", + "identifier": "https://agent.example/" + }, + { + "value": "https://agent.example/a/../keys", + "fetch": "https://agent.example/a/../keys", + "identifier": "https://agent.example/keys" + }, + { + "value": "https://agent.example/k%c3%a9ys", + "fetch": "https://agent.example/k%c3%a9ys", + "identifier": "https://agent.example/k%C3%A9ys" + }, + { + "value": "https://agent.example/keys#frag", + "fetch": "https://agent.example/keys", + "identifier": "https://agent.example/keys" + }, + { + "value": "https://agent.example:8443/keys", + "fetch": "https://agent.example:8443/keys", + "identifier": "https://agent.example:8443/keys" + }, + { + "value": "https://crawler.agent.example/jwks.json", + "fetch": "https://crawler.agent.example/jwks.json", + "identifier": "https://crawler.agent.example/jwks.json" + } + ], + "invalid": [ + "http://agent.example", + "https:///keys", + "https://user:pw@agent.example/keys", + "agent.example", + "" + ] +} diff --git a/packages/webdecoy/src/agent/signature.ts b/packages/webdecoy/src/agent/signature.ts index 5ea11e7..c6c59c0 100644 --- a/packages/webdecoy/src/agent/signature.ts +++ b/packages/webdecoy/src/agent/signature.ts @@ -12,7 +12,8 @@ * `atob`, and `URL` are used. */ -import { parseDictionary, type DictEntry, type BareItem } from './sfv'; +import { parseDictionary, parseBareString, type DictEntry, type BareItem } from './sfv'; +import { resolveSignatureAgent } from './identity'; const TAG = 'web-bot-auth'; const encoder = new TextEncoder(); @@ -33,8 +34,12 @@ export interface VerificationKey { exp?: number; } -/** Resolves a JWK-thumbprint keyid to a verification key, or undefined if unknown. */ -export type KeyResolver = (keyId: string) => Promise; +/** + * Resolves a JWK-thumbprint keyid, as published by one agent identity (a + * resolved Signature-Agent URL), to a verification key, or undefined if that + * identity publishes no such key. + */ +export type KeyResolver = (keyId: string, agent: string) => Promise; export type CoreOutcome = 'none' | 'verified' | 'impersonation' | 'claimed'; @@ -116,7 +121,18 @@ export async function verifyWebBotAuth( // Must bind the host, per the profile. if (!m.components.some((c) => c.name === '@authority' || c.name === '@target-uri')) continue; - const key = await resolveKey(keyid); + // The key is looked up under the Signature-Agent identity this signature + // covers, so a key reused by two directories never borrows the other's + // name. A signature that binds no identity cannot be attributed. + let agent: string; + try { + agent = coveredSignatureAgent(req, m); + } catch { + best = higher(best, { outcome: 'claimed', keyId: keyid, reason: 'unbound-agent' }); + continue; + } + + const key = await resolveKey(keyid, agent); if (!key) { // Unknown agent — a claim we can't confirm or refute. best = higher(best, { outcome: 'claimed', keyId: keyid, reason: 'unknown-key' }); @@ -248,6 +264,35 @@ function dictMemberValue(header: string, key: string): string { return found.raw; } +/** + * The identity of the Signature-Agent member a signature covers. Draft 01 + * §5.2.1: members are https URLs, a member with parameters MUST be ignored, + * and the legacy bare-string form is still accepted for deployed signers. + */ +function coveredSignatureAgent(req: NormalizedRequest, m: SigInputMember): string { + const header = req.headers.get('signature-agent'); + if (header === null) throw new Error('missing signature-agent'); + const component = m.components.find((c) => c.name.toLowerCase() === 'signature-agent'); + if (!component) throw new Error('signature-agent is not covered'); + const trimmed = header.trim(); + let value: string; + if (trimmed.startsWith('"')) { + if (component.keyParam !== undefined) throw new Error('legacy string cannot select a member'); + value = parseBareString(trimmed); + } else { + // Covering the whole dictionary would not say which member is ours. + if (component.keyParam === undefined) throw new Error('dictionary member is not covered'); + const member = parseDictionary(trimmed).find((d) => d.key === component.keyParam); + const item = member?.items[0]; + if (!member || member.isList || member.items.length !== 1 || typeof item?.value !== 'string') { + throw new Error('signature-agent member is not a string'); + } + if (item.params.size > 0) throw new Error('signature-agent member carries parameters'); + value = item.value; + } + return resolveSignatureAgent(value).identifier; +} + // --- header parsing --- function parseSignatureInput(value: string): SigInputMember[] { diff --git a/packages/webdecoy/src/agent/web-bot-auth.test.ts b/packages/webdecoy/src/agent/web-bot-auth.test.ts index 17de1ad..42efb52 100644 --- a/packages/webdecoy/src/agent/web-bot-auth.test.ts +++ b/packages/webdecoy/src/agent/web-bot-auth.test.ts @@ -13,6 +13,8 @@ import { AgentVerifier } from './verifier'; import { DEFAULT_SIGNED_AGENT_DIRECTORIES } from './directory'; import { jwkThumbprint, rsaThumbprint } from './thumbprint'; import type { SignedAgentDirectory } from './types'; +import { resolveSignatureAgent } from './identity'; +import resolution from './signature-agent-resolution.json'; const enc = new TextEncoder(); @@ -51,6 +53,14 @@ interface SignOpts { tag?: string; tamper?: boolean; label?: string; + /** + * The Signature-Agent member value as serialized on the wire (an sf-string, + * optionally with parameters). Covered as `"signature-agent";key=