/** * Webhook delivery dispatcher. * * Invoked from the per-minute cron at /api/webhooks/dispatch/cron. Picks up * pending + retry-due deliveries (FOR UPDATE SKIP LOCKED so multiple cron * invocations don't double-deliver), POSTs each one with HMAC signature, * and updates the row to one of: * * - delivered (2xx response) — terminal * - failed (5xx / network / 4xx — non-terminal until attempts * other than 410) exhausted; bumps next_attempt_at * by exponential backoff * - dead (HTTP 410 OR — terminal * attempts exhausted) * * The receiver is expected to respond within 10 seconds; we time out * aggressively so a slow receiver doesn't block the per-minute cron. * * On HTTP 410 we additionally disable the webhook (sets disabled_at + * disabled_reason='HTTP 410 from receiver') so future events don't even * enqueue against it. */ import type { SupabaseClient } from '@supabase/supabase-js' import { signPayload } from './signing' import { pinnedHttpsFetch, type PinnedFetchResult } from './pinned-fetch' import { createLogger } from '@/lib/logger' const log = createLogger('webhooks/dispatcher') /** 7 retries over ~72h. Index = attempts BEFORE this one. */ const RETRY_BACKOFF_SECONDS: ReadonlyArray = [ 60, // 1m — first retry 5 * 60, // 5m 30 * 60, // 30m 2 * 60 * 60, // 2h 12 * 60 * 60, // 12h 24 * 60 * 60, // 24h 48 * 60 * 60, // 48h — final retry ] const MAX_ATTEMPTS = RETRY_BACKOFF_SECONDS.length + 1 // initial + 7 retries = 8 total const REQUEST_TIMEOUT_MS = 10_000 const MAX_RESPONSE_BODY_BYTES = 4096 interface DueDelivery { id: string webhook_id: string company_id: string event_type: string payload: Record previous_attributes: Record | null api_version: string attempts: number } interface WebhookForDelivery { id: string company_id: string webhook_url: string secret: string } export interface DispatchSummary { picked: number delivered: number failed: number dead: number } /** * Run one dispatch cycle. Picks up to `batchSize` due deliveries and * processes them sequentially (the per-minute cadence + small batch size * makes parallelism unnecessary; in-process serial is also gentler on the * receiver if many events fan out to the same URL). */ export async function dispatchDueDeliveries(args: { supabase: SupabaseClient /** Max rows to claim per cron tick. Default 50. */ batchSize?: number /** Override for tests. */ now?: Date /** Override for tests; injected pinned-fetch implementation. */ pinnedFetchImpl?: typeof pinnedHttpsFetch }): Promise { const batchSize = args.batchSize ?? 50 const now = args.now ?? new Date() const pinnedFetchImpl = args.pinnedFetchImpl ?? pinnedHttpsFetch const summary: DispatchSummary = { picked: 0, delivered: 0, failed: 0, dead: 0 } // Recover stuck in_flight rows: a previous tick that was killed mid-flight // (Vercel function timeout, hard crash, manual termination) leaves rows // marked in_flight forever otherwise. Sweep them back to 'failed' so the // retry loop picks them up at next_attempt_at. // // Threshold = 2× REQUEST_TIMEOUT_MS. A live attempt takes at most // REQUEST_TIMEOUT_MS plus the body read; doubling that gives an // unambiguous "this is stuck, not in-flight" boundary. await recoverStuckInFlight(args.supabase, now) const due = await claimDueDeliveries(args.supabase, batchSize, now) summary.picked = due.length if (due.length === 0) return summary // Dedupe webhook lookups within a single cycle. const webhookIds = Array.from(new Set(due.map((d) => d.webhook_id))) const webhookMap = await loadWebhooksByIds(args.supabase, webhookIds) for (const delivery of due) { const webhook = webhookMap.get(delivery.webhook_id) if (!webhook) { // The webhook was deleted between enqueue and dispatch. Mark dead; // there's no receiver to deliver to. The webhook_deliveries.webhook_id // FK is ON DELETE SET NULL (migration 20260515170000), so the row // stays in the audit trail under status='dead'. await markDead(args.supabase, delivery.id, 'webhook_deleted') summary.dead++ continue } // Defense-in-depth tenancy check: the webhook the delivery row points // at MUST belong to the same company as the delivery row. Mismatch // indicates a poisoned row — refuse to dispatch (which would sign with // the wrong tenant's secret and POST to the wrong receiver). if (webhook.company_id !== delivery.company_id) { log.error('cross-tenant delivery refused', new Error('company_id mismatch'), { deliveryId: delivery.id, deliveryCompanyId: delivery.company_id, webhookId: webhook.id, webhookCompanyId: webhook.company_id, }) await markDead(args.supabase, delivery.id, 'cross_tenant_mismatch') summary.dead++ continue } const outcome = await attemptDelivery({ delivery, webhook, pinnedFetchImpl, now, }) // Structured per-delivery outcome log. Keeps companyId / webhookId / // deliveryId available in log aggregation for per-tenant audit-trail // reconstruction without grepping through individual mark*-helper // writes (V16 — security event correlation). const logCtx = { deliveryId: delivery.id, webhookId: webhook.id, companyId: delivery.company_id, eventType: delivery.event_type, attempt: delivery.attempts + 1, } switch (outcome.kind) { case 'delivered': await markDelivered(args.supabase, delivery.id, outcome) log.info('delivery succeeded', { ...logCtx, responseStatus: outcome.responseStatus }) summary.delivered++ break case 'dead': await markDead(args.supabase, delivery.id, outcome.reason, outcome) log.warn('delivery dead', { ...logCtx, reason: outcome.reason, responseStatus: outcome.responseStatus }) summary.dead++ if (outcome.disableWebhook) { await disableWebhook(args.supabase, webhook.id, outcome.reason) log.warn('webhook auto-disabled', { ...logCtx, reason: outcome.reason }) } break case 'failed': if (delivery.attempts + 1 >= MAX_ATTEMPTS) { await markDead(args.supabase, delivery.id, 'attempts_exhausted', outcome) log.warn('delivery dead — attempts exhausted', { ...logCtx, lastError: outcome.error }) summary.dead++ } else { await markFailedForRetry(args.supabase, delivery.id, delivery.attempts, outcome, now) log.info('delivery failed — retry scheduled', { ...logCtx, error: outcome.error, responseStatus: outcome.responseStatus }) summary.failed++ } break } } return summary } // ────────────────────────────────────────────────────────────────────── // DB ops // ────────────────────────────────────────────────────────────────────── /** * Mark in_flight rows whose updated_at is older than the stuck-threshold * back to 'failed' with next_attempt_at = now so they re-enter the * dispatch queue. Best-effort — a write failure here is logged but * doesn't block the rest of the cycle. */ async function recoverStuckInFlight(supabase: SupabaseClient, now: Date): Promise { const stuckBefore = new Date(now.getTime() - 2 * REQUEST_TIMEOUT_MS) // Under READ COMMITTED (Postgres default), UPDATE re-evaluates the WHERE // clause against each row's current value when it acquires the row lock. // A row that raced from 'in_flight' to 'delivered'/'dead' between scan // and lock will fail status='in_flight' on re-evaluation and be skipped // entirely — the immutability trigger never fires, so a mid-flight // terminal flip cannot abort the bulk update. const { data, error } = await supabase .from('webhook_deliveries') .update({ status: 'failed', next_attempt_at: now.toISOString(), error: 'recovered_from_in_flight_timeout', }) .eq('status', 'in_flight') .lt('updated_at', stuckBefore.toISOString()) .select('id') if (error) { log.warn('stuck in_flight recovery failed', { code: error.code }) return } if (data && data.length > 0) { log.warn('recovered stuck in_flight rows', { count: data.length }) } } async function claimDueDeliveries( supabase: SupabaseClient, batchSize: number, now: Date, ): Promise { // Atomic FOR UPDATE SKIP LOCKED claim via the SQL function shipped in // migration 20260515220000. PostgREST can't express SKIP LOCKED through // the JS client, so the function form is the documented entry point — // see the migration comment for the full rationale (one round trip, // no CAS contention, rows locked by a concurrent tick are simply // invisible to the second caller). // // All filter semantics from the previous JS path are preserved inside // the function: status IN ('pending','failed'), next_attempt_at <= now, // webhook_id IS NOT NULL, ORDER BY next_attempt_at ASC, LIMIT batchSize. const { data, error } = await supabase.rpc('claim_due_webhook_deliveries', { p_batch_size: batchSize, p_now: now.toISOString(), }) if (error) { log.error('claim_due_webhook_deliveries rpc failed', error as Error) return [] } return (data ?? []) as DueDelivery[] } async function loadWebhooksByIds( supabase: SupabaseClient, ids: string[], ): Promise> { // Include company_id so the dispatch loop can assert that the delivery // row's company_id matches the webhook's — defense in depth against a // poisoned delivery row pointing at another tenant's webhook // (compromised service-role path, faulty INSERT in a future code path, // etc.). The DB trigger added in 20260515190000 enforces the same // invariant at INSERT time; this is the application-layer mirror. const { data, error } = await supabase .from('webhooks') .select('id, company_id, webhook_url, secret') .in('id', ids) if (error || !data) { log.error('webhook lookup for dispatch failed', error as Error) return new Map() } return new Map((data as WebhookForDelivery[]).map((w) => [w.id, w])) } async function markDelivered( supabase: SupabaseClient, id: string, outcome: DeliveredOutcome, ): Promise { const { error } = await supabase .from('webhook_deliveries') .update({ status: 'delivered', delivered_at: new Date().toISOString(), attempts: outcome.attempts, response_status: outcome.responseStatus, response_body: outcome.responseBody, response_headers: outcome.responseHeaders, error: null, }) .eq('id', id) if (error) log.warn('mark delivered update failed', { id, code: error.code }) } async function markFailedForRetry( supabase: SupabaseClient, id: string, priorAttempts: number, outcome: FailedOutcome, now: Date, ): Promise { const nextAttemptIndex = priorAttempts // 0-indexed lookup into RETRY_BACKOFF_SECONDS const backoffSeconds = RETRY_BACKOFF_SECONDS[Math.min(nextAttemptIndex, RETRY_BACKOFF_SECONDS.length - 1)] const nextAttemptAt = new Date(now.getTime() + backoffSeconds * 1000) const { error } = await supabase .from('webhook_deliveries') .update({ status: 'failed', attempts: priorAttempts + 1, next_attempt_at: nextAttemptAt.toISOString(), response_status: outcome.responseStatus ?? null, response_body: outcome.responseBody ?? null, response_headers: outcome.responseHeaders ?? null, error: outcome.error, }) .eq('id', id) if (error) log.warn('mark failed-for-retry update failed', { id, code: error.code }) } async function markDead( supabase: SupabaseClient, id: string, reason: string, outcome?: AttemptOutcome, ): Promise { // delivered_at means "the receiver acknowledged the event". For dead // rows (HTTP 410, attempts exhausted, webhook deleted, cross-tenant // mismatch, unsafe URL) the receiver did NOT acknowledge — leaving // delivered_at NULL keeps the audit semantics clean. An auditor // querying `WHERE delivered_at IS NOT NULL` correctly sees only // genuinely delivered rows. The terminal-state timestamp lives on // `updated_at` (auto-stamped by the table's BEFORE UPDATE trigger). const { error } = await supabase .from('webhook_deliveries') .update({ status: 'dead', attempts: outcome && 'attempts' in outcome ? outcome.attempts : undefined, response_status: outcome && 'responseStatus' in outcome ? outcome.responseStatus : null, response_body: outcome && 'responseBody' in outcome ? outcome.responseBody : null, response_headers: outcome && 'responseHeaders' in outcome ? outcome.responseHeaders : null, error: reason, }) .eq('id', id) if (error) log.warn('mark dead update failed', { id, code: error.code }) } async function disableWebhook( supabase: SupabaseClient, webhookId: string, reason: string, ): Promise { // Snapshot before the disable so the audit entry can record the prior // state. Service-role read; bypasses RLS. const { data: prior } = await supabase .from('webhooks') .select('user_id, company_id, name, active, disabled_at, disabled_reason') .eq('id', webhookId) .maybeSingle() const { error } = await supabase .from('webhooks') .update({ disabled_at: new Date().toISOString(), disabled_reason: reason, active: false, }) .eq('id', webhookId) if (error) { log.warn('webhook auto-disable failed', { webhookId, code: error.code }) return } // V16 security event log. Auto-disable is a privileged action taken by // the dispatcher (not a human caller), so actor_id is null. The // audit_log entry is written UNCONDITIONALLY — even when prior is null // or prior.user_id is null — because the SECURITY_EVENT must produce // a durable record (A.8.15 / V16.1.1 / CC7.2). The audit_log.user_id // column is nullable post-multi-tenant-refactor (20260330130000), so // a system-initiated event can legitimately write user_id=NULL. Such // rows are invisible under the user-scoped SELECT policy but remain // queryable under service-role review, which is appropriate for // system-initiated events. // // The reason discriminates between the three auto-disable paths // (http_410_gone / redirect_blocked / url_unsafe:) so SIEM // tooling can alert on systematic patterns. const p = prior as { user_id: string | null company_id: string | null name: string active: boolean disabled_at: string | null disabled_reason: string | null } | null const { error: auditErr } = await supabase.from('audit_log').insert({ user_id: p?.user_id ?? null, company_id: p?.company_id ?? null, action: 'SECURITY_EVENT', table_name: 'webhooks', record_id: webhookId, actor_id: null, description: p ? `Webhook auto-disabled by dispatcher: ${reason} (was "${p.name}")` : `Webhook auto-disabled by dispatcher: ${reason} (prior snapshot unavailable)`, old_state: p ? { active: p.active, disabled_at: p.disabled_at, disabled_reason: p.disabled_reason } : null, new_state: { active: false, disabled_reason: reason, disabled_at: new Date().toISOString() }, }) if (auditErr) { log.warn('audit_log insert failed for webhook auto-disable', { webhookId, reason, code: auditErr.code, }) } } // ────────────────────────────────────────────────────────────────────── // HTTP attempt // ────────────────────────────────────────────────────────────────────── type DeliveredOutcome = { kind: 'delivered' attempts: number responseStatus: number responseBody: string | null responseHeaders: Record | null } type FailedOutcome = { kind: 'failed' attempts: number responseStatus: number | null responseBody: string | null responseHeaders: Record | null error: string } type DeadOutcome = { kind: 'dead' reason: string disableWebhook: boolean attempts: number responseStatus: number | null responseBody: string | null responseHeaders: Record | null error?: string } type AttemptOutcome = DeliveredOutcome | FailedOutcome | DeadOutcome async function attemptDelivery(args: { delivery: DueDelivery webhook: WebhookForDelivery pinnedFetchImpl: typeof pinnedHttpsFetch now: Date }): Promise { const { delivery, webhook, pinnedFetchImpl, now } = args const attempts = delivery.attempts + 1 const requestId = `whdel_${delivery.id}` const body = JSON.stringify({ id: delivery.id, type: delivery.event_type, api_version: delivery.api_version, created: Math.floor(now.getTime() / 1000), data: { object: delivery.payload }, previous_attributes: delivery.previous_attributes, }) const { header } = signPayload({ body, secret: webhook.secret, timestamp: Math.floor(now.getTime() / 1000), }) // pinnedHttpsFetch performs DNS validation AND opens the socket against // the validated IP in a single call. The previous shape (separate // validateWebhookUrl + fetch calls) left a DNS-rebinding window between // the two — closed here. SNI + Host header continue to carry the // original hostname so receiver-side TLS + vhost routing still work. const result = await pinnedFetchImpl(webhook.webhook_url, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Gnubok-Signature': header, 'X-Gnubok-Event': delivery.event_type, 'X-Gnubok-Delivery': delivery.id, 'X-Gnubok-Api-Version': delivery.api_version, 'X-Request-Id': requestId, 'User-Agent': 'gnubok-webhook/1', }, body, timeoutMs: REQUEST_TIMEOUT_MS, maxResponseBytes: MAX_RESPONSE_BODY_BYTES, }) switch (result.kind) { case 'unsafe_url': return { kind: 'dead', reason: `url_unsafe:${result.reason}`, disableWebhook: true, attempts, responseStatus: null, responseBody: null, responseHeaders: null, error: result.detail, } case 'redirect_blocked': return { kind: 'dead', reason: 'redirect_blocked', disableWebhook: true, attempts, responseStatus: result.status, responseBody: null, responseHeaders: null, error: truncateError(result.detail), } case 'timeout': case 'transport_error': return { kind: 'failed', attempts, responseStatus: null, responseBody: null, responseHeaders: null, error: truncateError(result.detail), } case 'ok': { const responseHeaders = filterResponseHeaders(result.headers) const responseBody = isSafeContentType(result.headers['content-type'] ?? '') ? result.body : null // HTTP 410 — receiver explicitly asks us to stop. Auto-disable. if (result.status === 410) { return { kind: 'dead', reason: 'http_410_gone', disableWebhook: true, attempts, responseStatus: 410, responseBody, responseHeaders, } } if (result.status >= 200 && result.status < 300) { return { kind: 'delivered', attempts, responseStatus: result.status, responseBody, responseHeaders, } } return { kind: 'failed', attempts, responseStatus: result.status, responseBody, responseHeaders, error: `HTTP ${result.status}`, } } } } function truncateError(message: string): string { return message.length > 500 ? `${message.slice(0, 497)}...` : message } // Content-Type prefixes for which we persist response_body verbatim. Other // types (text/html error pages, application/octet-stream, ...) get dropped // because they routinely echo PII back from receiver-side error renderers // (Art.32(1)(b), A.8.12). A null body is just as useful for debugging // when the operator can see the response_status and response_headers. const SAFE_BODY_CONTENT_TYPE_PREFIXES = ['text/plain', 'application/json'] function isSafeContentType(contentType: string): boolean { const lower = contentType.toLowerCase() return SAFE_BODY_CONTENT_TYPE_PREFIXES.some((p) => lower.startsWith(p)) } // Allowlist for response_headers persistence. Receiver-side headers like // Set-Cookie, Authorization, WWW-Authenticate, internal tracing, and // vendor x-* headers can carry credentials or sensitive identifiers; we // don't need them for delivery diagnostics. (CC7.2 / Art.32(1)(b)) // // 'server' is deliberately NOT in the allowlist (A.8.12): it carries no // diagnostic value but routinely leaks receiver infrastructure version // strings (nginx/1.21.6, Apache/2.4.41, ...) into a multi-tenant audit // table. const SAFE_RESPONSE_HEADERS = new Set([ 'content-type', 'content-length', 'date', 'x-request-id', 'cf-ray', ]) function filterResponseHeaders(headers: Record): Record { const obj: Record = {} for (const [k, v] of Object.entries(headers)) { if (SAFE_RESPONSE_HEADERS.has(k.toLowerCase())) { obj[k] = v } } return obj } export const __TESTING__ = { RETRY_BACKOFF_SECONDS, MAX_ATTEMPTS, REQUEST_TIMEOUT_MS, MAX_RESPONSE_BODY_BYTES, }