feat(whatsapp-inbox): GDPR retention cron and RoPA entry (#1341)

PR5b, the final code piece of the WhatsApp intake track. A daily cron (04:15) enforces the channel's retention table; the receipt itself stays 7-year WORM under BFL and is never touched. Retention actions (lib/retention.ts, each isolated and idempotent): - whatsapp_messages transcripts past 90 days: body_text + raw_payload cleared in id batches under a wall-clock budget; the row skeleton (wamid, direction, timestamps, status, inbox_item_id) survives for audit. Only rows still carrying content match. - Rows with phone_link_id IS NULL (unknown senders, orphans) past 30 days: deleted. - Link codes expired more than 24h ago: deleted, used or not. - Sender rate counters idle 2+ days: deleted (minute/day window keys are dead weight after that). - Links revoked 90+ days ago: phone_enc crypto-shredded to '' (column is NOT NULL), one-shot via neq guard; phone_hash and phone_masked kept for uniqueness history and audit display. Route mirrors the sweep cron exactly: withCronContext + registry gate (503 EXTENSION_DISABLED when the extension is off). vercel.json gets the schedule and both Docker crontabs are regenerated. Compliance: new whatsapp.receipt_intake activity in .compliance/ropa.yaml covering purpose, Art 6(1)(b)/(c)/(f) bases with the Art 14(5)(b) note for third-party attendee names, Meta Platforms Ireland as processor (Cloud API, EU SCC addendum, Local Storage region DE), the differentiated retention table, and security measures. Co-authored-by: Jakob Wennberg <jakob.wennberg@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-05 15:35:41 +02:00
parent 629069e281
commit bf5ca2c615
9 changed files with 619 additions and 0 deletions
@@ -0,0 +1,209 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { createQueuedMockSupabase } from '@/tests/helpers'
+import type { SupabaseClient } from '@supabase/supabase-js'
+import {
+  BATCH,
+  LINK_CODE_GRACE_HOURS,
+  RATE_COUNTER_RETENTION_DAYS,
+  REVOKED_LINK_SHRED_DAYS,
+  TRANSCRIPT_RETENTION_DAYS,
+  UNKNOWN_SENDER_RETENTION_DAYS,
+  runRetention,
+} from '@/extensions/general/whatsapp-inbox/lib/retention'
+
+const DAY_MS = 24 * 60 * 60 * 1000
+const HOUR_MS = 60 * 60 * 1000
+
+function daysAgo(days: number): string {
+  return new Date(Date.now() - days * DAY_MS).toISOString()
+}
+
+function hoursAgo(hours: number): string {
+  return new Date(Date.now() - hours * HOUR_MS).toISOString()
+}
+
+/** The recorded cutoff must sit within a minute of `now - expectedMs`. */
+function expectCutoffAt(cutoffIso: unknown, expectedMs: number) {
+  expect(typeof cutoffIso).toBe('string')
+  const drift = Math.abs(Date.now() - expectedMs - new Date(cutoffIso as string).getTime())
+  expect(drift).toBeLessThan(60 * 1000)
+}
+
+/**
+ * Enqueue one full no-op pass in execution order:
+ * unknown-sender delete, transcript select (empty ends the loop),
+ * link codes, rate counters, revoked-link shred.
+ */
+function enqueueEmptyRun(
+  enqueue: (r: { data?: unknown; error?: unknown; count?: number | null }) => void,
+  overrides: Partial<
+    Record<'unknown' | 'select' | 'codes' | 'counters' | 'shred', {
+      data?: unknown
+      error?: unknown
+      count?: number | null
+    }>
+  > = {},
+) {
+  enqueue(overrides.unknown ?? { data: null, count: 0 })
+  enqueue(overrides.select ?? { data: [] })
+  enqueue(overrides.codes ?? { data: null, count: 0 })
+  enqueue(overrides.counters ?? { data: null, count: 0 })
+  enqueue(overrides.shred ?? { data: null, count: 0 })
+}
+
+describe('runRetention', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('purges transcript content at the 90-day boundary: 89-day row untouched, 91-day row purged', async () => {
+    const { supabase, enqueue, findCalls, calls } = createQueuedMockSupabase()
+    enqueue({ data: null, count: 0 }) // unknown-sender delete
+    // The DB filter (created_at < cutoff) returns only the 91-day row.
+    enqueue({ data: [{ id: 'm-91d' }] }) // transcript select, batch 1
+    enqueue({ data: null }) // transcript update, batch 1 (partial batch ends loop)
+    enqueue({ data: null, count: 0 }) // link codes
+    enqueue({ data: null, count: 0 }) // counters
+    enqueue({ data: null, count: 0 }) // shred
+
+    const summary = await runRetention(supabase as unknown as SupabaseClient)
+
+    // Second lt() on whatsapp_messages belongs to the transcript select
+    // (the first is the unknown-sender delete's 30-day cutoff).
+    const ltCalls = findCalls('whatsapp_messages', 'lt')
+    const [, cutoff] = ltCalls[1] as [string, string]
+    expect(ltCalls[1][0]).toBe('created_at')
+    expectCutoffAt(cutoff, TRANSCRIPT_RETENTION_DAYS * DAY_MS)
+    // Predicate boundary: an 89-day row does NOT satisfy created_at < cutoff,
+    // a 91-day row does.
+    expect(daysAgo(89) < cutoff).toBe(false)
+    expect(daysAgo(91) < cutoff).toBe(true)
+
+    // Idempotency: only rows still carrying content are selected.
+    const orCall = calls.find((c) => c.table === 'whatsapp_messages' && c.method === 'or')
+    expect(orCall?.args[0]).toBe('body_text.not.is.null,raw_payload.not.is.null')
+
+    // The purge NULLs content only; the row skeleton survives.
+    const patch = findCalls('whatsapp_messages', 'update')[0][0] as Record<string, unknown>
+    expect(patch).toEqual({ body_text: null, raw_payload: null })
+    const inCall = calls.find((c) => c.table === 'whatsapp_messages' && c.method === 'in')
+    expect(inCall?.args).toEqual(['id', ['m-91d']])
+
+    expect(summary.purgedTranscripts).toBe(1)
+  })
+
+  it('loops the transcript purge in batches until a partial batch', async () => {
+    const { supabase, enqueue, findCalls } = createQueuedMockSupabase()
+    const fullBatch = Array.from({ length: BATCH }, (_, i) => ({ id: `m-${i}` }))
+    enqueue({ data: null, count: 0 }) // unknown-sender delete
+    enqueue({ data: fullBatch }) // select, batch 1 (full -> loop again)
+    enqueue({ data: null }) // update, batch 1
+    enqueue({ data: [{ id: 'm-last' }] }) // select, batch 2 (partial -> stop)
+    enqueue({ data: null }) // update, batch 2
+    enqueue({ data: null, count: 0 }) // link codes
+    enqueue({ data: null, count: 0 }) // counters
+    enqueue({ data: null, count: 0 }) // shred
+
+    const summary = await runRetention(supabase as unknown as SupabaseClient)
+
+    expect(findCalls('whatsapp_messages', 'update')).toHaveLength(2)
+    expect(summary.purgedTranscripts).toBe(BATCH + 1)
+  })
+
+  it('deletes unknown-sender rows at the 30-day boundary, never linked rows', async () => {
+    const { supabase, enqueue, findCalls, calls } = createQueuedMockSupabase()
+    enqueueEmptyRun(enqueue, { unknown: { data: null, count: 3 } })
+
+    const summary = await runRetention(supabase as unknown as SupabaseClient)
+
+    const isCall = calls.find((c) => c.table === 'whatsapp_messages' && c.method === 'is')
+    expect(isCall?.args).toEqual(['phone_link_id', null])
+
+    const [column, cutoff] = findCalls('whatsapp_messages', 'lt')[0] as [string, string]
+    expect(column).toBe('created_at')
+    expectCutoffAt(cutoff, UNKNOWN_SENDER_RETENTION_DAYS * DAY_MS)
+    expect(daysAgo(29) < cutoff).toBe(false)
+    expect(daysAgo(31) < cutoff).toBe(true)
+
+    expect(summary.deletedUnknownSenderMessages).toBe(3)
+  })
+
+  it('deletes link codes 24h after expiry, used or not', async () => {
+    const { supabase, enqueue, findCalls } = createQueuedMockSupabase()
+    enqueueEmptyRun(enqueue, { codes: { data: null, count: 2 } })
+
+    const summary = await runRetention(supabase as unknown as SupabaseClient)
+
+    expect(findCalls('whatsapp_link_codes', 'delete')).toHaveLength(1)
+    const [column, cutoff] = findCalls('whatsapp_link_codes', 'lt')[0] as [string, string]
+    expect(column).toBe('expires_at')
+    expectCutoffAt(cutoff, LINK_CODE_GRACE_HOURS * HOUR_MS)
+    // No used_at filter: used and unused codes go alike.
+    expect(hoursAgo(23) < cutoff).toBe(false)
+    expect(hoursAgo(25) < cutoff).toBe(true)
+
+    expect(summary.deletedLinkCodes).toBe(2)
+  })
+
+  it('deletes rate counters idle for more than 2 days', async () => {
+    const { supabase, enqueue, findCalls } = createQueuedMockSupabase()
+    enqueueEmptyRun(enqueue, { counters: { data: null, count: 5 } })
+
+    const summary = await runRetention(supabase as unknown as SupabaseClient)
+
+    expect(findCalls('whatsapp_sender_rate_counters', 'delete')).toHaveLength(1)
+    const [column, cutoff] = findCalls('whatsapp_sender_rate_counters', 'lt')[0] as [
+      string,
+      string,
+    ]
+    expect(column).toBe('updated_at')
+    expectCutoffAt(cutoff, RATE_COUNTER_RETENTION_DAYS * DAY_MS)
+
+    expect(summary.deletedRateCounters).toBe(5)
+  })
+
+  it('crypto-shreds revoked links only after 90 days, only once, keeping hash and mask', async () => {
+    const { supabase, enqueue, findCalls, calls } = createQueuedMockSupabase()
+    enqueueEmptyRun(enqueue, { shred: { data: null, count: 1 } })
+
+    const summary = await runRetention(supabase as unknown as SupabaseClient)
+
+    const [patch, options] = findCalls('whatsapp_phone_links', 'update')[0] as [
+      Record<string, unknown>,
+      Record<string, unknown>,
+    ]
+    // phone_enc is NOT NULL in the schema: '' is the cleared marker. The
+    // patch must never touch phone_hash (uniqueness history) or phone_masked
+    // (audit display).
+    expect(patch).toEqual({ phone_enc: '' })
+    expect(options).toEqual({ count: 'exact' })
+
+    const [column, cutoff] = findCalls('whatsapp_phone_links', 'lt')[0] as [string, string]
+    expect(column).toBe('revoked_at')
+    expectCutoffAt(cutoff, REVOKED_LINK_SHRED_DAYS * DAY_MS)
+    // Boundary: revoked 89 days ago stays readable, 91 days ago is shredded.
+    expect(daysAgo(89) < cutoff).toBe(false)
+    expect(daysAgo(91) < cutoff).toBe(true)
+    // Only once: already-cleared rows are excluded by the neq guard, so a
+    // re-run never re-touches (or re-counts) them.
+    const neqCall = calls.find((c) => c.table === 'whatsapp_phone_links' && c.method === 'neq')
+    expect(neqCall?.args).toEqual(['phone_enc', ''])
+
+    expect(summary.shreddedRevokedLinks).toBe(1)
+  })
+
+  it('isolates failures: one failing action never blocks the others', async () => {
+    const { supabase, enqueue } = createQueuedMockSupabase()
+    enqueueEmptyRun(enqueue, {
+      unknown: { error: { message: 'boom' } },
+      codes: { data: null, count: 4 },
+      shred: { data: null, count: 2 },
+    })
+
+    const summary = await runRetention(supabase as unknown as SupabaseClient)
+
+    expect(summary.deletedUnknownSenderMessages).toBe(0)
+    expect(summary.deletedLinkCodes).toBe(4)
+    expect(summary.shreddedRevokedLinks).toBe(2)
+  })
+})
@@ -0,0 +1,168 @@
+/**
+ * Daily GDPR retention purge for the WhatsApp channel.
+ *
+ * The receipt itself is 7-year WORM räkenskapsinformation (BFL 7 kap) and is
+ * untouched here; everything conversational around it is short-lived. Each
+ * pass enforces the retention table documented in .compliance/ropa.yaml
+ * (whatsapp.receipt_intake):
+ *
+ *  1. Chat transcripts (body_text + raw_payload on whatsapp_messages) are
+ *     purged after 90 days. The row skeleton survives for audit: wamid,
+ *     direction, timestamps, processing_status and inbox_item_id keep the
+ *     "a message existed and produced this Underlag row" trail without the
+ *     content.
+ *  2. Rows with no phone_link_id (unknown senders, plus rows orphaned by a
+ *     link deletion) are DELETED after 30 days: pre-binding traffic has no
+ *     contractual basis to keep, only the abuse-throttle window.
+ *  3. Link codes are deleted 24h after expiry, used or not. The 10-minute
+ *     TTL plus single-use flag already made them dead; this removes the
+ *     hashes entirely.
+ *  4. Sender rate counters older than 2 days are deleted: window keys are
+ *     minute/day, so anything older can never be read again.
+ *  5. Revoked phone links are crypto-shredded 90 days after revocation:
+ *     phone_enc is cleared to '' (the column is NOT NULL, so empty string is
+ *     the cleared marker). phone_hash stays (uniqueness history: the same
+ *     phone re-linking later must still be resolvable) and phone_masked
+ *     stays (display in any audit surface).
+ *
+ * Every action is idempotent (predicates only match rows still carrying the
+ * data) and isolated in its own try/catch: one failing table never blocks
+ * the purge of another. Volume-unbounded actions loop in id batches under a
+ * shared wall-clock budget, mirroring the sweep's stance that a partial pass
+ * is a latency regression, never a correctness problem: tomorrow's run picks
+ * up the remainder.
+ */
+
+import type { SupabaseClient } from '@supabase/supabase-js'
+import { createLogger } from '@/lib/logger'
+
+const log = createLogger('whatsapp-inbox/retention')
+
+const DAY_MS = 24 * 60 * 60 * 1000
+const HOUR_MS = 60 * 60 * 1000
+
+export const TRANSCRIPT_RETENTION_DAYS = 90
+export const UNKNOWN_SENDER_RETENTION_DAYS = 30
+export const LINK_CODE_GRACE_HOURS = 24
+export const RATE_COUNTER_RETENTION_DAYS = 2
+export const REVOKED_LINK_SHRED_DAYS = 90
+
+export const BATCH = 200
+/** maxDuration on the cron route is 300s; leave headroom for the response. */
+const TIME_BUDGET_MS = 4 * 60 * 1000
+
+export interface RetentionSummary {
+  purgedTranscripts: number
+  deletedUnknownSenderMessages: number
+  deletedLinkCodes: number
+  deletedRateCounters: number
+  shreddedRevokedLinks: number
+}
+
+/** Run one retention pass. Never throws. */
+export async function runRetention(supabase: SupabaseClient): Promise<RetentionSummary> {
+  const summary: RetentionSummary = {
+    purgedTranscripts: 0,
+    deletedUnknownSenderMessages: 0,
+    deletedLinkCodes: 0,
+    deletedRateCounters: 0,
+    shreddedRevokedLinks: 0,
+  }
+  const startedAt = Date.now()
+  const budgetLeft = () => Date.now() - startedAt < TIME_BUDGET_MS
+
+  // ── 1. Unknown-sender rows past 30 days: DELETE ─────────────
+  // Runs before the transcript purge so a row that is both link-less and
+  // past 90 days is deleted outright instead of being content-purged first.
+  // Also catches outbound greeting rows (persistOutbound stores them with
+  // phone_link_id null) and rows orphaned by ON DELETE SET NULL.
+  try {
+    const cutoff = new Date(startedAt - UNKNOWN_SENDER_RETENTION_DAYS * DAY_MS).toISOString()
+    const { count, error } = await supabase
+      .from('whatsapp_messages')
+      .delete({ count: 'exact' })
+      .is('phone_link_id', null)
+      .lt('created_at', cutoff)
+    if (error) throw error
+    summary.deletedUnknownSenderMessages = count ?? 0
+  } catch (err) {
+    log.error('retention: unknown-sender delete failed', err)
+  }
+
+  // ── 2. Transcripts past 90 days: purge content, keep skeleton ──
+  // Batched: the first pass over a backlog can touch many rows, and an
+  // unbounded UPDATE risks the statement timeout. Only rows still carrying
+  // content match, so re-runs never churn already-purged rows.
+  try {
+    const cutoff = new Date(startedAt - TRANSCRIPT_RETENTION_DAYS * DAY_MS).toISOString()
+    while (budgetLeft()) {
+      const { data, error } = await supabase
+        .from('whatsapp_messages')
+        .select('id')
+        .lt('created_at', cutoff)
+        .or('body_text.not.is.null,raw_payload.not.is.null')
+        .order('created_at', { ascending: true })
+        .limit(BATCH)
+      if (error) throw error
+      const ids = ((data ?? []) as { id: string }[]).map((row) => row.id)
+      if (ids.length === 0) break
+      const { error: updateError } = await supabase
+        .from('whatsapp_messages')
+        .update({ body_text: null, raw_payload: null })
+        .in('id', ids)
+      if (updateError) throw updateError
+      summary.purgedTranscripts += ids.length
+      if (ids.length < BATCH) break
+    }
+  } catch (err) {
+    log.error('retention: transcript purge failed', err)
+  }
+
+  // ── 3. Link codes expired more than 24h ago: DELETE ─────────
+  try {
+    const cutoff = new Date(startedAt - LINK_CODE_GRACE_HOURS * HOUR_MS).toISOString()
+    const { count, error } = await supabase
+      .from('whatsapp_link_codes')
+      .delete({ count: 'exact' })
+      .lt('expires_at', cutoff)
+    if (error) throw error
+    summary.deletedLinkCodes = count ?? 0
+  } catch (err) {
+    log.error('retention: link-code cleanup failed', err)
+  }
+
+  // ── 4. Rate counters idle for 2+ days: DELETE ───────────────
+  // Window keys are minute/day; a counter not touched for 2 days can never
+  // be incremented or read again. (inbox_rate_counters, the per-company
+  // sibling, has no equivalent cleanup anywhere yet; ours starts clean.)
+  try {
+    const cutoff = new Date(startedAt - RATE_COUNTER_RETENTION_DAYS * DAY_MS).toISOString()
+    const { count, error } = await supabase
+      .from('whatsapp_sender_rate_counters')
+      .delete({ count: 'exact' })
+      .lt('updated_at', cutoff)
+    if (error) throw error
+    summary.deletedRateCounters = count ?? 0
+  } catch (err) {
+    log.error('retention: rate-counter cleanup failed', err)
+  }
+
+  // ── 5. Links revoked 90+ days ago: crypto-shred phone_enc ───
+  // phone_enc is NOT NULL, so '' is the cleared marker; the neq guard makes
+  // the shred one-shot. phone_hash and phone_masked survive on purpose (see
+  // module docblock).
+  try {
+    const cutoff = new Date(startedAt - REVOKED_LINK_SHRED_DAYS * DAY_MS).toISOString()
+    const { count, error } = await supabase
+      .from('whatsapp_phone_links')
+      .update({ phone_enc: '' }, { count: 'exact' })
+      .lt('revoked_at', cutoff)
+      .neq('phone_enc', '')
+    if (error) throw error
+    summary.shreddedRevokedLinks = count ?? 0
+  } catch (err) {
+    log.error('retention: revoked-link shred failed', err)
+  }
+
+  return summary
+}