diff --git a/.compliance/ropa.yaml b/.compliance/ropa.yaml index 09544fc7..664dfa62 100644 --- a/.compliance/ropa.yaml +++ b/.compliance/ropa.yaml @@ -662,3 +662,93 @@ processing_activities: - support_ticket_identity_hmac_signed # POSTHOG_SECRET_API_KEY, serverside - support_message_body_never_in_event_properties # breadcrumb bär endast subject - email_remains_the_delivery_guarantee # ärendet är ett komplement, ej leveransväg + + - id: whatsapp.receipt_intake + name: Kvittoinlämning och klargörande dialog via WhatsApp + purpose: >- + Ta emot kvitton som foto eller PDF skickade till Accounteds delade + WhatsApp-nummer och landa dem i dokumentinkorgen (Underlag) som + räkenskapsinformation, samt föra en kort klargörande dialog i samma + chatt: bekräfta mottagning, fråga vilket företag underlaget tillhör och + samla in de representationsuppgifter Skatteverket kräver (deltagarnas + namn och företag samt syfte). Avsändare är enbart befintliga + Accounted-användare som själva knutit sitt nummer med en engångskod; + okända nummer får ett enda begränsat standardsvar och behandlas aldrig + vidare (ingen AI, ingen mediahämtning, inget innehåll sparas). + Deltagarnamn i representationssvar är tredjepartsuppgifter som samlas + in från användaren, inte från de registrerade själva; individuell + information till varje deltagare vore oproportionerlig i förhållande + till den lagstadgade dokumentationsplikten och undantaget i art. + 14.5 b tillämpas (uppgifterna framgår av verifikationen, som bevaras + enligt BFL). + # art 6.1 b: tjänsten (mottagning, dialog, inkorgsleverans). + # art 6.1 c: BFL 7 kap: kvittot bevaras 7 år som räkenskapsinformation. + # art 6.1 f: tredjeparts deltagarnamn i representationssvar (med + # art 14.5 b-undantaget dokumenterat i purpose ovan). + lawful_basis: art_6_1_b_c_and_f + special_category_basis: null + controller: gnubok-tenant + processor: supabase_and_meta + data_subjects: + - business_owner + - company_member + - representation_attendee # tredjepart: namn + företag i representationssvar + data_categories: + - user.contact.phone_number # HMAC-hash med peppar + AES-256-GCM + mask; aldrig i klartext i DB + - user.communication_content # chattext och rå webhook-payload, 90 dagars retention + - user.document # kvittofiler, 7 år WORM enligt BFL 7 kap + - user.name # deltagarnamn i representationssvar (tredjepart) + - user.activity_timestamp + recipients: + - name: Meta Platforms Ireland Ltd (WhatsApp Business Platform, Cloud API) + country: IE + role: processor + - name: Supabase + country: EU + role: processor + # AI-tolkningen av kvittobilder och fritextsvar sker via Amazon Bedrock + # och täcks av den befintliga aktiviteten ai.inference ovan. + international_transfers: + applicable: true + mechanism: scc_2021_c2p + note: >- + Meta Platforms Ireland behandlar meddelandena under WhatsApp Business + dataskyddsvillkor med EU SCC-addendum (Module 2) för vidareöverföring + till Meta Platforms Inc. (US). Cloud API Local Storage är aktiverat + med region DE, så meddelandeinnehåll lagras i vila inom EU hos Meta. + retention: + # Differentierad per datakategori; verkställs dagligen av + # /api/extensions/whatsapp-inbox/retention/cron: + # chattranskript (body_text, raw_payload) 90 dagar, sedan rensas + # innehållet men radskelettet (wamid, riktning, tidsstämplar, + # status, inbox-koppling) bevaras som behandlingsspår + # okända avsändares rader 30 dagar, sedan radering + # engångskoder 24 h efter utgång, radering + # telefonkoppling (phone_enc) tills användaren återkallar, + # därefter kryptoshredning 90 dagar efter revokering (phone_hash och + # phone_masked bevaras för unikhetshistorik respektive visning) + # kvittofilen 7 år, WORM (BFL 7 kap), + # hanteras av dokumentarkivet, ej av denna cron + duration: differentiated_per_data_category + basis: gdpr_storage_limitation_and_bfl_7_kap + stored_in: + - whatsapp_messages + - whatsapp_phone_links + - whatsapp_link_codes + - whatsapp_sender_rate_counters + - whatsapp_conversations + - invoice_inbox_items.channel_context + - document_attachments # kvittofilen, WORM + security_measures: + - phone_number_hmac_peppered_hash_lookup_only + - phone_number_aes_256_gcm_at_rest + - phone_number_masked_in_ui + - dedicated_whatsapp_key_material_not_bankid_keys + - webhook_signature_verified_over_raw_body + - service_role_only_tables_rls_no_policies + - unknown_senders_no_content_persistence_no_llm + - pre_binding_sender_rate_limits + - daily_retention_cron_with_crypto_shred + - llm_reads_answers_as_untrusted_data_no_tools + - receipt_files_worm_protected_bfl + - opt_out_stop_keyword_honored diff --git a/DECISIONS.md b/DECISIONS.md index 9211c5d4..135a7028 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -784,3 +784,4 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-08-02] whatsapp-inbox PR4 re-send (M9): a sharper re-sent file creates a NEW inbox item and marks the old one channel_context.quality.superseded=true instead of swapping the old item's document. Replacing in place has no supported path (attach-document 409s when document_id is set) and would orphan the original WORM document against the anchored-doc invariant (see 2026-07-27 floating-underlag work). [2026-08-02] whatsapp-inbox PR4 schema: ONE new migration (20260802210000) adds whatsapp_messages.acked_at. Burst-ack membership (which ingested rows the single combined M4/M5 covered) must be derivable relationally across webhook invocations; deriving it from "last outbound ack timestamp" breaks the moment one outbound insert fails, and conversation-context accumulation races. [2026-08-02] whatsapp-inbox M6 body omits merchant/amount (spec showed them): for multi-company senders the item is created only after the company answer, so nothing is extracted when the question is asked. Body degrades to "Vilket företag gäller kvittot/kvittona?". +[2026-08-02] whatsapp-inbox PR5b retention: revoked links crypto-shred by setting phone_enc='' (column is NOT NULL; empty string is the cleared marker, guarded by neq for one-shot idempotency) while phone_hash and phone_masked survive on purpose (re-link uniqueness history + audit display). Scoped to whatsapp_* tables only: inbox_rate_counters (the per-company sibling) has NO cleanup anywhere and stays that way here; adding one is a separate change, noted in the PR body. diff --git a/app/api/extensions/whatsapp-inbox/retention/cron/__tests__/route.test.ts b/app/api/extensions/whatsapp-inbox/retention/cron/__tests__/route.test.ts new file mode 100644 index 00000000..b0a10aab --- /dev/null +++ b/app/api/extensions/whatsapp-inbox/retention/cron/__tests__/route.test.ts @@ -0,0 +1,97 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { NextResponse } from 'next/server' + +vi.mock('@/lib/extensions/loader', () => ({ + loadExtensions: vi.fn(), +})) + +vi.mock('@/lib/extensions/registry', () => ({ + extensionRegistry: { + get: vi.fn(), + }, +})) + +vi.mock('@/lib/auth/api-keys', () => ({ + createServiceClientNoCookies: vi.fn().mockReturnValue({}), +})) + +vi.mock('@/extensions/general/whatsapp-inbox/lib/retention', () => ({ + runRetention: vi.fn(), +})) + +vi.mock('@/lib/auth/cron', () => ({ + verifyCronSecret: vi.fn().mockReturnValue(null), +})) + +import { GET } from '../route' +import { extensionRegistry } from '@/lib/extensions/registry' +import { loadExtensions } from '@/lib/extensions/loader' +import { runRetention } from '@/extensions/general/whatsapp-inbox/lib/retention' +import { verifyCronSecret } from '@/lib/auth/cron' + +const mockRegistryGet = vi.mocked(extensionRegistry.get) +const mockVerifyCronSecret = vi.mocked(verifyCronSecret) +const mockRunRetention = vi.mocked(runRetention) + +function makeRequest() { + return new Request('http://localhost/api/extensions/whatsapp-inbox/retention/cron', { + headers: { authorization: 'Bearer synthetic-cron-secret' }, + }) +} + +beforeEach(() => { + vi.clearAllMocks() + mockVerifyCronSecret.mockReturnValue(null) +}) + +describe('GET /api/extensions/whatsapp-inbox/retention/cron', () => { + it('returns 401 when the cron secret is rejected', async () => { + mockVerifyCronSecret.mockReturnValue( + NextResponse.json({ error: 'Unauthorized' }, { status: 401 }), + ) + + const response = await GET(makeRequest()) + + expect(response.status).toBe(401) + expect(mockRunRetention).not.toHaveBeenCalled() + }) + + it('returns 503 EXTENSION_DISABLED when the extension is not in the registry', async () => { + // Physical extension routes deploy in every build; the registry, generated + // from extensions.config.json, is what turns them on. Disabled must mean + // no purging AND a visible failure if the cron is scheduled anyway. + mockRegistryGet.mockReturnValue(undefined) + + const response = await GET(makeRequest()) + const body = await response.json() + + expect(response.status).toBe(503) + expect(body.code).toBe('EXTENSION_DISABLED') + expect(mockRunRetention).not.toHaveBeenCalled() + }) + + it('runs the retention pass and returns its summary when enabled', async () => { + mockRegistryGet.mockReturnValue({ id: 'whatsapp-inbox' } as never) + mockRunRetention.mockResolvedValue({ + purgedTranscripts: 12, + deletedUnknownSenderMessages: 4, + deletedLinkCodes: 2, + deletedRateCounters: 9, + shreddedRevokedLinks: 1, + }) + + const response = await GET(makeRequest()) + const body = await response.json() + + expect(loadExtensions).toHaveBeenCalled() + expect(mockRegistryGet).toHaveBeenCalledWith('whatsapp-inbox') + expect(response.status).toBe(200) + expect(body.data).toEqual({ + purgedTranscripts: 12, + deletedUnknownSenderMessages: 4, + deletedLinkCodes: 2, + deletedRateCounters: 9, + shreddedRevokedLinks: 1, + }) + }) +}) diff --git a/app/api/extensions/whatsapp-inbox/retention/cron/route.ts b/app/api/extensions/whatsapp-inbox/retention/cron/route.ts new file mode 100644 index 00000000..9394dfc4 --- /dev/null +++ b/app/api/extensions/whatsapp-inbox/retention/cron/route.ts @@ -0,0 +1,48 @@ +import { NextResponse } from 'next/server' +import { loadExtensions } from '@/lib/extensions/loader' +import { extensionRegistry } from '@/lib/extensions/registry' +import { withCronContext } from '@/lib/api/with-cron-context' +import { createServiceClientNoCookies } from '@/lib/auth/api-keys' +import { runRetention } from '@/extensions/general/whatsapp-inbox/lib/retention' + +/** + * GET /api/extensions/whatsapp-inbox/retention/cron: daily GDPR purge for + * the WhatsApp channel. Enforces the retention table in .compliance/ropa.yaml + * (whatsapp.receipt_intake): 90-day chat transcripts, 30-day unknown-sender + * rows, expired link codes, dead rate counters, and the 90-day crypto-shred + * of revoked phone bindings. Receipts themselves are 7-year WORM under BFL + * and are never touched here. Scheduled in vercel.json (and the generated + * Docker crontabs). + * + * Overlap with a slow previous run is safe: every predicate only matches + * rows still carrying the data, so a second pass is a no-op. + */ + +// The transcript purge loops over a possibly large first-run backlog. +export const maxDuration = 300 + +export const GET = withCronContext('cron.whatsapp_retention', async (_request, ctx) => { + // Load the registry so it reflects extensions.config.json. + loadExtensions() + + // Physical routes under app/api/extensions// compile into EVERY build, + // including the core-with-zero-extensions one: the registry (generated from + // extensions.config.json) is what actually switches an extension on. Mirror + // the ext/[...path] dispatcher: a disabled extension must not expose a live + // surface, and a scheduled-but-disabled cron must fail visibly (503) + // instead of quietly doing the work anyway. + if (!extensionRegistry.get('whatsapp-inbox')) { + ctx.log.warn('whatsapp-inbox extension is not enabled; cron refused') + return NextResponse.json( + { error: 'WhatsApp inbox extension is not enabled', code: 'EXTENSION_DISABLED' }, + { status: 503 }, + ) + } + + const supabase = createServiceClientNoCookies() + const summary = await runRetention(supabase) + + ctx.log.info('whatsapp retention complete', { ...summary }) + + return NextResponse.json({ data: summary }) +}) diff --git a/docker/crontab.hosted b/docker/crontab.hosted index 12bf41ab..8dfcc472 100644 --- a/docker/crontab.hosted +++ b/docker/crontab.hosted @@ -40,4 +40,5 @@ 30 */2 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/skatteverket/vat/kvittenser/cron * * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/webhooks/dispatch/cron * * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/sweep/cron +15 4 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/retention/cron 15 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/bookkeeping/accruals/post-due/cron diff --git a/docker/crontab.self-hosted b/docker/crontab.self-hosted index 50172612..c62e52a5 100644 --- a/docker/crontab.self-hosted +++ b/docker/crontab.self-hosted @@ -40,4 +40,5 @@ 30 */2 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/skatteverket/vat/kvittenser/cron * * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/webhooks/dispatch/cron * * * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/sweep/cron +15 4 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/extensions/whatsapp-inbox/retention/cron 15 5 * * * curl -sf -H "Authorization: Bearer ${CRON_SECRET}" ${APP_URL}/api/bookkeeping/accruals/post-due/cron diff --git a/extensions/general/whatsapp-inbox/__tests__/retention.test.ts b/extensions/general/whatsapp-inbox/__tests__/retention.test.ts new file mode 100644 index 00000000..55449c42 --- /dev/null +++ b/extensions/general/whatsapp-inbox/__tests__/retention.test.ts @@ -0,0 +1,209 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { createQueuedMockSupabase } from '@/tests/helpers' +import type { SupabaseClient } from '@supabase/supabase-js' +import { + BATCH, + LINK_CODE_GRACE_HOURS, + RATE_COUNTER_RETENTION_DAYS, + REVOKED_LINK_SHRED_DAYS, + TRANSCRIPT_RETENTION_DAYS, + UNKNOWN_SENDER_RETENTION_DAYS, + runRetention, +} from '@/extensions/general/whatsapp-inbox/lib/retention' + +const DAY_MS = 24 * 60 * 60 * 1000 +const HOUR_MS = 60 * 60 * 1000 + +function daysAgo(days: number): string { + return new Date(Date.now() - days * DAY_MS).toISOString() +} + +function hoursAgo(hours: number): string { + return new Date(Date.now() - hours * HOUR_MS).toISOString() +} + +/** The recorded cutoff must sit within a minute of `now - expectedMs`. */ +function expectCutoffAt(cutoffIso: unknown, expectedMs: number) { + expect(typeof cutoffIso).toBe('string') + const drift = Math.abs(Date.now() - expectedMs - new Date(cutoffIso as string).getTime()) + expect(drift).toBeLessThan(60 * 1000) +} + +/** + * Enqueue one full no-op pass in execution order: + * unknown-sender delete, transcript select (empty ends the loop), + * link codes, rate counters, revoked-link shred. + */ +function enqueueEmptyRun( + enqueue: (r: { data?: unknown; error?: unknown; count?: number | null }) => void, + overrides: Partial< + Record<'unknown' | 'select' | 'codes' | 'counters' | 'shred', { + data?: unknown + error?: unknown + count?: number | null + }> + > = {}, +) { + enqueue(overrides.unknown ?? { data: null, count: 0 }) + enqueue(overrides.select ?? { data: [] }) + enqueue(overrides.codes ?? { data: null, count: 0 }) + enqueue(overrides.counters ?? { data: null, count: 0 }) + enqueue(overrides.shred ?? { data: null, count: 0 }) +} + +describe('runRetention', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('purges transcript content at the 90-day boundary: 89-day row untouched, 91-day row purged', async () => { + const { supabase, enqueue, findCalls, calls } = createQueuedMockSupabase() + enqueue({ data: null, count: 0 }) // unknown-sender delete + // The DB filter (created_at < cutoff) returns only the 91-day row. + enqueue({ data: [{ id: 'm-91d' }] }) // transcript select, batch 1 + enqueue({ data: null }) // transcript update, batch 1 (partial batch ends loop) + enqueue({ data: null, count: 0 }) // link codes + enqueue({ data: null, count: 0 }) // counters + enqueue({ data: null, count: 0 }) // shred + + const summary = await runRetention(supabase as unknown as SupabaseClient) + + // Second lt() on whatsapp_messages belongs to the transcript select + // (the first is the unknown-sender delete's 30-day cutoff). + const ltCalls = findCalls('whatsapp_messages', 'lt') + const [, cutoff] = ltCalls[1] as [string, string] + expect(ltCalls[1][0]).toBe('created_at') + expectCutoffAt(cutoff, TRANSCRIPT_RETENTION_DAYS * DAY_MS) + // Predicate boundary: an 89-day row does NOT satisfy created_at < cutoff, + // a 91-day row does. + expect(daysAgo(89) < cutoff).toBe(false) + expect(daysAgo(91) < cutoff).toBe(true) + + // Idempotency: only rows still carrying content are selected. + const orCall = calls.find((c) => c.table === 'whatsapp_messages' && c.method === 'or') + expect(orCall?.args[0]).toBe('body_text.not.is.null,raw_payload.not.is.null') + + // The purge NULLs content only; the row skeleton survives. + const patch = findCalls('whatsapp_messages', 'update')[0][0] as Record + expect(patch).toEqual({ body_text: null, raw_payload: null }) + const inCall = calls.find((c) => c.table === 'whatsapp_messages' && c.method === 'in') + expect(inCall?.args).toEqual(['id', ['m-91d']]) + + expect(summary.purgedTranscripts).toBe(1) + }) + + it('loops the transcript purge in batches until a partial batch', async () => { + const { supabase, enqueue, findCalls } = createQueuedMockSupabase() + const fullBatch = Array.from({ length: BATCH }, (_, i) => ({ id: `m-${i}` })) + enqueue({ data: null, count: 0 }) // unknown-sender delete + enqueue({ data: fullBatch }) // select, batch 1 (full -> loop again) + enqueue({ data: null }) // update, batch 1 + enqueue({ data: [{ id: 'm-last' }] }) // select, batch 2 (partial -> stop) + enqueue({ data: null }) // update, batch 2 + enqueue({ data: null, count: 0 }) // link codes + enqueue({ data: null, count: 0 }) // counters + enqueue({ data: null, count: 0 }) // shred + + const summary = await runRetention(supabase as unknown as SupabaseClient) + + expect(findCalls('whatsapp_messages', 'update')).toHaveLength(2) + expect(summary.purgedTranscripts).toBe(BATCH + 1) + }) + + it('deletes unknown-sender rows at the 30-day boundary, never linked rows', async () => { + const { supabase, enqueue, findCalls, calls } = createQueuedMockSupabase() + enqueueEmptyRun(enqueue, { unknown: { data: null, count: 3 } }) + + const summary = await runRetention(supabase as unknown as SupabaseClient) + + const isCall = calls.find((c) => c.table === 'whatsapp_messages' && c.method === 'is') + expect(isCall?.args).toEqual(['phone_link_id', null]) + + const [column, cutoff] = findCalls('whatsapp_messages', 'lt')[0] as [string, string] + expect(column).toBe('created_at') + expectCutoffAt(cutoff, UNKNOWN_SENDER_RETENTION_DAYS * DAY_MS) + expect(daysAgo(29) < cutoff).toBe(false) + expect(daysAgo(31) < cutoff).toBe(true) + + expect(summary.deletedUnknownSenderMessages).toBe(3) + }) + + it('deletes link codes 24h after expiry, used or not', async () => { + const { supabase, enqueue, findCalls } = createQueuedMockSupabase() + enqueueEmptyRun(enqueue, { codes: { data: null, count: 2 } }) + + const summary = await runRetention(supabase as unknown as SupabaseClient) + + expect(findCalls('whatsapp_link_codes', 'delete')).toHaveLength(1) + const [column, cutoff] = findCalls('whatsapp_link_codes', 'lt')[0] as [string, string] + expect(column).toBe('expires_at') + expectCutoffAt(cutoff, LINK_CODE_GRACE_HOURS * HOUR_MS) + // No used_at filter: used and unused codes go alike. + expect(hoursAgo(23) < cutoff).toBe(false) + expect(hoursAgo(25) < cutoff).toBe(true) + + expect(summary.deletedLinkCodes).toBe(2) + }) + + it('deletes rate counters idle for more than 2 days', async () => { + const { supabase, enqueue, findCalls } = createQueuedMockSupabase() + enqueueEmptyRun(enqueue, { counters: { data: null, count: 5 } }) + + const summary = await runRetention(supabase as unknown as SupabaseClient) + + expect(findCalls('whatsapp_sender_rate_counters', 'delete')).toHaveLength(1) + const [column, cutoff] = findCalls('whatsapp_sender_rate_counters', 'lt')[0] as [ + string, + string, + ] + expect(column).toBe('updated_at') + expectCutoffAt(cutoff, RATE_COUNTER_RETENTION_DAYS * DAY_MS) + + expect(summary.deletedRateCounters).toBe(5) + }) + + it('crypto-shreds revoked links only after 90 days, only once, keeping hash and mask', async () => { + const { supabase, enqueue, findCalls, calls } = createQueuedMockSupabase() + enqueueEmptyRun(enqueue, { shred: { data: null, count: 1 } }) + + const summary = await runRetention(supabase as unknown as SupabaseClient) + + const [patch, options] = findCalls('whatsapp_phone_links', 'update')[0] as [ + Record, + Record, + ] + // phone_enc is NOT NULL in the schema: '' is the cleared marker. The + // patch must never touch phone_hash (uniqueness history) or phone_masked + // (audit display). + expect(patch).toEqual({ phone_enc: '' }) + expect(options).toEqual({ count: 'exact' }) + + const [column, cutoff] = findCalls('whatsapp_phone_links', 'lt')[0] as [string, string] + expect(column).toBe('revoked_at') + expectCutoffAt(cutoff, REVOKED_LINK_SHRED_DAYS * DAY_MS) + // Boundary: revoked 89 days ago stays readable, 91 days ago is shredded. + expect(daysAgo(89) < cutoff).toBe(false) + expect(daysAgo(91) < cutoff).toBe(true) + // Only once: already-cleared rows are excluded by the neq guard, so a + // re-run never re-touches (or re-counts) them. + const neqCall = calls.find((c) => c.table === 'whatsapp_phone_links' && c.method === 'neq') + expect(neqCall?.args).toEqual(['phone_enc', '']) + + expect(summary.shreddedRevokedLinks).toBe(1) + }) + + it('isolates failures: one failing action never blocks the others', async () => { + const { supabase, enqueue } = createQueuedMockSupabase() + enqueueEmptyRun(enqueue, { + unknown: { error: { message: 'boom' } }, + codes: { data: null, count: 4 }, + shred: { data: null, count: 2 }, + }) + + const summary = await runRetention(supabase as unknown as SupabaseClient) + + expect(summary.deletedUnknownSenderMessages).toBe(0) + expect(summary.deletedLinkCodes).toBe(4) + expect(summary.shreddedRevokedLinks).toBe(2) + }) +}) diff --git a/extensions/general/whatsapp-inbox/lib/retention.ts b/extensions/general/whatsapp-inbox/lib/retention.ts new file mode 100644 index 00000000..1253bce8 --- /dev/null +++ b/extensions/general/whatsapp-inbox/lib/retention.ts @@ -0,0 +1,168 @@ +/** + * Daily GDPR retention purge for the WhatsApp channel. + * + * The receipt itself is 7-year WORM räkenskapsinformation (BFL 7 kap) and is + * untouched here; everything conversational around it is short-lived. Each + * pass enforces the retention table documented in .compliance/ropa.yaml + * (whatsapp.receipt_intake): + * + * 1. Chat transcripts (body_text + raw_payload on whatsapp_messages) are + * purged after 90 days. The row skeleton survives for audit: wamid, + * direction, timestamps, processing_status and inbox_item_id keep the + * "a message existed and produced this Underlag row" trail without the + * content. + * 2. Rows with no phone_link_id (unknown senders, plus rows orphaned by a + * link deletion) are DELETED after 30 days: pre-binding traffic has no + * contractual basis to keep, only the abuse-throttle window. + * 3. Link codes are deleted 24h after expiry, used or not. The 10-minute + * TTL plus single-use flag already made them dead; this removes the + * hashes entirely. + * 4. Sender rate counters older than 2 days are deleted: window keys are + * minute/day, so anything older can never be read again. + * 5. Revoked phone links are crypto-shredded 90 days after revocation: + * phone_enc is cleared to '' (the column is NOT NULL, so empty string is + * the cleared marker). phone_hash stays (uniqueness history: the same + * phone re-linking later must still be resolvable) and phone_masked + * stays (display in any audit surface). + * + * Every action is idempotent (predicates only match rows still carrying the + * data) and isolated in its own try/catch: one failing table never blocks + * the purge of another. Volume-unbounded actions loop in id batches under a + * shared wall-clock budget, mirroring the sweep's stance that a partial pass + * is a latency regression, never a correctness problem: tomorrow's run picks + * up the remainder. + */ + +import type { SupabaseClient } from '@supabase/supabase-js' +import { createLogger } from '@/lib/logger' + +const log = createLogger('whatsapp-inbox/retention') + +const DAY_MS = 24 * 60 * 60 * 1000 +const HOUR_MS = 60 * 60 * 1000 + +export const TRANSCRIPT_RETENTION_DAYS = 90 +export const UNKNOWN_SENDER_RETENTION_DAYS = 30 +export const LINK_CODE_GRACE_HOURS = 24 +export const RATE_COUNTER_RETENTION_DAYS = 2 +export const REVOKED_LINK_SHRED_DAYS = 90 + +export const BATCH = 200 +/** maxDuration on the cron route is 300s; leave headroom for the response. */ +const TIME_BUDGET_MS = 4 * 60 * 1000 + +export interface RetentionSummary { + purgedTranscripts: number + deletedUnknownSenderMessages: number + deletedLinkCodes: number + deletedRateCounters: number + shreddedRevokedLinks: number +} + +/** Run one retention pass. Never throws. */ +export async function runRetention(supabase: SupabaseClient): Promise { + const summary: RetentionSummary = { + purgedTranscripts: 0, + deletedUnknownSenderMessages: 0, + deletedLinkCodes: 0, + deletedRateCounters: 0, + shreddedRevokedLinks: 0, + } + const startedAt = Date.now() + const budgetLeft = () => Date.now() - startedAt < TIME_BUDGET_MS + + // ── 1. Unknown-sender rows past 30 days: DELETE ───────────── + // Runs before the transcript purge so a row that is both link-less and + // past 90 days is deleted outright instead of being content-purged first. + // Also catches outbound greeting rows (persistOutbound stores them with + // phone_link_id null) and rows orphaned by ON DELETE SET NULL. + try { + const cutoff = new Date(startedAt - UNKNOWN_SENDER_RETENTION_DAYS * DAY_MS).toISOString() + const { count, error } = await supabase + .from('whatsapp_messages') + .delete({ count: 'exact' }) + .is('phone_link_id', null) + .lt('created_at', cutoff) + if (error) throw error + summary.deletedUnknownSenderMessages = count ?? 0 + } catch (err) { + log.error('retention: unknown-sender delete failed', err) + } + + // ── 2. Transcripts past 90 days: purge content, keep skeleton ── + // Batched: the first pass over a backlog can touch many rows, and an + // unbounded UPDATE risks the statement timeout. Only rows still carrying + // content match, so re-runs never churn already-purged rows. + try { + const cutoff = new Date(startedAt - TRANSCRIPT_RETENTION_DAYS * DAY_MS).toISOString() + while (budgetLeft()) { + const { data, error } = await supabase + .from('whatsapp_messages') + .select('id') + .lt('created_at', cutoff) + .or('body_text.not.is.null,raw_payload.not.is.null') + .order('created_at', { ascending: true }) + .limit(BATCH) + if (error) throw error + const ids = ((data ?? []) as { id: string }[]).map((row) => row.id) + if (ids.length === 0) break + const { error: updateError } = await supabase + .from('whatsapp_messages') + .update({ body_text: null, raw_payload: null }) + .in('id', ids) + if (updateError) throw updateError + summary.purgedTranscripts += ids.length + if (ids.length < BATCH) break + } + } catch (err) { + log.error('retention: transcript purge failed', err) + } + + // ── 3. Link codes expired more than 24h ago: DELETE ───────── + try { + const cutoff = new Date(startedAt - LINK_CODE_GRACE_HOURS * HOUR_MS).toISOString() + const { count, error } = await supabase + .from('whatsapp_link_codes') + .delete({ count: 'exact' }) + .lt('expires_at', cutoff) + if (error) throw error + summary.deletedLinkCodes = count ?? 0 + } catch (err) { + log.error('retention: link-code cleanup failed', err) + } + + // ── 4. Rate counters idle for 2+ days: DELETE ─────────────── + // Window keys are minute/day; a counter not touched for 2 days can never + // be incremented or read again. (inbox_rate_counters, the per-company + // sibling, has no equivalent cleanup anywhere yet; ours starts clean.) + try { + const cutoff = new Date(startedAt - RATE_COUNTER_RETENTION_DAYS * DAY_MS).toISOString() + const { count, error } = await supabase + .from('whatsapp_sender_rate_counters') + .delete({ count: 'exact' }) + .lt('updated_at', cutoff) + if (error) throw error + summary.deletedRateCounters = count ?? 0 + } catch (err) { + log.error('retention: rate-counter cleanup failed', err) + } + + // ── 5. Links revoked 90+ days ago: crypto-shred phone_enc ─── + // phone_enc is NOT NULL, so '' is the cleared marker; the neq guard makes + // the shred one-shot. phone_hash and phone_masked survive on purpose (see + // module docblock). + try { + const cutoff = new Date(startedAt - REVOKED_LINK_SHRED_DAYS * DAY_MS).toISOString() + const { count, error } = await supabase + .from('whatsapp_phone_links') + .update({ phone_enc: '' }, { count: 'exact' }) + .lt('revoked_at', cutoff) + .neq('phone_enc', '') + if (error) throw error + summary.shreddedRevokedLinks = count ?? 0 + } catch (err) { + log.error('retention: revoked-link shred failed', err) + } + + return summary +} diff --git a/vercel.json b/vercel.json index 352e5618..1094f3ae 100644 --- a/vercel.json +++ b/vercel.json @@ -69,6 +69,10 @@ "path": "/api/extensions/whatsapp-inbox/sweep/cron", "schedule": "* * * * *" }, + { + "path": "/api/extensions/whatsapp-inbox/retention/cron", + "schedule": "15 4 * * *" + }, { "path": "/api/bookkeeping/accruals/post-due/cron", "schedule": "15 5 * * *"