* fix(whatsapp): app-side unmute, close silent intake paths, health visibility - add POST /link/unmute and a Reactivate control on the Pausad state - company resolution: transient query errors release the row for sweep retry; genuine zero-options sends M19 instead of parking silently - media from unlinked senders bypasses the hourly greeting throttle (10 min burst window, daily cap kept) - GET /link returns 7-day failed-delivery and parked-inbound counts; sweep summary logs outboundFailed24h Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(documents): real HEIC/HEIF magic-byte validation, bilingual upload errors - detect ISO-BMFF ftyp brands (heic/heix/heim/heis/hevc/hevx/hevm/hevs, mif1/msf1) instead of exempting image/heic from validation; declared heic/heif accepts either family member (iOS labels vary) - new INBOX_UPLOAD_* structured error codes replace raw English strings on the inbox upload and attach-document routes - registry doc corrected to the real 10 MB cap Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(inbox): staged upload with instant ack and deferred AI extraction - web uploads insert the inbox item as status processing and respond immediately; Bedrock extraction and supplier match run via after() with a CAS flip to received (email and WhatsApp channels keep the synchronous path) - widen invoice_inbox_items.status CHECK to include processing (migration 20260813180000, pg-real test included) - crash-recovery sweep cron (*/2) flips stale processing rows; bulk-book skips extraction_in_progress items - workspace: processing chip, in-flight rows disable actions, realtime flip, retry-extraction button for empty extractions - picker accept list drops HEIC/HEIF so iOS transcodes library photos to JPEG; server allowlists unchanged (supersedes 2026-08-01 HEIC decision, see DECISIONS.md) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(migrations): bump inbox processing-status migration past main's latest Main merged 20260813210000 while this PR was in flight; an inserted version older than the latest applied aborts the prod db push at merge. Renamed 20260813180000 to 20260813213000 and updated references. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * docs(decisions): log preview-tracker orphan repair after migration rename Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
694 lines
27 KiB
TypeScript
694 lines
27 KiB
TypeScript
import { after } from 'next/server'
|
||
import { uploadDocument } from '@/lib/core/documents/document-service'
|
||
import { extractInvoiceFields, emptyResult } from './extract-invoice-fields'
|
||
import { hasCapability } from '@/lib/entitlements/has-capability'
|
||
import { CAPABILITY } from '@/lib/entitlements/keys'
|
||
import { appendProcessingHistory } from '@/lib/processing-history/append'
|
||
import { createServiceClientNoCookies } from '@/lib/auth/api-keys'
|
||
import type { InvoiceExtractionResult } from '@/types'
|
||
import { PDFDocument } from 'pdf-lib'
|
||
import path from 'node:path'
|
||
|
||
/**
|
||
* Defensive filename sanitisation for content arriving from .eml inner
|
||
* attachments and rejected-attachment metadata. The document-service already
|
||
* sanitises before storage paths are built (lib/core/documents/document-service.ts),
|
||
* so this is defense-in-depth: strip directory traversal sequences and exotic
|
||
* characters before they ever flow into DB columns or downstream consumers.
|
||
*/
|
||
export function sanitiseFilename(raw: string | null | undefined, fallback: string): string {
|
||
const base = path.basename(String(raw ?? '').trim())
|
||
const cleaned = base.replace(/[^\w.-]/g, '_').slice(0, 200)
|
||
return cleaned || fallback
|
||
}
|
||
|
||
export function sanitiseMime(raw: string | null | undefined): string {
|
||
const value = String(raw ?? '').trim().slice(0, 120)
|
||
return /^[\w./+-]+$/.test(value) ? value : 'application/octet-stream'
|
||
}
|
||
|
||
export const MAX_FILE_SIZE = 10 * 1024 * 1024
|
||
|
||
// AI extraction is tuned for single-page receipts/invoices. Documents above
|
||
// this page count tend to be sales reports, bank statements, or contracts:
|
||
// Bedrock churns for minutes and still extracts nothing useful (issue #553).
|
||
// Above the limit we skip extraction entirely; the document still lands in
|
||
// the inbox and can be attached to a transaction or converted manually.
|
||
export const MAX_PAGES_FOR_AUTO_EXTRACT = 3
|
||
|
||
// Returns the page count for a PDF buffer, or null if the buffer isn't a
|
||
// parseable PDF. Errors fall through so callers can treat "unknown" the same
|
||
// as "small enough": preserves today's behavior on malformed inputs.
|
||
export async function countPdfPages(buffer: ArrayBuffer): Promise<number | null> {
|
||
try {
|
||
const pdf = await PDFDocument.load(buffer, { updateMetadata: false })
|
||
return pdf.getPageCount()
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
// Long PDFs used to skip extraction entirely (issue #553). Invoice data
|
||
// almost always sits on the first page(s), so instead we extract from a
|
||
// slim copy of the first MAX_PAGES_FOR_AUTO_EXTRACT pages and record the
|
||
// truncation in extracted_data.pages. Returns null when slicing fails
|
||
// (encrypted/malformed PDF) so the caller can fall back to the old skip.
|
||
export async function slicePdfForExtraction(
|
||
buffer: ArrayBuffer,
|
||
maxPages: number
|
||
): Promise<ArrayBuffer | null> {
|
||
try {
|
||
const src = await PDFDocument.load(buffer, { updateMetadata: false })
|
||
const dst = await PDFDocument.create()
|
||
const pages = await dst.copyPages(
|
||
src,
|
||
Array.from({ length: Math.min(maxPages, src.getPageCount()) }, (_, i) => i)
|
||
)
|
||
for (const page of pages) dst.addPage(page)
|
||
const bytes = await dst.save()
|
||
// Copy into a fresh ArrayBuffer: Uint8Array.buffer is ArrayBufferLike
|
||
// (possibly SharedArrayBuffer-backed) and may span more than the view.
|
||
const out = new ArrayBuffer(bytes.byteLength)
|
||
new Uint8Array(out).set(bytes)
|
||
return out
|
||
} catch {
|
||
return null
|
||
}
|
||
}
|
||
|
||
// Sandbox companies (24h anonymous demo accounts) skip the Bedrock extraction
|
||
// pipeline entirely. The document still uploads, the inbox row still lands,
|
||
// and the user can fill the fields in by hand, but no Claude tokens are
|
||
// spent on a throwaway account. See migration 20260311120000 for the column.
|
||
export async function isSandboxCompany(
|
||
supabase: import('@supabase/supabase-js').SupabaseClient,
|
||
companyId: string,
|
||
): Promise<boolean> {
|
||
const { data, error } = await supabase
|
||
.from('company_settings')
|
||
.select('is_sandbox')
|
||
.eq('company_id', companyId)
|
||
.maybeSingle()
|
||
if (error || !data) return false
|
||
return data.is_sandbox === true
|
||
}
|
||
|
||
export const UPLOAD_ALLOWED_MIME_TYPES = new Set([
|
||
'application/pdf',
|
||
'image/jpeg',
|
||
'image/png',
|
||
'image/heic',
|
||
'image/heif',
|
||
'image/webp',
|
||
])
|
||
|
||
// The email pipeline additionally accepts HTML: many suppliers send the
|
||
// invoice as the mail body or an attached .html file, and for those the mail
|
||
// IS the underlag (BFL: the received form must be preserved). Deliberately
|
||
// NOT added to UPLOAD_ALLOWED_MIME_TYPES: HTML only enters through the
|
||
// inbound webhook, where the preview surfaces render it fully sandboxed.
|
||
export const EMAIL_ALLOWED_MIME_TYPES = new Set([...UPLOAD_ALLOWED_MIME_TYPES, 'text/html'])
|
||
|
||
function escapeHtml(raw: string): string {
|
||
return raw
|
||
.replace(/&/g, '&')
|
||
.replace(/</g, '<')
|
||
.replace(/>/g, '>')
|
||
.replace(/"/g, '"')
|
||
.replace(/'/g, ''')
|
||
}
|
||
|
||
/**
|
||
* Wrap HTML in a minimal document shell unless it already is a full document.
|
||
* Mail bodies are usually fragments (a bare <div>/<table>); the document
|
||
* archive should hold self-contained files, and the document-service
|
||
* magic-byte check requires HTML content to start with a doctype/root element.
|
||
*/
|
||
export function ensureHtmlDocument(html: string): ArrayBuffer {
|
||
const head = html.replace(/^[\s]+/, '').slice(0, 256).toLowerCase()
|
||
const isFullDocument =
|
||
head.startsWith('<!doctype html') || head.startsWith('<html') || head.startsWith('<?xml')
|
||
const full = isFullDocument
|
||
? html
|
||
: `<!doctype html>\n<html>\n<head><meta charset="utf-8"></head>\n<body>\n${html}\n</body>\n</html>\n`
|
||
const bytes = new TextEncoder().encode(full)
|
||
// Copy into a fresh ArrayBuffer: Uint8Array.buffer is ArrayBufferLike
|
||
// (possibly SharedArrayBuffer-backed) and may span more than the view.
|
||
const out = new ArrayBuffer(bytes.byteLength)
|
||
new Uint8Array(out).set(bytes)
|
||
return out
|
||
}
|
||
|
||
/**
|
||
* Build a text/html document from a mail's body parts, preferring the HTML
|
||
* part. A plain-text-only mail is escaped into a <pre> so whitespace, amounts
|
||
* and OCR numbers survive verbatim. Returns null when the mail has no body
|
||
* worth storing.
|
||
*/
|
||
export function buildEmailBodyHtmlDocument(
|
||
html: string | null,
|
||
text: string | null
|
||
): ArrayBuffer | null {
|
||
if (html?.trim()) return ensureHtmlDocument(html)
|
||
if (text?.trim()) return ensureHtmlDocument(`<pre>${escapeHtml(text)}</pre>`)
|
||
return null
|
||
}
|
||
|
||
export interface EmailMeta {
|
||
from?: string | null
|
||
subject?: string | null
|
||
receivedAt?: string | null
|
||
messageId?: string | null
|
||
bodyText?: string | null
|
||
resendEmailId?: string | null
|
||
resendAttachmentId?: string | null
|
||
}
|
||
|
||
// Chat-channel provenance (whatsapp-inbox extension). When present, the inbox
|
||
// row links back to the delivering whatsapp_messages row and seeds
|
||
// channel_context (kept OUT of extracted_data: retry-extraction overwrites
|
||
// that container wholesale, and chat context must survive it).
|
||
export interface ChannelMeta {
|
||
whatsappMessageId?: string
|
||
caption?: string | null
|
||
}
|
||
|
||
// Captions are attacker-adjacent free text from a chat client: strip control
|
||
// characters and cap length before they land in a jsonb column read by the UI.
|
||
function sanitiseCaption(raw: string | null | undefined): string | null {
|
||
if (!raw) return null
|
||
// eslint-disable-next-line no-control-regex
|
||
const cleaned = raw.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '').trim().slice(0, 500)
|
||
return cleaned || null
|
||
}
|
||
|
||
// ── Shared helper: upload + extract + create inbox item ──────
|
||
|
||
export async function uploadAndExtract(
|
||
supabase: import('@supabase/supabase-js').SupabaseClient,
|
||
userId: string,
|
||
companyId: string,
|
||
file: { name: string; buffer: ArrayBuffer; type: string },
|
||
source: 'upload' | 'email' | 'whatsapp',
|
||
emailMeta?: EmailMeta,
|
||
// Pre-match the new inbox item to a bank transaction. Set when the caller
|
||
// already knows which transaction this receipt belongs to (e.g. the
|
||
// VerifyAndBookOverlay opened from a transaction row's paperclip or from
|
||
// a transaction-anchored chat). Skipped silently if missing.
|
||
matchedTransactionId?: string | null,
|
||
opts: {
|
||
skipExtraction?: boolean
|
||
channelMeta?: ChannelMeta
|
||
/** Overrides the system actor id on the DocumentIngested history event.
|
||
* Omitted = today's behavior (resend-inbound for email, user otherwise). */
|
||
actorId?: string
|
||
/**
|
||
* Staged upload (web route only). When true AND extraction would actually
|
||
* call Bedrock, the inbox row is inserted first (status 'processing',
|
||
* extracted_data NULL), the function returns immediately, and extraction
|
||
* runs after the response via a deferred worker that CAS-flips the row to
|
||
* 'received'. Verdicts that never reach Bedrock (no AI entitlement,
|
||
* sandbox, client opt-out) stay on the synchronous path: they are quick
|
||
* and their response contract (empty skeleton + skip_reason) is
|
||
* unchanged. skipExtraction in particular MUST stay synchronous: a
|
||
* BYO-extraction agent PUTs its fields right after upload, and a deferred
|
||
* flip would overwrite them. Default false = today's synchronous behavior
|
||
* (email and WhatsApp callers are untouched).
|
||
*/
|
||
deferExtraction?: boolean
|
||
} = {},
|
||
) {
|
||
const correlationId = crypto.randomUUID()
|
||
|
||
const doc = await uploadDocument(supabase, userId, companyId, {
|
||
name: file.name,
|
||
buffer: file.buffer,
|
||
type: file.type,
|
||
}, {
|
||
upload_source: source === 'email' ? 'email' : source === 'whatsapp' ? 'whatsapp' : 'file_upload',
|
||
// Every inbox channel dedupes on content: the same receipt forwarded to
|
||
// two inboxes, re-hunted by a sweep, or uploaded twice must not become a
|
||
// second archived document.
|
||
dedupeByContent: true,
|
||
})
|
||
|
||
if (doc.deduplicated) {
|
||
// The company already archived this exact content. If an inbox item
|
||
// exists for it, adopt that item so the caller always gets a real
|
||
// inbox_item_id; only when the document entered outside the inbox do we
|
||
// fall through and file an item for the EXISTING document (no copy).
|
||
const { data: existingItem, error: itemLookupError } = await supabase
|
||
.from('invoice_inbox_items')
|
||
.select('id, status, extracted_data, matched_supplier_id, matched_transaction_id')
|
||
.eq('company_id', companyId)
|
||
.eq('document_id', doc.id)
|
||
.order('created_at', { ascending: true })
|
||
.limit(1)
|
||
if (itemLookupError) {
|
||
// Fail closed: falling through would file a second item for a document
|
||
// that may already carry one. The delivering channel retries.
|
||
throw new Error(`Duplicate-item lookup failed: ${itemLookupError.message}`)
|
||
}
|
||
const adopted = (existingItem as Array<{
|
||
id: string
|
||
status: string
|
||
extracted_data: unknown
|
||
matched_supplier_id: string | null
|
||
matched_transaction_id: string | null
|
||
}> | null)?.[0]
|
||
if (adopted) {
|
||
try {
|
||
await appendProcessingHistory({
|
||
companyId,
|
||
correlationId,
|
||
aggregateType: 'Document',
|
||
aggregateId: doc.id,
|
||
eventType: 'DocumentDuplicateSkipped',
|
||
payload: {
|
||
channel: source,
|
||
document_id: doc.id,
|
||
inbox_item_id: adopted.id,
|
||
reason: 'duplicate_content',
|
||
},
|
||
actor: opts.actorId
|
||
? { type: 'system', id: opts.actorId }
|
||
: source === 'email'
|
||
? { type: 'system', id: 'resend-inbound' }
|
||
: { type: 'user', id: userId },
|
||
occurredAt: new Date(),
|
||
})
|
||
} catch (err) {
|
||
console.error('[invoice-inbox] Failed to append DocumentDuplicateSkipped:', err)
|
||
}
|
||
return {
|
||
document_id: doc.id,
|
||
inbox_item_id: adopted.id,
|
||
status: adopted.status,
|
||
extracted_data: adopted.extracted_data,
|
||
matched_supplier_id: adopted.matched_supplier_id,
|
||
matched_transaction_id: adopted.matched_transaction_id,
|
||
extraction_skipped: true,
|
||
skip_reason: 'duplicate_content' as const,
|
||
page_count: null,
|
||
duplicate: true as const,
|
||
}
|
||
}
|
||
}
|
||
|
||
try {
|
||
await appendProcessingHistory({
|
||
companyId,
|
||
correlationId,
|
||
aggregateType: 'Document',
|
||
aggregateId: doc.id,
|
||
eventType: 'DocumentIngested',
|
||
payload: {
|
||
channel: source,
|
||
document_id: doc.id,
|
||
mime_type: file.type,
|
||
size_bytes: file.buffer.byteLength,
|
||
},
|
||
actor: opts.actorId
|
||
? { type: 'system', id: opts.actorId }
|
||
: source === 'email'
|
||
? { type: 'system', id: 'resend-inbound' }
|
||
: { type: 'user', id: userId },
|
||
occurredAt: new Date(),
|
||
})
|
||
} catch (err) {
|
||
console.error('[invoice-inbox] Failed to append DocumentIngested:', err)
|
||
}
|
||
|
||
// Page-count gate (issue #553): PDFs above MAX_PAGES_FOR_AUTO_EXTRACT
|
||
// skip extraction. Bedrock would otherwise block the upload response for
|
||
// minutes on a 6-page sales report and return nothing useful. Images and
|
||
// non-PDFs are never gated (single-page by definition). countPdfPages
|
||
// returns null on malformed PDFs: we treat null as "not gated" and fall
|
||
// through to the existing extraction path so today's behavior is preserved.
|
||
const pageCount =
|
||
file.type === 'application/pdf' ? await countPdfPages(file.buffer) : null
|
||
const gatedByPageCount =
|
||
pageCount != null && pageCount > MAX_PAGES_FOR_AUTO_EXTRACT
|
||
const sandbox = await isSandboxCompany(supabase, companyId)
|
||
// Paid-tier gate: AI document OCR (Bedrock, via extractInvoiceFields) is the
|
||
// `ai` capability. A company without it (free/manual tier) must never trigger
|
||
// paid extraction: we seed an empty skeleton exactly like the sandbox / BYO-
|
||
// extraction path, so the document is still stored and can be filled in
|
||
// manually. Highest priority (a hard paywall rule, not a heuristic).
|
||
const hasAiEntitlement = await hasCapability(supabase, companyId, CAPABILITY.ai)
|
||
// The verdicts that avoid Bedrock entirely, in the established priority
|
||
// order: no-AI-entitlement > sandbox > client opt-out. All three are
|
||
// decidable without touching the PDF; too_many_pages is not (it only fires
|
||
// when the slice fallback fails) and is resolved below, on whichever path
|
||
// (sync or deferred) actually attempts the slice.
|
||
const syncSkipReason: 'no_ai_entitlement' | 'client_opt_out' | 'sandbox' | null =
|
||
!hasAiEntitlement
|
||
? 'no_ai_entitlement'
|
||
: sandbox
|
||
? 'sandbox'
|
||
: opts.skipExtraction
|
||
? 'client_opt_out'
|
||
: null
|
||
|
||
if (opts.deferExtraction && syncSkipReason === null) {
|
||
// Staged path: extraction WILL call Bedrock, so create the row now and
|
||
// let the response go. The deferred worker (or, after a crash, the sweep
|
||
// cron at /api/extensions/invoice-inbox/sweep/cron) flips the row to
|
||
// 'received'. extracted_data stays NULL until then: that is what keeps
|
||
// the MCP create-from-inbox guard ("re-run extraction first") holding
|
||
// for free on in-flight rows.
|
||
const { data: inbox, error: inboxError } = await supabase
|
||
.from('invoice_inbox_items')
|
||
.insert({
|
||
company_id: companyId,
|
||
user_id: userId,
|
||
status: 'processing',
|
||
source,
|
||
document_id: doc.id,
|
||
extracted_data: null,
|
||
extraction_skipped: false,
|
||
matched_supplier_id: null,
|
||
email_from: emailMeta?.from || null,
|
||
email_subject: emailMeta?.subject || null,
|
||
email_received_at: emailMeta?.receivedAt || null,
|
||
email_body_text: emailMeta?.bodyText || null,
|
||
resend_email_id: emailMeta?.resendEmailId || null,
|
||
resend_attachment_id: emailMeta?.resendAttachmentId || null,
|
||
raw_email_payload: emailMeta?.messageId
|
||
? { messageId: emailMeta.messageId, filename: file.name }
|
||
: null,
|
||
correlation_id: correlationId,
|
||
matched_transaction_id: matchedTransactionId ?? null,
|
||
whatsapp_message_id: opts.channelMeta?.whatsappMessageId ?? null,
|
||
channel_context: opts.channelMeta
|
||
? { channel: 'whatsapp', caption: sanitiseCaption(opts.channelMeta.caption) }
|
||
: null,
|
||
})
|
||
.select('*')
|
||
.single()
|
||
|
||
if (inboxError) throw new Error(`Failed to create inbox item: ${inboxError.message}`)
|
||
|
||
scheduleDeferredExtraction({
|
||
itemId: inbox.id,
|
||
documentId: doc.id,
|
||
companyId,
|
||
correlationId,
|
||
file,
|
||
pageCount,
|
||
gatedByPageCount,
|
||
})
|
||
|
||
return {
|
||
document_id: doc.id,
|
||
inbox_item_id: inbox.id,
|
||
status: inbox.status,
|
||
extracted_data: null,
|
||
matched_supplier_id: inbox.matched_supplier_id,
|
||
matched_transaction_id: inbox.matched_transaction_id,
|
||
extraction_skipped: false,
|
||
skip_reason: null,
|
||
page_count: pageCount,
|
||
}
|
||
}
|
||
|
||
// Long PDFs are sliced to their first pages instead of skipped, but only
|
||
// when extraction would actually run: slicing after an entitlement/sandbox/
|
||
// opt-out verdict would be wasted CPU.
|
||
const slicedBuffer =
|
||
gatedByPageCount && syncSkipReason === null
|
||
? await slicePdfForExtraction(file.buffer, MAX_PAGES_FOR_AUTO_EXTRACT)
|
||
: null
|
||
// Skip-reason priority: no-AI-entitlement > sandbox > client opt-out >
|
||
// page-count. Opt-out outranks the page gate (an opted-out caller never
|
||
// extracts regardless of length), and too_many_pages only fires when the
|
||
// slice fallback also failed (encrypted/malformed PDF).
|
||
const skipReason: 'no_ai_entitlement' | 'too_many_pages' | 'client_opt_out' | 'sandbox' | null =
|
||
syncSkipReason ??
|
||
(gatedByPageCount && slicedBuffer == null ? 'too_many_pages' : null)
|
||
const skipExtraction = skipReason !== null
|
||
|
||
// Bring-your-own-extraction: skip the Bedrock call entirely and seed an
|
||
// empty extraction skeleton. The caller is expected to PUT the parsed
|
||
// fields via /items/:id/extracted-data before converting to a supplier
|
||
// invoice. extracted_data is never null in the DB; an empty skeleton
|
||
// keeps downstream readers (UI, MCP) happy.
|
||
const { data: extracted, rawText } = skipExtraction
|
||
? { data: emptyResult(), rawText: null }
|
||
: await extractInvoiceFields({
|
||
buffer: Buffer.from(slicedBuffer ?? file.buffer),
|
||
mimeType: file.type,
|
||
fileName: file.name,
|
||
})
|
||
if (!skipExtraction && slicedBuffer != null && pageCount != null) {
|
||
extracted.pages = { total: pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT }
|
||
}
|
||
|
||
// Supplier match by org-nr, then case-insensitive name (no AI fuzz).
|
||
let matchedSupplierId: string | null = null
|
||
if (extracted.supplier.orgNumber) {
|
||
const { data: s } = await supabase
|
||
.from('suppliers')
|
||
.select('id')
|
||
.eq('company_id', companyId)
|
||
.eq('org_number', extracted.supplier.orgNumber)
|
||
.limit(1)
|
||
.maybeSingle()
|
||
if (s) matchedSupplierId = s.id
|
||
}
|
||
if (!matchedSupplierId && extracted.supplier.name) {
|
||
const { data: s } = await supabase
|
||
.from('suppliers')
|
||
.select('id')
|
||
.eq('company_id', companyId)
|
||
.ilike('name', extracted.supplier.name)
|
||
.limit(1)
|
||
.maybeSingle()
|
||
if (s) matchedSupplierId = s.id
|
||
}
|
||
|
||
const { data: inbox, error: inboxError } = await supabase
|
||
.from('invoice_inbox_items')
|
||
.insert({
|
||
company_id: companyId,
|
||
user_id: userId,
|
||
status: 'received',
|
||
source,
|
||
document_id: doc.id,
|
||
extracted_data: extracted as unknown as Record<string, unknown>,
|
||
extraction_skipped: skipExtraction,
|
||
matched_supplier_id: matchedSupplierId,
|
||
email_from: emailMeta?.from || null,
|
||
email_subject: emailMeta?.subject || null,
|
||
email_received_at: emailMeta?.receivedAt || null,
|
||
email_body_text: emailMeta?.bodyText || null,
|
||
resend_email_id: emailMeta?.resendEmailId || null,
|
||
resend_attachment_id: emailMeta?.resendAttachmentId || null,
|
||
raw_email_payload: emailMeta?.messageId
|
||
? { messageId: emailMeta.messageId, filename: file.name }
|
||
: null,
|
||
correlation_id: correlationId,
|
||
matched_transaction_id: matchedTransactionId ?? null,
|
||
// Chat-channel provenance. Explicit nulls (not a conditional spread)
|
||
// keep the payload statically checkable; a null insert is identical to
|
||
// omitting the column, so the email/upload behavior is unchanged.
|
||
whatsapp_message_id: opts.channelMeta?.whatsappMessageId ?? null,
|
||
channel_context: opts.channelMeta
|
||
? { channel: 'whatsapp', caption: sanitiseCaption(opts.channelMeta.caption) }
|
||
: null,
|
||
})
|
||
.select('*')
|
||
.single()
|
||
|
||
if (inboxError) throw new Error(`Failed to create inbox item: ${inboxError.message}`)
|
||
|
||
try {
|
||
await appendProcessingHistory({
|
||
companyId,
|
||
correlationId,
|
||
aggregateType: 'Document',
|
||
aggregateId: doc.id,
|
||
eventType: 'DocumentExtractionAttempted',
|
||
payload: {
|
||
document_id: doc.id,
|
||
inbox_item_id: inbox.id,
|
||
succeeded: rawText != null && rawText.length > 0,
|
||
extracted_total: extracted.totals.total,
|
||
has_org_number: extracted.supplier.orgNumber != null,
|
||
has_ocr: extracted.invoice.paymentReference != null,
|
||
skipped: skipExtraction,
|
||
skip_reason: skipReason,
|
||
page_count: pageCount,
|
||
},
|
||
actor: { type: 'system', id: 'invoice-inbox-extract' },
|
||
occurredAt: new Date(),
|
||
})
|
||
} catch (err) {
|
||
console.error('[invoice-inbox] Failed to append DocumentExtractionAttempted:', err)
|
||
}
|
||
|
||
return {
|
||
document_id: doc.id,
|
||
inbox_item_id: inbox.id,
|
||
status: inbox.status,
|
||
extracted_data: extracted,
|
||
matched_supplier_id: inbox.matched_supplier_id,
|
||
matched_transaction_id: inbox.matched_transaction_id,
|
||
extraction_skipped: skipExtraction,
|
||
skip_reason: skipReason,
|
||
page_count: pageCount,
|
||
}
|
||
}
|
||
|
||
// ── Deferred extraction (staged web upload) ──────────────────
|
||
|
||
interface DeferredExtractionJob {
|
||
itemId: string
|
||
documentId: string
|
||
companyId: string
|
||
correlationId: string
|
||
file: { name: string; buffer: ArrayBuffer; type: string }
|
||
pageCount: number | null
|
||
gatedByPageCount: boolean
|
||
}
|
||
|
||
/**
|
||
* Run extraction after the upload response is sent, then CAS-flip the
|
||
* 'processing' row to 'received'. Exact dispatch-kick idiom (whatsapp-inbox
|
||
* kickInboundProcessing): never awaited, never throws out, falls back to a
|
||
* microtask outside a request scope (tests). The request-scoped supabase
|
||
* client may be gone once the response is flushed, so the worker builds its
|
||
* own cookieless service client.
|
||
*/
|
||
function scheduleDeferredExtraction(job: DeferredExtractionJob): void {
|
||
const run = async (): Promise<void> => {
|
||
try {
|
||
const supabase = createServiceClientNoCookies()
|
||
|
||
// Mirrors the synchronous path: extraction failures are swallowed into
|
||
// the empty skeleton (extractInvoiceFields does most of that itself);
|
||
// an unsliceable long PDF becomes the too_many_pages skip.
|
||
let extracted: InvoiceExtractionResult = emptyResult()
|
||
let rawText: string | null = null
|
||
let extractionSkipped = false
|
||
let skipReason: 'too_many_pages' | null = null
|
||
try {
|
||
const slicedBuffer = job.gatedByPageCount
|
||
? await slicePdfForExtraction(job.file.buffer, MAX_PAGES_FOR_AUTO_EXTRACT)
|
||
: null
|
||
if (job.gatedByPageCount && slicedBuffer == null) {
|
||
extractionSkipped = true
|
||
skipReason = 'too_many_pages'
|
||
} else {
|
||
const result = await extractInvoiceFields({
|
||
buffer: Buffer.from(slicedBuffer ?? job.file.buffer),
|
||
mimeType: job.file.type,
|
||
fileName: job.file.name,
|
||
})
|
||
extracted = result.data
|
||
rawText = result.rawText
|
||
if (slicedBuffer != null && job.pageCount != null) {
|
||
extracted.pages = { total: job.pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT }
|
||
}
|
||
}
|
||
} catch (err) {
|
||
// Persist the empty skeleton below, exactly like the sync path does
|
||
// on a swallowed failure: the row becomes a normal received item with
|
||
// empty fields, and the manual-edit / retry affordances take over.
|
||
console.error('[invoice-inbox] Deferred extraction failed:', err)
|
||
}
|
||
|
||
// Supplier match by org-nr, then case-insensitive name (no AI fuzz).
|
||
let matchedSupplierId: string | null = null
|
||
if (extracted.supplier.orgNumber) {
|
||
const { data: s } = await supabase
|
||
.from('suppliers')
|
||
.select('id')
|
||
.eq('company_id', job.companyId)
|
||
.eq('org_number', extracted.supplier.orgNumber)
|
||
.limit(1)
|
||
.maybeSingle()
|
||
if (s) matchedSupplierId = s.id
|
||
}
|
||
if (!matchedSupplierId && extracted.supplier.name) {
|
||
const { data: s } = await supabase
|
||
.from('suppliers')
|
||
.select('id')
|
||
.eq('company_id', job.companyId)
|
||
.ilike('name', extracted.supplier.name)
|
||
.limit(1)
|
||
.maybeSingle()
|
||
if (s) matchedSupplierId = s.id
|
||
}
|
||
|
||
// CAS: only the row still waiting on THIS worker flips. A user retry
|
||
// or the sweep cron may have claimed it first; their result wins.
|
||
const { data: claimed, error: updateError } = await supabase
|
||
.from('invoice_inbox_items')
|
||
.update({
|
||
status: 'received',
|
||
extracted_data: extracted as unknown as Record<string, unknown>,
|
||
extraction_skipped: extractionSkipped,
|
||
matched_supplier_id: matchedSupplierId,
|
||
})
|
||
.eq('id', job.itemId)
|
||
.eq('status', 'processing')
|
||
.select('id')
|
||
if (updateError) {
|
||
// The sweep cron flips the row to 'received' once it goes stale.
|
||
console.error('[invoice-inbox] Deferred extraction flip failed:', updateError)
|
||
return
|
||
}
|
||
if (!Array.isArray(claimed) || claimed.length === 0) return
|
||
|
||
try {
|
||
await appendProcessingHistory({
|
||
companyId: job.companyId,
|
||
correlationId: job.correlationId,
|
||
aggregateType: 'Document',
|
||
aggregateId: job.documentId,
|
||
eventType: 'DocumentExtractionAttempted',
|
||
payload: {
|
||
document_id: job.documentId,
|
||
inbox_item_id: job.itemId,
|
||
succeeded: rawText != null && rawText.length > 0,
|
||
extracted_total: extracted.totals.total,
|
||
has_org_number: extracted.supplier.orgNumber != null,
|
||
has_ocr: extracted.invoice.paymentReference != null,
|
||
skipped: extractionSkipped,
|
||
skip_reason: skipReason,
|
||
page_count: job.pageCount,
|
||
},
|
||
actor: { type: 'system', id: 'invoice-inbox-extract' },
|
||
occurredAt: new Date(),
|
||
})
|
||
} catch (err) {
|
||
console.error('[invoice-inbox] Failed to append DocumentExtractionAttempted:', err)
|
||
}
|
||
} catch (err) {
|
||
console.error('[invoice-inbox] Deferred extraction worker crashed:', err)
|
||
// Best-effort flip so the row does not sit in 'processing' until the
|
||
// sweep: same CAS, same empty skeleton the sync swallow persists.
|
||
try {
|
||
await createServiceClientNoCookies()
|
||
.from('invoice_inbox_items')
|
||
.update({
|
||
status: 'received',
|
||
extracted_data: emptyResult() as unknown as Record<string, unknown>,
|
||
extraction_skipped: false,
|
||
})
|
||
.eq('id', job.itemId)
|
||
.eq('status', 'processing')
|
||
} catch {
|
||
// The sweep cron is the recovery of last resort.
|
||
}
|
||
}
|
||
}
|
||
|
||
try {
|
||
after(() => run())
|
||
} catch {
|
||
queueMicrotask(() => void run())
|
||
}
|
||
}
|