Files
accounted/extensions/general/invoice-inbox/lib/upload-and-extract.ts
T
f266c386f3 chore: repo-wide bloat sweep, remove dead code and fold duplicate helpers (#2150)
* chore: repo-wide bloat sweep, remove dead code and fold duplicate helpers

Remove 33 dead files, ~270 unreferenced exports/types, 13 dead i18n
namespaces and 4 unused dependencies; fold byte-identical helper copies
into one canonical home each (lib/utils chunk/sleep/utcDateStamp,
lib/dates/iso, lib/invariants/uuid, lib/xml/escape, lib/reports/sru/format,
lib/pdf/number-text, lib/browser/panel-request, lib/api/v1/body +
v1ValidationError rolled out to ~55 v1 routes, booking-template schemas).

No behaviour change: v1 bodies and status codes, MCP tool schemas, DB
writes and money math are untouched. Naive ore rounding was deliberately
not swapped for roundOre; see DECISIONS.md 2026-09-02 for the full list
of things left alone on purpose.

tsc, lint, 19588 unit tests and check:guards green; antipattern baseline
ratcheted (naive-ore-round 622 -> 620, hand-rolled-invariant 115 -> 113).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* test(transactions): import RawTransaction from @/types after the ingest re-export removal

CI's type ratchet (check:types, full tsconfig) caught the one test file
that still imported the type through lib/transactions/ingest.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-02 11:51:16 +02:00

772 lines
31 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { after } from 'next/server'
import { uploadDocument } from '@/lib/core/documents/document-service'
import { extractInvoiceFields, emptyResult, fetchOwnCompanyIdentity } from './extract-invoice-fields'
import { mirrorExtractionToDocument } from './mirror-extraction'
import type { InboxKindHint } from './resend-inbound'
import { getAiStatus } from '@/lib/ai'
import { hasCapability } from '@/lib/entitlements/has-capability'
import { CAPABILITY } from '@/lib/entitlements/keys'
import { appendProcessingHistory } from '@/lib/processing-history/append'
import { createServiceClientNoCookies } from '@/lib/auth/api-keys'
import { matchSupplierId } from '@/lib/suppliers/match-supplier'
import type { InvoiceExtractionResult } from '@/types'
import { PDFDocument } from 'pdf-lib'
import path from 'node:path'
import { escapeHtml } from '@/lib/email/user-text'
// Verdicts decidable before touching the file. `ai_unconfigured` is the
// deployment-level "no AI backend" state (self-host without a key), distinct
// from the per-company paywall (`no_ai_entitlement`).
type SyncSkipReason =
| 'no_ai_entitlement'
| 'client_opt_out'
| 'sandbox'
| 'ai_unconfigured'
| null
/**
* Defensive filename sanitisation for content arriving from .eml inner
* attachments and rejected-attachment metadata. The document-service already
* sanitises before storage paths are built (lib/core/documents/document-service.ts),
* so this is defense-in-depth: strip directory traversal sequences and exotic
* characters before they ever flow into DB columns or downstream consumers.
*/
export function sanitiseFilename(raw: string | null | undefined, fallback: string): string {
const base = path.basename(String(raw ?? '').trim())
const cleaned = base.replace(/[^\w.-]/g, '_').slice(0, 200)
return cleaned || fallback
}
export function sanitiseMime(raw: string | null | undefined): string {
const value = String(raw ?? '').trim().slice(0, 120)
return /^[\w./+-]+$/.test(value) ? value : 'application/octet-stream'
}
export const MAX_FILE_SIZE = 10 * 1024 * 1024
// Page budget for auto-extraction. Documents far above it tend to be sales
// reports, bank statements, or contracts (issue #553); those are sliced (see
// slicePdfForExtraction) rather than read whole. The budget depends on how
// the backend reads PDFs:
// - native (Claude on hosted): the model reads PDF bytes directly and
// handles long documents fine; 8 pages covers real multi-page invoices
// whose totals sit on a later page without opening the door to 100-page
// statements.
// - rasterize (OpenAI-compatible self-host): every page becomes a PNG in
// the prompt, so the old conservative budget of 3 stays.
export const MAX_PAGES_FOR_AUTO_EXTRACT = 3
export const MAX_PAGES_FOR_AUTO_EXTRACT_NATIVE = 8
export function maxPagesForAutoExtract(): number {
return getAiStatus().capabilities.pdfNative
? MAX_PAGES_FOR_AUTO_EXTRACT_NATIVE
: MAX_PAGES_FOR_AUTO_EXTRACT
}
// Returns the page count for a PDF buffer, or null if the buffer isn't a
// parseable PDF. Errors fall through so callers can treat "unknown" the same
// as "small enough": preserves today's behavior on malformed inputs.
export async function countPdfPages(buffer: ArrayBuffer): Promise<number | null> {
try {
const pdf = await PDFDocument.load(buffer, { updateMetadata: false })
return pdf.getPageCount()
} catch {
return null
}
}
// Long PDFs used to skip extraction entirely (issue #553). Instead we extract
// from a slim copy and record the truncation in extracted_data.pages. The
// slice is the first maxPages-1 pages PLUS the last page: on multi-page
// invoices the totals, OCR number and "Att betala" routinely sit on the final
// page, and a first-pages-only slice read everything except the amounts.
// Returns null when slicing fails (encrypted/malformed PDF) so the caller can
// fall back to the old skip.
export async function slicePdfForExtraction(
buffer: ArrayBuffer,
maxPages: number
): Promise<ArrayBuffer | null> {
try {
const src = await PDFDocument.load(buffer, { updateMetadata: false })
const dst = await PDFDocument.create()
const total = src.getPageCount()
const indices =
total > maxPages && maxPages >= 2
? [...Array.from({ length: maxPages - 1 }, (_, i) => i), total - 1]
: Array.from({ length: Math.min(maxPages, total) }, (_, i) => i)
const pages = await dst.copyPages(src, indices)
for (const page of pages) dst.addPage(page)
const bytes = await dst.save()
// Copy into a fresh ArrayBuffer: Uint8Array.buffer is ArrayBufferLike
// (possibly SharedArrayBuffer-backed) and may span more than the view.
const out = new ArrayBuffer(bytes.byteLength)
new Uint8Array(out).set(bytes)
return out
} catch {
return null
}
}
// Sandbox companies (24h anonymous demo accounts) skip the Bedrock extraction
// pipeline entirely. The document still uploads, the inbox row still lands,
// and the user can fill the fields in by hand, but no Claude tokens are
// spent on a throwaway account. See migration 20260311120000 for the column.
export async function isSandboxCompany(
supabase: import('@supabase/supabase-js').SupabaseClient,
companyId: string,
): Promise<boolean> {
const { data, error } = await supabase
.from('company_settings')
.select('is_sandbox')
.eq('company_id', companyId)
.maybeSingle()
if (error || !data) return false
return data.is_sandbox === true
}
export const UPLOAD_ALLOWED_MIME_TYPES = new Set([
'application/pdf',
'image/jpeg',
'image/png',
'image/heic',
'image/heif',
'image/webp',
])
// The email pipeline additionally accepts HTML: many suppliers send the
// invoice as the mail body or an attached .html file, and for those the mail
// IS the underlag (BFL: the received form must be preserved). Deliberately
// NOT added to UPLOAD_ALLOWED_MIME_TYPES: HTML only enters through the
// inbound webhook, where the preview surfaces render it fully sandboxed.
export const EMAIL_ALLOWED_MIME_TYPES = new Set([...UPLOAD_ALLOWED_MIME_TYPES, 'text/html'])
/**
* Wrap HTML in a minimal document shell unless it already is a full document.
* Mail bodies are usually fragments (a bare <div>/<table>); the document
* archive should hold self-contained files, and the document-service
* magic-byte check requires HTML content to start with a doctype/root element.
*/
export function ensureHtmlDocument(html: string): ArrayBuffer {
const head = html.replace(/^[\s]+/, '').slice(0, 256).toLowerCase()
const isFullDocument =
head.startsWith('<!doctype html') || head.startsWith('<html') || head.startsWith('<?xml')
const full = isFullDocument
? html
: `<!doctype html>\n<html>\n<head><meta charset="utf-8"></head>\n<body>\n${html}\n</body>\n</html>\n`
const bytes = new TextEncoder().encode(full)
// Copy into a fresh ArrayBuffer: Uint8Array.buffer is ArrayBufferLike
// (possibly SharedArrayBuffer-backed) and may span more than the view.
const out = new ArrayBuffer(bytes.byteLength)
new Uint8Array(out).set(bytes)
return out
}
/**
* Build a text/html document from a mail's body parts, preferring the HTML
* part. A plain-text-only mail is escaped into a <pre> so whitespace, amounts
* and OCR numbers survive verbatim. Returns null when the mail has no body
* worth storing.
*/
export function buildEmailBodyHtmlDocument(
html: string | null,
text: string | null
): ArrayBuffer | null {
if (html?.trim()) return ensureHtmlDocument(html)
if (text?.trim()) return ensureHtmlDocument(`<pre>${escapeHtml(text)}</pre>`)
return null
}
export interface EmailMeta {
from?: string | null
subject?: string | null
receivedAt?: string | null
messageId?: string | null
bodyText?: string | null
resendEmailId?: string | null
resendAttachmentId?: string | null
// Sender-declared document kind from the +lev / +ver plus-address tag.
// Lands in its own column (not extracted_data) so re-extraction cannot
// overwrite what the sender said.
kindHint?: InboxKindHint | null
}
// Chat-channel provenance (whatsapp-inbox extension). When present, the inbox
// row links back to the delivering whatsapp_messages row and seeds
// channel_context (kept OUT of extracted_data: retry-extraction overwrites
// that container wholesale, and chat context must survive it).
export interface ChannelMeta {
whatsappMessageId?: string
caption?: string | null
}
// Captions are attacker-adjacent free text from a chat client: strip control
// characters and cap length before they land in a jsonb column read by the UI.
function sanitiseCaption(raw: string | null | undefined): string | null {
if (!raw) return null
// eslint-disable-next-line no-control-regex
const cleaned = raw.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '').trim().slice(0, 500)
return cleaned || null
}
// ── Shared helper: upload + extract + create inbox item ──────
export interface ArchivedDocumentProcessingOptions {
skipExtraction?: boolean
channelMeta?: ChannelMeta
/** Overrides the system actor id on the DocumentIngested history event.
* Omitted = today's behavior (resend-inbound for email, user otherwise). */
actorId?: string
/**
* Staged upload (web route only). When true AND extraction would actually
* call Bedrock, the inbox row is inserted first (status 'processing',
* extracted_data NULL), the function returns immediately, and extraction
* runs after the response via a deferred worker that CAS-flips the row to
* 'received'. Verdicts that never reach Bedrock (no AI entitlement,
* sandbox, client opt-out) stay on the synchronous path: they are quick
* and their response contract (empty skeleton + skip_reason) is
* unchanged. skipExtraction in particular MUST stay synchronous: a
* BYO-extraction agent PUTs its fields right after upload, and a deferred
* flip would overwrite them. Default false = today's synchronous behavior
* (email and WhatsApp callers are untouched).
*/
deferExtraction?: boolean
}
export async function uploadAndExtract(
supabase: import('@supabase/supabase-js').SupabaseClient,
userId: string,
companyId: string,
file: { name: string; buffer: ArrayBuffer; type: string },
source: 'upload' | 'email' | 'whatsapp',
emailMeta?: EmailMeta,
// Pre-match the new inbox item to a bank transaction. Set when the caller
// already knows which transaction this receipt belongs to (e.g. the
// VerifyAndBookOverlay opened from a transaction row's paperclip or from
// a transaction-anchored chat). Skipped silently if missing.
matchedTransactionId?: string | null,
opts: ArchivedDocumentProcessingOptions = {},
) {
const doc = await uploadDocument(supabase, userId, companyId, {
name: file.name,
buffer: file.buffer,
type: file.type,
}, {
upload_source: source === 'email' ? 'email' : source === 'whatsapp' ? 'whatsapp' : 'file_upload',
// Every inbox channel dedupes on content: the same receipt forwarded to
// two inboxes, re-hunted by a sweep, or uploaded twice must not become a
// second archived document.
dedupeByContent: true,
// This function extracts (sync or deferred) and mirrors the outcome onto
// the document row; the document-extraction extension must not race it.
extractionOwner: 'invoice-inbox',
})
return processArchivedDocument(
supabase,
userId,
companyId,
doc,
file,
source,
emailMeta,
matchedTransactionId,
opts,
)
}
/**
* Everything the inbox does with a document once it sits in the archive:
* adopt an existing inbox item for deduplicated content, record the
* DocumentIngested history, apply the page-count gate, decide the
* entitlement/sandbox/opt-out verdict, extract (synchronously or deferred)
* and insert the inbox row. Split out of uploadAndExtract so the
* direct-to-storage route (signed upload URL, bytes never in a function
* body) can archive through completePendingDocumentUpload and then join the
* exact same pipeline as the multipart route.
*
* `doc` is the archived document (or the existing one it deduplicated to),
* `file` carries the same bytes the archive holds: the pipeline reads them
* for page counting and extraction.
*/
export async function processArchivedDocument(
supabase: import('@supabase/supabase-js').SupabaseClient,
userId: string,
companyId: string,
doc: { id: string; deduplicated?: boolean },
file: { name: string; buffer: ArrayBuffer; type: string },
source: 'upload' | 'email' | 'whatsapp',
emailMeta?: EmailMeta,
matchedTransactionId?: string | null,
opts: ArchivedDocumentProcessingOptions = {},
) {
const correlationId = crypto.randomUUID()
if (doc.deduplicated) {
// The company already archived this exact content. If an inbox item
// exists for it, adopt that item so the caller always gets a real
// inbox_item_id; only when the document entered outside the inbox do we
// fall through and file an item for the EXISTING document (no copy).
const { data: existingItem, error: itemLookupError } = await supabase
.from('invoice_inbox_items')
.select('id, status, extracted_data, matched_supplier_id, matched_transaction_id')
.eq('company_id', companyId)
.eq('document_id', doc.id)
.order('created_at', { ascending: true })
.limit(1)
if (itemLookupError) {
// Fail closed: falling through would file a second item for a document
// that may already carry one. The delivering channel retries.
throw new Error(`Duplicate-item lookup failed: ${itemLookupError.message}`)
}
const adopted = (existingItem as Array<{
id: string
status: string
extracted_data: unknown
matched_supplier_id: string | null
matched_transaction_id: string | null
}> | null)?.[0]
if (adopted) {
try {
await appendProcessingHistory({
companyId,
correlationId,
aggregateType: 'Document',
aggregateId: doc.id,
eventType: 'DocumentDuplicateSkipped',
payload: {
channel: source,
document_id: doc.id,
inbox_item_id: adopted.id,
reason: 'duplicate_content',
},
actor: opts.actorId
? { type: 'system', id: opts.actorId }
: source === 'email'
? { type: 'system', id: 'resend-inbound' }
: { type: 'user', id: userId },
occurredAt: new Date(),
})
} catch (err) {
console.error('[invoice-inbox] Failed to append DocumentDuplicateSkipped:', err)
}
return {
document_id: doc.id,
inbox_item_id: adopted.id,
status: adopted.status,
extracted_data: adopted.extracted_data,
matched_supplier_id: adopted.matched_supplier_id,
matched_transaction_id: adopted.matched_transaction_id,
extraction_skipped: true,
skip_reason: 'duplicate_content' as const,
page_count: null,
duplicate: true as const,
}
}
}
try {
await appendProcessingHistory({
companyId,
correlationId,
aggregateType: 'Document',
aggregateId: doc.id,
eventType: 'DocumentIngested',
payload: {
channel: source,
document_id: doc.id,
mime_type: file.type,
size_bytes: file.buffer.byteLength,
},
actor: opts.actorId
? { type: 'system', id: opts.actorId }
: source === 'email'
? { type: 'system', id: 'resend-inbound' }
: { type: 'user', id: userId },
occurredAt: new Date(),
})
} catch (err) {
console.error('[invoice-inbox] Failed to append DocumentIngested:', err)
}
// Page-count gate (issue #553): PDFs above the auto-extract page budget
// are sliced before extraction. Bedrock would otherwise block the upload
// response for minutes on a long sales report and return nothing useful.
// Images and non-PDFs are never gated (single-page by definition).
// countPdfPages returns null on malformed PDFs: we treat null as "not
// gated" and fall through to the existing extraction path so today's
// behavior is preserved.
const maxAutoExtractPages = maxPagesForAutoExtract()
const pageCount =
file.type === 'application/pdf' ? await countPdfPages(file.buffer) : null
const gatedByPageCount =
pageCount != null && pageCount > maxAutoExtractPages
const sandbox = await isSandboxCompany(supabase, companyId)
// Paid-tier gate: AI document OCR (Bedrock, via extractInvoiceFields) is the
// `ai` capability. A company without it (free/manual tier) must never trigger
// paid extraction: we seed an empty skeleton exactly like the sandbox / BYO-
// extraction path, so the document is still stored and can be filled in
// manually. Highest priority (a hard paywall rule, not a heuristic).
const hasAiEntitlement = await hasCapability(supabase, companyId, CAPABILITY.ai)
// The verdicts that avoid Bedrock entirely, in the established priority
// order: no-AI-entitlement > sandbox > client opt-out. All three are
// decidable without touching the PDF; too_many_pages is not (it only fires
// when the slice fallback fails) and is resolved below, on whichever path
// (sync or deferred) actually attempts the slice.
// `ai_unconfigured` (no AI credentials/model on this deployment, the
// self-host "key not set yet" state) is decided up front as well: a call
// that can never succeed must not take the deferred path and leave the row
// in 'processing' until the sweep.
const syncSkipReason: SyncSkipReason =
!hasAiEntitlement
? 'no_ai_entitlement'
: sandbox
? 'sandbox'
: opts.skipExtraction
? 'client_opt_out'
: !getAiStatus().configured
? 'ai_unconfigured'
: null
if (opts.deferExtraction && syncSkipReason === null) {
// Staged path: extraction WILL call Bedrock, so create the row now and
// let the response go. The deferred worker (or, after a crash, the sweep
// cron at /api/extensions/invoice-inbox/sweep/cron) flips the row to
// 'received'. extracted_data stays NULL until then: that is what keeps
// the MCP create-from-inbox guard ("re-run extraction first") holding
// for free on in-flight rows.
const { data: inbox, error: inboxError } = await supabase
.from('invoice_inbox_items')
.insert({
company_id: companyId,
user_id: userId,
status: 'processing',
source,
document_id: doc.id,
extracted_data: null,
extraction_skipped: false,
matched_supplier_id: null,
email_from: emailMeta?.from || null,
email_subject: emailMeta?.subject || null,
email_received_at: emailMeta?.receivedAt || null,
email_body_text: emailMeta?.bodyText || null,
resend_email_id: emailMeta?.resendEmailId || null,
resend_attachment_id: emailMeta?.resendAttachmentId || null,
kind_hint: emailMeta?.kindHint ?? null,
raw_email_payload: emailMeta?.messageId
? { messageId: emailMeta.messageId, filename: file.name }
: null,
correlation_id: correlationId,
matched_transaction_id: matchedTransactionId ?? null,
whatsapp_message_id: opts.channelMeta?.whatsappMessageId ?? null,
channel_context: opts.channelMeta
? { channel: 'whatsapp', caption: sanitiseCaption(opts.channelMeta.caption) }
: null,
})
.select('*')
.single()
if (inboxError) throw new Error(`Failed to create inbox item: ${inboxError.message}`)
scheduleDeferredExtraction({
itemId: inbox.id,
documentId: doc.id,
companyId,
correlationId,
file,
pageCount,
gatedByPageCount,
maxAutoExtractPages,
})
return {
document_id: doc.id,
inbox_item_id: inbox.id,
status: inbox.status,
extracted_data: null,
matched_supplier_id: inbox.matched_supplier_id,
matched_transaction_id: inbox.matched_transaction_id,
extraction_skipped: false,
skip_reason: null,
page_count: pageCount,
}
}
// Long PDFs are sliced (first pages + the last page) instead of skipped,
// but only when extraction would actually run: slicing after an
// entitlement/sandbox/opt-out verdict would be wasted CPU.
const slicedBuffer =
gatedByPageCount && syncSkipReason === null
? await slicePdfForExtraction(file.buffer, maxAutoExtractPages)
: null
// Skip-reason priority: no-AI-entitlement > sandbox > client opt-out >
// page-count. Opt-out outranks the page gate (an opted-out caller never
// extracts regardless of length), and too_many_pages only fires when the
// slice fallback also failed (encrypted/malformed PDF).
const skipReason: SyncSkipReason | 'too_many_pages' =
syncSkipReason ??
(gatedByPageCount && slicedBuffer == null ? 'too_many_pages' : null)
const skipExtraction = skipReason !== null
// Bring-your-own-extraction: skip the Bedrock call entirely and seed an
// empty extraction skeleton. The caller is expected to PUT the parsed
// fields via /items/:id/extracted-data before converting to a supplier
// invoice. extracted_data is never null in the DB; an empty skeleton
// keeps downstream readers (UI, MCP) happy.
const extraction = skipExtraction
? { data: emptyResult(), rawText: null, model: null, skipped: null }
: await extractInvoiceFields({
buffer: Buffer.from(slicedBuffer ?? file.buffer),
mimeType: file.type,
fileName: file.name,
ownCompany: await fetchOwnCompanyIdentity(supabase, companyId),
})
const { data: extracted, rawText } = extraction
if (!skipExtraction && slicedBuffer != null && pageCount != null) {
extracted.pages = { total: pageCount, analyzed: maxAutoExtractPages }
}
// Supplier match by org-nr, then VAT number, then case-insensitive name
// (no AI fuzz).
const matchedSupplierId = await matchSupplierId(supabase, companyId, extracted.supplier)
const { data: inbox, error: inboxError } = await supabase
.from('invoice_inbox_items')
.insert({
company_id: companyId,
user_id: userId,
status: 'received',
source,
document_id: doc.id,
extracted_data: extracted as unknown as Record<string, unknown>,
extraction_skipped: skipExtraction,
matched_supplier_id: matchedSupplierId,
email_from: emailMeta?.from || null,
email_subject: emailMeta?.subject || null,
email_received_at: emailMeta?.receivedAt || null,
email_body_text: emailMeta?.bodyText || null,
resend_email_id: emailMeta?.resendEmailId || null,
resend_attachment_id: emailMeta?.resendAttachmentId || null,
kind_hint: emailMeta?.kindHint ?? null,
raw_email_payload: emailMeta?.messageId
? { messageId: emailMeta.messageId, filename: file.name }
: null,
correlation_id: correlationId,
matched_transaction_id: matchedTransactionId ?? null,
// Chat-channel provenance. Explicit nulls (not a conditional spread)
// keep the payload statically checkable; a null insert is identical to
// omitting the column, so the email/upload behavior is unchanged.
whatsapp_message_id: opts.channelMeta?.whatsappMessageId ?? null,
channel_context: opts.channelMeta
? { channel: 'whatsapp', caption: sanitiseCaption(opts.channelMeta.caption) }
: null,
})
.select('*')
.single()
if (inboxError) throw new Error(`Failed to create inbox item: ${inboxError.message}`)
// The extension yielded to us (extractionOwner); put the outcome where it
// would have written it, so the document row tells the same story.
await mirrorExtractionToDocument(doc.id, {
data: extracted,
rawText,
model: extraction.model ?? null,
skipped: skipReason ?? extraction.skipped ?? null,
})
try {
await appendProcessingHistory({
companyId,
correlationId,
aggregateType: 'Document',
aggregateId: doc.id,
eventType: 'DocumentExtractionAttempted',
payload: {
document_id: doc.id,
inbox_item_id: inbox.id,
succeeded: rawText != null && rawText.length > 0,
extracted_total: extracted.totals.total,
has_org_number: extracted.supplier.orgNumber != null,
has_ocr: extracted.invoice.paymentReference != null,
skipped: skipExtraction,
skip_reason: skipReason,
page_count: pageCount,
},
actor: { type: 'system', id: 'invoice-inbox-extract' },
occurredAt: new Date(),
})
} catch (err) {
console.error('[invoice-inbox] Failed to append DocumentExtractionAttempted:', err)
}
return {
document_id: doc.id,
inbox_item_id: inbox.id,
status: inbox.status,
extracted_data: extracted,
matched_supplier_id: inbox.matched_supplier_id,
matched_transaction_id: inbox.matched_transaction_id,
extraction_skipped: skipExtraction,
skip_reason: skipReason,
page_count: pageCount,
}
}
// ── Deferred extraction (staged web upload) ──────────────────
interface DeferredExtractionJob {
itemId: string
documentId: string
companyId: string
correlationId: string
file: { name: string; buffer: ArrayBuffer; type: string }
pageCount: number | null
gatedByPageCount: boolean
/** Snapshot of maxPagesForAutoExtract() from the request, so the gate
* verdict and the slice/stamp below cannot disagree. */
maxAutoExtractPages: number
}
/**
* Run extraction after the upload response is sent, then CAS-flip the
* 'processing' row to 'received'. Exact dispatch-kick idiom (whatsapp-inbox
* kickInboundProcessing): never awaited, never throws out, falls back to a
* microtask outside a request scope (tests). The request-scoped supabase
* client may be gone once the response is flushed, so the worker builds its
* own cookieless service client.
*/
function scheduleDeferredExtraction(job: DeferredExtractionJob): void {
const run = async (): Promise<void> => {
try {
const supabase = createServiceClientNoCookies()
// Mirrors the synchronous path: extraction failures are swallowed into
// the empty skeleton (extractInvoiceFields does most of that itself);
// an unsliceable long PDF becomes the too_many_pages skip.
let extracted: InvoiceExtractionResult = emptyResult()
let rawText: string | null = null
let model: string | null = null
let extractionSkipped = false
let skipReason: string | null = null
try {
const slicedBuffer = job.gatedByPageCount
? await slicePdfForExtraction(job.file.buffer, job.maxAutoExtractPages)
: null
if (job.gatedByPageCount && slicedBuffer == null) {
extractionSkipped = true
skipReason = 'too_many_pages'
} else {
const result = await extractInvoiceFields({
buffer: Buffer.from(slicedBuffer ?? job.file.buffer),
mimeType: job.file.type,
fileName: job.file.name,
ownCompany: await fetchOwnCompanyIdentity(supabase, job.companyId),
})
extracted = result.data
rawText = result.rawText
model = result.model ?? null
if (result.skipped) {
// No model call could be made (AI unconfigured, no vision on this
// backend, rasterizer missing). Same UI affordance as the other
// skips: the row lands with the empty skeleton and the hint.
extractionSkipped = true
skipReason = result.skipped
}
if (slicedBuffer != null && job.pageCount != null) {
extracted.pages = { total: job.pageCount, analyzed: job.maxAutoExtractPages }
}
}
} catch (err) {
// Persist the empty skeleton below, exactly like the sync path does
// on a swallowed failure: the row becomes a normal received item with
// empty fields, and the manual-edit / retry affordances take over.
console.error('[invoice-inbox] Deferred extraction failed:', err)
}
// Supplier match by org-nr, then VAT number, then case-insensitive name
// (no AI fuzz).
const matchedSupplierId = await matchSupplierId(
supabase,
job.companyId,
extracted.supplier,
)
// CAS: only the row still waiting on THIS worker flips. A user retry
// or the sweep cron may have claimed it first; their result wins.
const { data: claimed, error: updateError } = await supabase
.from('invoice_inbox_items')
.update({
status: 'received',
extracted_data: extracted as unknown as Record<string, unknown>,
extraction_skipped: extractionSkipped,
matched_supplier_id: matchedSupplierId,
})
.eq('id', job.itemId)
.eq('status', 'processing')
.select('id')
if (updateError) {
// The sweep cron flips the row to 'received' once it goes stale.
console.error('[invoice-inbox] Deferred extraction flip failed:', updateError)
return
}
if (!Array.isArray(claimed) || claimed.length === 0) return
await mirrorExtractionToDocument(job.documentId, {
data: extracted,
rawText,
model,
skipped: skipReason,
})
try {
await appendProcessingHistory({
companyId: job.companyId,
correlationId: job.correlationId,
aggregateType: 'Document',
aggregateId: job.documentId,
eventType: 'DocumentExtractionAttempted',
payload: {
document_id: job.documentId,
inbox_item_id: job.itemId,
succeeded: rawText != null && rawText.length > 0,
extracted_total: extracted.totals.total,
has_org_number: extracted.supplier.orgNumber != null,
has_ocr: extracted.invoice.paymentReference != null,
skipped: extractionSkipped,
skip_reason: skipReason,
page_count: job.pageCount,
},
actor: { type: 'system', id: 'invoice-inbox-extract' },
occurredAt: new Date(),
})
} catch (err) {
console.error('[invoice-inbox] Failed to append DocumentExtractionAttempted:', err)
}
} catch (err) {
console.error('[invoice-inbox] Deferred extraction worker crashed:', err)
// Best-effort flip so the row does not sit in 'processing' until the
// sweep: same CAS, same empty skeleton the sync swallow persists.
try {
await createServiceClientNoCookies()
.from('invoice_inbox_items')
.update({
status: 'received',
extracted_data: emptyResult() as unknown as Record<string, unknown>,
extraction_skipped: false,
})
.eq('id', job.itemId)
.eq('status', 'processing')
await mirrorExtractionToDocument(job.documentId, { data: null, rawText: null })
} catch {
// The sweep cron is the recovery of last resort.
}
}
}
try {
after(() => run())
} catch {
queueMicrotask(() => void run())
}
}