feat(invoice-inbox): receipt-aware extraction + OCR pipeline fixes (#1331)
Receipts and invoices were extracted through one invoice-shaped prompt with no document classification. The extractor now also returns documentKind, payment method (+ card last4), purchaseTime, merchantCategory and legibility, validated with .catch(null) so a hallucinated label degrades to unknown instead of sinking the parse. The FieldsRail shows type and payment method above the editable fields. Pipeline fixes, all verified against real failure paths: - PDFs >3 pages: extract from a pdf-lib slice of the first 3 pages instead of skipping entirely (issue #553 gate); truncation recorded in extracted_data.pages and shown in the UI. - Oversized images (>4 MB, over Bedrock's 5 MB cap): downscale to <=2000px JPEG via sharp before base64, instead of erroring to an empty result. - HEIC/HEIF: attempt sharp transcode to JPEG; when libvips lacks HEIF (prebuilt binaries), fall through to today's behavior but show an explicit hint instead of silently blank fields. - Oresavrundning: prompt rule + totals.roundingAmount so receipt totals reconcile with subtotal+VAT for exact-amount transaction matching. - retry-extraction overwrites extracted_data wholesale: a confirm now guards against silently destroying manual field edits. Deliberately NOT added: retry-on-transient-Bedrock-error; the SDK already retries twice by default (maxRetries=2). Co-authored-by: Jakob Wennberg <jakob.wennberg@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Jakob Wennberg
Claude Fable 5
Jakob Wennberg
Claude Fable 5
parent
b1f71912a2
commit
3b3adf96c7
@@ -769,3 +769,6 @@ One line per decision: `[YYYY-MM-DD] <decision>: <why>`. Appended by agents and
|
||||
[2026-08-03] kompletteringsregel_20 with a positive basis and zero acquisition cohorts is refused rather than computed: reducing over an empty cohort set would claim a full write-off the cohort evidence does not support (IL 18 kap. 17 §), so such periods require manual review instead of an automatic deduction.
|
||||
[2026-08-04] Balance-sheet synthetic result = complement of classes 1-2 (class 0/9/null rows included), not classes 3-8: a resultatavslut posted to 2099 without zeroing class 3-8 then self-cancels inside the residual instead of double-counting equity; mirrors balansrapport's residual definition (#1333)
|
||||
[2026-08-04] v1 settings PATCH writes directly (no staging), following the v1 customers precedent: REST callers are already scope-gated, staging is an MCP segregation-of-duties concept.
|
||||
[2026-08-01] OCR quick-fix "bounded retry on transient Bedrock errors" dropped: @anthropic-ai/bedrock-sdk already retries 429/5xx twice by default (maxRetries=2, not overridden in extract-invoice-fields.ts); duplicating it would triple worst-case latency for no reliability gain.
|
||||
[2026-08-01] Page-count gate (issue #553) changed from skip-extraction to slice-first-3-pages via pdf-lib: invoice data sits on page 1, and a >3-page supplier invoice getting ZERO fields was the worse failure mode. too_many_pages skip remains only for unsliceable (encrypted/malformed) PDFs; truncation recorded in extracted_data.pages. Side effect: client_opt_out now outranks too_many_pages in skip-reason priority (an opted-out caller never extracts regardless of length).
|
||||
[2026-08-01] Extraction classification fields (documentKind/payment/merchantCategory/legibility) validate with .catch(null) instead of strict enums: a hallucinated label must degrade to unknown, not sink the whole document parse; amounts keep strict parsing on purpose. HEIC handling = attempt sharp transcode at runtime, fall through to today's empty-extraction when libvips lacks HEIF (prebuilt binaries exclude it for patent reasons), with a UI hint replacing the silence.
|
||||
|
||||
@@ -1128,6 +1128,7 @@ export default function InvoiceInboxWorkspace(_props: WorkspaceComponentProps) {
|
||||
{selected ? (
|
||||
<FieldsRail
|
||||
item={selected}
|
||||
docMime={docMime}
|
||||
accountingMethod={accountingMethod}
|
||||
onDelete={() => handleDelete(selected.id)}
|
||||
onBookDirect={() => setBookDirectOpen(true)}
|
||||
@@ -1741,6 +1742,7 @@ function EmptyPreview({
|
||||
|
||||
function FieldsRail({
|
||||
item,
|
||||
docMime,
|
||||
accountingMethod,
|
||||
onDelete,
|
||||
onBookDirect,
|
||||
@@ -1753,6 +1755,7 @@ function FieldsRail({
|
||||
onRetryRequested,
|
||||
}: {
|
||||
item: InboxItem
|
||||
docMime: string | null
|
||||
accountingMethod: AccountingMethod
|
||||
onDelete: () => void
|
||||
onBookDirect: () => void
|
||||
@@ -1787,6 +1790,9 @@ function FieldsRail({
|
||||
!!extractedSupplierName
|
||||
|
||||
const handleRetry = async () => {
|
||||
// Retry overwrites extracted_data wholesale server-side, including any
|
||||
// manual field edits: make the user opt into that loss explicitly.
|
||||
if (hasAnyExtractedField(data) && !confirm(t('retry_overwrite_confirm'))) return
|
||||
setIsRetrying(true)
|
||||
try {
|
||||
const res = await fetch(
|
||||
@@ -1835,6 +1841,38 @@ function FieldsRail({
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* AI classification: what kind of document this is and how it was
|
||||
paid. Read-only context above the editable fields; absent for
|
||||
extractions from before the fields existed. */}
|
||||
{(data?.documentKind || data?.payment?.method || data?.pages) && (
|
||||
<div className="border-b px-4 py-3 text-xs space-y-1">
|
||||
{data?.documentKind && (
|
||||
<div className="flex gap-2">
|
||||
<span className="text-muted-foreground w-14 shrink-0">{t('doc_kind_label')}</span>
|
||||
<span>{t(`doc_kind_${data.documentKind}`)}</span>
|
||||
</div>
|
||||
)}
|
||||
{data?.payment?.method && (
|
||||
<div className="flex gap-2">
|
||||
<span className="text-muted-foreground w-14 shrink-0">{t('payment_label')}</span>
|
||||
<span>
|
||||
{t(`payment_${data.payment.method}`)}
|
||||
{data.payment.cardLast4 ? ` •• ${data.payment.cardLast4}` : ''}
|
||||
{data.purchaseTime ? ` · ${data.purchaseTime}` : ''}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
{data?.pages && (
|
||||
<div className="text-muted-foreground">
|
||||
{t('pages_partial_note', {
|
||||
analyzed: data.pages.analyzed,
|
||||
total: data.pages.total,
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{item.error_message && (
|
||||
<div className="border-b border-destructive/30 bg-destructive/5 px-4 py-3 text-xs space-y-2">
|
||||
<div className="flex items-start gap-2">
|
||||
@@ -1894,15 +1932,28 @@ function FieldsRail({
|
||||
)}
|
||||
|
||||
{/* Skipped-extraction hint: explains the empty fields and points the
|
||||
user to the manual paths (transaction link or supplier invoice). */}
|
||||
user to the manual paths (transaction link or supplier invoice).
|
||||
Deliberately reason-agnostic: skip covers sandbox, BYO-extraction
|
||||
and unsliceable PDFs, not just page count anymore. */}
|
||||
{item.extraction_skipped && !isResolved && (
|
||||
<div className="border-b bg-muted/30 px-4 py-2 text-xs text-muted-foreground">
|
||||
AI-tolkning skippades p.g.a. dokumentets storlek (fler än 3 sidor).
|
||||
Du kan koppla dokumentet till en transaktion eller skapa
|
||||
leverantörsfaktura manuellt.
|
||||
{t('skipped_hint')}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* HEIC: extraction ran but Bedrock cannot read the format, so every
|
||||
field came back empty with no error. Tell the user why instead of
|
||||
leaving a silently blank rail (iPhone photos default to HEIC). */}
|
||||
{!item.extraction_skipped &&
|
||||
!isResolved &&
|
||||
hasAi &&
|
||||
(docMime === 'image/heic' || docMime === 'image/heif') &&
|
||||
!hasAnyExtractedField(data) && (
|
||||
<div className="border-b bg-muted/30 px-4 py-2 text-xs text-muted-foreground">
|
||||
{t('heic_hint')}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Extracted fields */}
|
||||
<div className="flex-1 overflow-y-auto px-4 py-3">
|
||||
<h3 className="text-xs uppercase tracking-wide text-muted-foreground font-medium mb-3">
|
||||
|
||||
@@ -12,6 +12,25 @@ vi.mock('@anthropic-ai/bedrock-sdk', () => {
|
||||
return { default: FakeBedrock }
|
||||
})
|
||||
|
||||
// sharp is imported lazily by normalizeImageForExtraction. The default mock
|
||||
// (no implementation → TypeError on .rotate()) mimics a build without HEIF
|
||||
// support: normalization fails, the original buffer flows on. Individual
|
||||
// tests install a working chain via sharpMock.mockImplementationOnce.
|
||||
const sharpMock = vi.fn()
|
||||
vi.mock('sharp', () => ({
|
||||
default: (...args: unknown[]) => sharpMock(...args),
|
||||
}))
|
||||
|
||||
function workingSharpChain(outputBuffer: Buffer) {
|
||||
const chain = {
|
||||
rotate: vi.fn(() => chain),
|
||||
resize: vi.fn(() => chain),
|
||||
jpeg: vi.fn(() => chain),
|
||||
toBuffer: vi.fn().mockResolvedValue(outputBuffer),
|
||||
}
|
||||
return chain
|
||||
}
|
||||
|
||||
const ORIG_AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID
|
||||
const ORIG_AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY
|
||||
|
||||
@@ -182,6 +201,134 @@ describe('extractInvoiceFields', () => {
|
||||
expect(data.lineItems[0].accountSuggestion).toBeNull()
|
||||
})
|
||||
|
||||
// ── Receipt-aware classification fields (2026-08) ──────────
|
||||
|
||||
it('parses the classification fields when the model returns them', async () => {
|
||||
mockCreate.mockReturnValueOnce(
|
||||
aiResponse({
|
||||
...VALID_RESULT,
|
||||
documentKind: 'receipt',
|
||||
merchantCategory: 'restaurant',
|
||||
legibility: 'good',
|
||||
purchaseTime: '12:41',
|
||||
payment: { method: 'card', cardLast4: '1234' },
|
||||
totals: { ...VALID_RESULT.totals, roundingAmount: -0.25 },
|
||||
})
|
||||
)
|
||||
const { data } = await extractInvoiceFields({
|
||||
buffer: Buffer.from('%PDF'),
|
||||
mimeType: 'application/pdf',
|
||||
fileName: 'kvitto.pdf',
|
||||
})
|
||||
expect(data.documentKind).toBe('receipt')
|
||||
expect(data.merchantCategory).toBe('restaurant')
|
||||
expect(data.legibility).toBe('good')
|
||||
expect(data.purchaseTime).toBe('12:41')
|
||||
expect(data.payment).toEqual({ method: 'card', cardLast4: '1234' })
|
||||
expect(data.totals.roundingAmount).toBe(-0.25)
|
||||
})
|
||||
|
||||
it('still parses cached outputs from before the classification fields existed', async () => {
|
||||
// VALID_RESULT has none of the new fields: the whole document must
|
||||
// validate, not fall back to the empty result.
|
||||
mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT))
|
||||
const { data } = await extractInvoiceFields({
|
||||
buffer: Buffer.from('%PDF'),
|
||||
mimeType: 'application/pdf',
|
||||
fileName: 'old.pdf',
|
||||
})
|
||||
expect(data.supplier.name).toBe('Anthropic, PBC')
|
||||
expect(data.documentKind).toBeUndefined()
|
||||
})
|
||||
|
||||
it('degrades hallucinated classification values to null instead of failing the parse', async () => {
|
||||
mockCreate.mockReturnValueOnce(
|
||||
aiResponse({
|
||||
...VALID_RESULT,
|
||||
documentKind: 'parking_ticket',
|
||||
merchantCategory: 'nightclub',
|
||||
legibility: 'excellent',
|
||||
purchaseTime: '25:99',
|
||||
payment: { method: 'bitcoin', cardLast4: 'abcd' },
|
||||
totals: { ...VALID_RESULT.totals, roundingAmount: 'noll' },
|
||||
})
|
||||
)
|
||||
const { data } = await extractInvoiceFields({
|
||||
buffer: Buffer.from('%PDF'),
|
||||
mimeType: 'application/pdf',
|
||||
fileName: 'kvitto.pdf',
|
||||
})
|
||||
// Amounts survived: the junk classification did not sink the document.
|
||||
expect(data.totals.total).toBe(6.25)
|
||||
expect(data.documentKind).toBeNull()
|
||||
expect(data.merchantCategory).toBeNull()
|
||||
expect(data.legibility).toBeNull()
|
||||
expect(data.purchaseTime).toBeNull()
|
||||
expect(data.payment).toEqual({ method: null, cardLast4: null })
|
||||
expect(data.totals.roundingAmount).toBeNull()
|
||||
})
|
||||
|
||||
// ── Image normalization (HEIC transcode + oversized downscale) ──
|
||||
|
||||
it('transcodes HEIC to JPEG and extracts when sharp can decode it', async () => {
|
||||
const converted = Buffer.from('converted-jpeg-bytes')
|
||||
sharpMock.mockImplementationOnce(() => workingSharpChain(converted))
|
||||
mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT))
|
||||
|
||||
const { data } = await extractInvoiceFields({
|
||||
buffer: Buffer.from('heic-bytes'),
|
||||
mimeType: 'image/heic',
|
||||
fileName: 'photo.heic',
|
||||
})
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledOnce()
|
||||
const content = mockCreate.mock.calls[0][0].messages[0].content
|
||||
expect(content[0].source.media_type).toBe('image/jpeg')
|
||||
expect(content[0].source.data).toBe(converted.toString('base64'))
|
||||
expect(data.supplier.name).toBe('Anthropic, PBC')
|
||||
})
|
||||
|
||||
it('downscales oversized JPEGs before sending to Bedrock', async () => {
|
||||
const converted = Buffer.from('downscaled-jpeg')
|
||||
sharpMock.mockImplementationOnce(() => workingSharpChain(converted))
|
||||
mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT))
|
||||
|
||||
await extractInvoiceFields({
|
||||
buffer: Buffer.alloc(5 * 1024 * 1024, 1),
|
||||
mimeType: 'image/jpeg',
|
||||
fileName: 'big-photo.jpg',
|
||||
})
|
||||
|
||||
expect(sharpMock).toHaveBeenCalledOnce()
|
||||
const content = mockCreate.mock.calls[0][0].messages[0].content
|
||||
expect(content[0].source.data).toBe(converted.toString('base64'))
|
||||
})
|
||||
|
||||
it('keeps the original buffer when downscaling an oversized image fails', async () => {
|
||||
// Default sharpMock throws: the original 5 MB buffer is still attempted.
|
||||
mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT))
|
||||
|
||||
await extractInvoiceFields({
|
||||
buffer: Buffer.alloc(5 * 1024 * 1024, 1),
|
||||
mimeType: 'image/jpeg',
|
||||
fileName: 'big-photo.jpg',
|
||||
})
|
||||
|
||||
expect(mockCreate).toHaveBeenCalledOnce()
|
||||
const content = mockCreate.mock.calls[0][0].messages[0].content
|
||||
expect(content[0].source.media_type).toBe('image/jpeg')
|
||||
})
|
||||
|
||||
it('does not invoke sharp for normal-sized supported images', async () => {
|
||||
mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT))
|
||||
await extractInvoiceFields({
|
||||
buffer: Buffer.from('JPEG'),
|
||||
mimeType: 'image/jpeg',
|
||||
fileName: 'photo.jpg',
|
||||
})
|
||||
expect(sharpMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// Restore env vars so other test files aren't affected.
|
||||
afterAll(() => {
|
||||
if (ORIG_AWS_ACCESS_KEY_ID) process.env.AWS_ACCESS_KEY_ID = ORIG_AWS_ACCESS_KEY_ID
|
||||
|
||||
@@ -121,20 +121,33 @@ beforeEach(() => {
|
||||
})
|
||||
|
||||
describe('POST /upload: page-count gate (issue #553)', () => {
|
||||
it('skips extraction and marks the row as skipped when PDF has more than 3 pages', async () => {
|
||||
it('slices long PDFs to the first 3 pages and extracts, instead of skipping', async () => {
|
||||
const captured: { row?: Record<string, unknown> } = {}
|
||||
const supabase = makeSupabase(captured)
|
||||
vi.mocked(extractInvoiceFields).mockResolvedValueOnce({
|
||||
data: emptyResult(),
|
||||
rawText: 'ok',
|
||||
})
|
||||
|
||||
const req = await makeUploadRequest(6)
|
||||
const res = await uploadRoute.handler(req, buildCtx(supabase))
|
||||
const { status, body } = await parseJsonResponse<{ data: Record<string, unknown> }>(res)
|
||||
const { status, body } = await parseJsonResponse<{
|
||||
data: { extracted_data?: { pages?: { total: number; analyzed: number } } } & Record<string, unknown>
|
||||
}>(res)
|
||||
|
||||
expect(status).toBe(200)
|
||||
expect(extractInvoiceFields).not.toHaveBeenCalled()
|
||||
expect(captured.row?.extraction_skipped).toBe(true)
|
||||
expect(body.data.extraction_skipped).toBe(true)
|
||||
expect(body.data.skip_reason).toBe('too_many_pages')
|
||||
expect(extractInvoiceFields).toHaveBeenCalledOnce()
|
||||
// The buffer handed to Bedrock is the sliced copy, not the original.
|
||||
const sentBuffer = vi.mocked(extractInvoiceFields).mock.calls[0][0].buffer
|
||||
const sentPdf = await PDFDocument.load(sentBuffer)
|
||||
expect(sentPdf.getPageCount()).toBe(3)
|
||||
// The row is a normal extracted row: the truncation is recorded in
|
||||
// extracted_data.pages rather than as a skip.
|
||||
expect(captured.row?.extraction_skipped).toBe(false)
|
||||
expect(body.data.extraction_skipped).toBe(false)
|
||||
expect(body.data.skip_reason).toBeNull()
|
||||
expect(body.data.page_count).toBe(6)
|
||||
expect(body.data.extracted_data?.pages).toEqual({ total: 6, analyzed: 3 })
|
||||
})
|
||||
|
||||
it('runs extraction normally for PDFs at or below the page-count limit', async () => {
|
||||
|
||||
@@ -90,6 +90,34 @@ async function countPdfPages(buffer: ArrayBuffer): Promise<number | null> {
|
||||
}
|
||||
}
|
||||
|
||||
// Long PDFs used to skip extraction entirely (issue #553). Invoice data
|
||||
// almost always sits on the first page(s), so instead we extract from a
|
||||
// slim copy of the first MAX_PAGES_FOR_AUTO_EXTRACT pages and record the
|
||||
// truncation in extracted_data.pages. Returns null when slicing fails
|
||||
// (encrypted/malformed PDF) so the caller can fall back to the old skip.
|
||||
async function slicePdfForExtraction(
|
||||
buffer: ArrayBuffer,
|
||||
maxPages: number
|
||||
): Promise<ArrayBuffer | null> {
|
||||
try {
|
||||
const src = await PDFDocument.load(buffer, { updateMetadata: false })
|
||||
const dst = await PDFDocument.create()
|
||||
const pages = await dst.copyPages(
|
||||
src,
|
||||
Array.from({ length: Math.min(maxPages, src.getPageCount()) }, (_, i) => i)
|
||||
)
|
||||
for (const page of pages) dst.addPage(page)
|
||||
const bytes = await dst.save()
|
||||
// Copy into a fresh ArrayBuffer: Uint8Array.buffer is ArrayBufferLike
|
||||
// (possibly SharedArrayBuffer-backed) and may span more than the view.
|
||||
const out = new ArrayBuffer(bytes.byteLength)
|
||||
new Uint8Array(out).set(bytes)
|
||||
return out
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// Sandbox companies (24h anonymous demo accounts) skip the Bedrock extraction
|
||||
// pipeline entirely. The document still uploads, the inbox row still lands,
|
||||
// and the user can fill the fields in by hand, but no Claude tokens are
|
||||
@@ -261,16 +289,26 @@ async function uploadAndExtract(
|
||||
// extraction path, so the document is still stored and can be filled in
|
||||
// manually. Highest priority (a hard paywall rule, not a heuristic).
|
||||
const hasAiEntitlement = await hasCapability(supabase, companyId, CAPABILITY.ai)
|
||||
// Skip-reason priority: no-AI-entitlement > sandbox > page-count > client opt-out.
|
||||
// Long PDFs are sliced to their first pages instead of skipped, but only
|
||||
// when extraction would actually run: slicing after an entitlement/sandbox/
|
||||
// opt-out verdict would be wasted CPU.
|
||||
const slicedBuffer =
|
||||
gatedByPageCount && hasAiEntitlement && !sandbox && !opts.skipExtraction
|
||||
? await slicePdfForExtraction(file.buffer, MAX_PAGES_FOR_AUTO_EXTRACT)
|
||||
: null
|
||||
// Skip-reason priority: no-AI-entitlement > sandbox > client opt-out >
|
||||
// page-count. Opt-out now outranks the page gate (an opted-out caller never
|
||||
// extracts regardless of length), and too_many_pages only fires when the
|
||||
// slice fallback also failed (encrypted/malformed PDF).
|
||||
const skipReason: 'no_ai_entitlement' | 'too_many_pages' | 'client_opt_out' | 'sandbox' | null =
|
||||
!hasAiEntitlement
|
||||
? 'no_ai_entitlement'
|
||||
: sandbox
|
||||
? 'sandbox'
|
||||
: gatedByPageCount
|
||||
? 'too_many_pages'
|
||||
: opts.skipExtraction
|
||||
? 'client_opt_out'
|
||||
: opts.skipExtraction
|
||||
? 'client_opt_out'
|
||||
: gatedByPageCount && slicedBuffer == null
|
||||
? 'too_many_pages'
|
||||
: null
|
||||
const skipExtraction = skipReason !== null
|
||||
|
||||
@@ -282,10 +320,13 @@ async function uploadAndExtract(
|
||||
const { data: extracted, rawText } = skipExtraction
|
||||
? { data: emptyResult(), rawText: null }
|
||||
: await extractInvoiceFields({
|
||||
buffer: Buffer.from(file.buffer),
|
||||
buffer: Buffer.from(slicedBuffer ?? file.buffer),
|
||||
mimeType: file.type,
|
||||
fileName: file.name,
|
||||
})
|
||||
if (!skipExtraction && slicedBuffer != null && pageCount != null) {
|
||||
extracted.pages = { total: pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT }
|
||||
}
|
||||
|
||||
// Supplier match by org-nr, then case-insensitive name (no AI fuzz).
|
||||
let matchedSupplierId: string | null = null
|
||||
@@ -843,9 +884,10 @@ export const invoiceInboxExtension: Extension = {
|
||||
upload_source: 'file_upload',
|
||||
})
|
||||
|
||||
// Same page-count gate as /upload (issue #553): attaching a 6-page
|
||||
// sales report to an existing inbox row should not block on Bedrock.
|
||||
// Sandbox companies skip Bedrock unconditionally.
|
||||
// Same page handling as /upload (issue #553): long PDFs extract
|
||||
// from a slice of their first pages; the skip only remains for
|
||||
// unsliceable (encrypted/malformed) PDFs. Sandbox companies skip
|
||||
// Bedrock unconditionally.
|
||||
const pageCount =
|
||||
file.type === 'application/pdf' ? await countPdfPages(buffer) : null
|
||||
const gatedByPageCount =
|
||||
@@ -855,12 +897,16 @@ export const invoiceInboxExtension: Extension = {
|
||||
// skeleton; the attached document is still stored). Same paywall as
|
||||
// the shared upload path above.
|
||||
const hasAiEntitlement = await hasCapability(ctx.supabase, ctx.companyId, CAPABILITY.ai)
|
||||
const slicedBuffer =
|
||||
gatedByPageCount && hasAiEntitlement && !sandbox
|
||||
? await slicePdfForExtraction(buffer, MAX_PAGES_FOR_AUTO_EXTRACT)
|
||||
: null
|
||||
const skipReason: 'no_ai_entitlement' | 'too_many_pages' | 'sandbox' | null =
|
||||
!hasAiEntitlement
|
||||
? 'no_ai_entitlement'
|
||||
: sandbox
|
||||
? 'sandbox'
|
||||
: gatedByPageCount
|
||||
: gatedByPageCount && slicedBuffer == null
|
||||
? 'too_many_pages'
|
||||
: null
|
||||
const skipExtraction = skipReason !== null
|
||||
@@ -868,10 +914,13 @@ export const invoiceInboxExtension: Extension = {
|
||||
const { data: extracted } = skipExtraction
|
||||
? { data: emptyResult() }
|
||||
: await extractInvoiceFields({
|
||||
buffer: Buffer.from(buffer),
|
||||
buffer: Buffer.from(slicedBuffer ?? buffer),
|
||||
mimeType: file.type,
|
||||
fileName: file.name,
|
||||
})
|
||||
if (!skipExtraction && slicedBuffer != null && pageCount != null) {
|
||||
extracted.pages = { total: pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT }
|
||||
}
|
||||
|
||||
const { error: linkError } = await ctx.supabase
|
||||
.from('invoice_inbox_items')
|
||||
|
||||
@@ -54,7 +54,48 @@ export interface ExtractionOutput {
|
||||
rawText: string | null
|
||||
}
|
||||
|
||||
// Classification fields are nullable AND .catch(null): a hallucinated enum
|
||||
// value must degrade to "unknown", never fail the whole document parse. The
|
||||
// amount/date fields keep strict parsing on purpose: a malformed amount
|
||||
// SHOULD reject the output rather than store garbage.
|
||||
const DocumentKind = z
|
||||
.enum(['receipt', 'supplier_invoice', 'government_letter', 'other'])
|
||||
.nullable()
|
||||
.catch(null)
|
||||
const PaymentMethod = z
|
||||
.enum(['card', 'swish', 'cash', 'invoice', 'other'])
|
||||
.nullable()
|
||||
.catch(null)
|
||||
const MerchantCategory = z
|
||||
.enum(['restaurant', 'cafe', 'taxi', 'parking', 'fuel', 'grocery', 'hotel', 'other'])
|
||||
.nullable()
|
||||
.catch(null)
|
||||
const Legibility = z.enum(['good', 'partial', 'unreadable']).nullable().catch(null)
|
||||
|
||||
export const ExtractionSchema = z.object({
|
||||
// All optional: raw model outputs cached before these fields existed must
|
||||
// still validate (same convention as servicePeriodStart/End below). These
|
||||
// route UI emphasis and clarifying questions only; they never book anything.
|
||||
documentKind: DocumentKind.optional(),
|
||||
merchantCategory: MerchantCategory.optional(),
|
||||
legibility: Legibility.optional(),
|
||||
purchaseTime: z
|
||||
.string()
|
||||
.regex(/^([01]\d|2[0-3]):[0-5]\d$/)
|
||||
.nullable()
|
||||
.catch(null)
|
||||
.optional(),
|
||||
payment: z
|
||||
.object({
|
||||
method: PaymentMethod,
|
||||
// Length + digits-only, deliberately not the shared four-digit
|
||||
// invariant from @/lib/invariants: this is the tail of a masked card
|
||||
// number, not a BAS account and not a fiscal year.
|
||||
cardLast4: z.string().length(4).regex(/^\d+$/).nullable().catch(null),
|
||||
})
|
||||
.nullable()
|
||||
.catch(null)
|
||||
.optional(),
|
||||
supplier: z.object({
|
||||
name: z.string().nullable(),
|
||||
orgNumber: z.string().nullable(),
|
||||
@@ -100,6 +141,9 @@ export const ExtractionSchema = z.object({
|
||||
subtotal: z.number().nullable(),
|
||||
vatAmount: z.number().nullable(),
|
||||
total: z.number().nullable(),
|
||||
// Öresavrundning line on Swedish receipts (can be negative). Optional so
|
||||
// cached raw outputs from before the field still validate.
|
||||
roundingAmount: z.number().nullable().catch(null).optional(),
|
||||
}),
|
||||
vatBreakdown: z.array(
|
||||
z.object({
|
||||
@@ -131,6 +175,11 @@ const SYSTEM_PROMPT = `You extract invoice and receipt fields from a single docu
|
||||
Return ONLY a single JSON object that matches this schema exactly. No prose, no markdown fences, no commentary.
|
||||
|
||||
{
|
||||
"documentKind": "receipt" | "supplier_invoice" | "government_letter" | "other" | null,
|
||||
"merchantCategory": "restaurant" | "cafe" | "taxi" | "parking" | "fuel" | "grocery" | "hotel" | "other" | null,
|
||||
"legibility": "good" | "partial" | "unreadable",
|
||||
"purchaseTime": string | null, // "HH:MM" 24h, receipts only
|
||||
"payment": { "method": "card" | "swish" | "cash" | "invoice" | "other" | null, "cardLast4": string | null } | null,
|
||||
"supplier": {
|
||||
"name": string | null,
|
||||
"orgNumber": string | null, // 10 digits, no hyphen, only when issued by a Swedish entity
|
||||
@@ -161,7 +210,8 @@ Return ONLY a single JSON object that matches this schema exactly. No prose, no
|
||||
"totals": {
|
||||
"subtotal": number | null, // amount excluding VAT
|
||||
"vatAmount": number | null, // total VAT
|
||||
"total": number | null // amount including VAT: what the buyer pays
|
||||
"total": number | null, // amount including VAT: what the buyer actually pays
|
||||
"roundingAmount": number | null // öresavrundning line, may be negative, e.g. -0.37
|
||||
},
|
||||
"vatBreakdown": [
|
||||
{ "rate": number, "base": number, "amount": number } // rate as percent integer, e.g. 25 for 25%
|
||||
@@ -172,6 +222,12 @@ VAT rate convention: BOTH lineItems[].vatRate AND vatBreakdown[].rate use the sa
|
||||
|
||||
Rules:
|
||||
- Output JSON only. The first character must be '{' and the last must be '}'.
|
||||
- documentKind: "receipt" = point-of-sale proof of a COMPLETED payment (kassakvitto, kortkvitto, taxi/parking slip, webshop order confirmation marked paid). "supplier_invoice" = a request for payment (has due date, OCR/payment reference, bankgiro, "Att betala senast"). "government_letter" = correspondence from a myndighet (Skatteverket, Bolagsverket, Försäkringskassan...). "other" = contracts, statements, reports. null only when truly indeterminate.
|
||||
- merchantCategory: judge from the merchant name and line items (a receipt from "Prinsen" listing food and wine is "restaurant" even without the word). Use "other" when unsure. null for non-receipts.
|
||||
- legibility: "good" = all key amounts and the merchant are readable. "partial" = some key fields are cut off, blurry, or unreadable. "unreadable" = the document is mostly illegible (too blurry/dark/small). Judge the IMAGE quality, not whether fields exist on the document.
|
||||
- payment: only for documents that show how payment was made. "card" for kort/VISA/Mastercard; cardLast4 only when a masked card number like ****1234 is printed. "invoice" means the document says it will be billed separately.
|
||||
- purchaseTime: the HH:MM time printed on a receipt. null when absent.
|
||||
- Öresavrundning: Swedish receipts often show an "Avrundning"/"Öresavrundning" line. "total" is ALWAYS the amount actually paid AFTER rounding; put the rounding line in totals.roundingAmount (negative when rounded down). When present: subtotal + vatAmount + roundingAmount = total.
|
||||
- Currency: detect from the document (symbol $/€/kr or explicit code). Use the ISO 4217 code. Do NOT default to SEK if the document clearly shows another currency.
|
||||
- "total" is the amount the buyer must pay (look for "Att betala", "Total", "Amount paid", "Amount due", "Balance"). Prefer this over Subtotal.
|
||||
- Dates: convert any format to YYYY-MM-DD. If the document only shows month/year, leave null.
|
||||
@@ -186,6 +242,11 @@ Rules:
|
||||
|
||||
export function emptyResult(): InvoiceExtractionResult {
|
||||
return {
|
||||
documentKind: null,
|
||||
merchantCategory: null,
|
||||
legibility: null,
|
||||
purchaseTime: null,
|
||||
payment: null,
|
||||
supplier: {
|
||||
name: null,
|
||||
orgNumber: null,
|
||||
@@ -204,12 +265,64 @@ export function emptyResult(): InvoiceExtractionResult {
|
||||
servicePeriodEnd: null,
|
||||
},
|
||||
lineItems: [],
|
||||
totals: { subtotal: null, vatAmount: null, total: null },
|
||||
totals: { subtotal: null, vatAmount: null, total: null, roundingAmount: null },
|
||||
vatBreakdown: [],
|
||||
confidence: 0,
|
||||
}
|
||||
}
|
||||
|
||||
// Anthropic rejects images above 5 MB (decoded bytes), and 12 MP phone photos
|
||||
// routinely exceed that: before this step they errored out to an empty
|
||||
// extraction. Downscaling to ≤2000px JPEG also cuts input tokens on every
|
||||
// large image. HEIC/HEIF (iPhone default) is transcoded to JPEG when the
|
||||
// local sharp/libvips build can decode it; prebuilt binaries usually cannot
|
||||
// (patent licensing), in which case the caller falls through to the
|
||||
// unsupported-type path exactly as before.
|
||||
const IMAGE_DOWNSCALE_THRESHOLD_BYTES = 4 * 1024 * 1024
|
||||
const IMAGE_MAX_DIMENSION = 2000
|
||||
|
||||
async function normalizeImageForExtraction(
|
||||
input: ExtractionInput
|
||||
): Promise<ExtractionInput> {
|
||||
const isHeic = input.mimeType === 'image/heic' || input.mimeType === 'image/heif'
|
||||
const isLargeSupportedImage =
|
||||
input.mimeType.startsWith('image/') &&
|
||||
SUPPORTED_MEDIA_TYPES.has(input.mimeType) &&
|
||||
input.buffer.byteLength > IMAGE_DOWNSCALE_THRESHOLD_BYTES
|
||||
if (!isHeic && !isLargeSupportedImage) return input
|
||||
|
||||
try {
|
||||
// Lazy import: sharp is a native module and only a fraction of
|
||||
// extractions need it; loading it at module scope would tax every
|
||||
// cold start of the extension route bundle.
|
||||
const sharp = (await import('sharp')).default
|
||||
const converted = await sharp(input.buffer)
|
||||
// Apply the EXIF orientation before it is lost in re-encoding:
|
||||
// phone photos are routinely stored rotated.
|
||||
.rotate()
|
||||
.resize({
|
||||
width: IMAGE_MAX_DIMENSION,
|
||||
height: IMAGE_MAX_DIMENSION,
|
||||
fit: 'inside',
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.jpeg({ quality: 80 })
|
||||
.toBuffer()
|
||||
return { buffer: converted, mimeType: 'image/jpeg', fileName: input.fileName }
|
||||
} catch (err) {
|
||||
// HEIC without libheif lands here → caller hits the unsupported-type
|
||||
// guard, same net behavior as before this step existed. For oversized
|
||||
// JPEG/PNG the original buffer is still worth attempting.
|
||||
log.warn('image normalization failed', {
|
||||
file_name_hash: createHash('sha256').update(input.fileName).digest('hex').slice(0, 12),
|
||||
mime_type: input.mimeType,
|
||||
byte_length: input.buffer.byteLength,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
})
|
||||
return input
|
||||
}
|
||||
}
|
||||
|
||||
function buildContent(input: ExtractionInput) {
|
||||
const base64 = input.buffer.toString('base64')
|
||||
if (input.mimeType === 'application/pdf') {
|
||||
@@ -240,8 +353,12 @@ function buildContent(input: ExtractionInput) {
|
||||
* always returns an InvoiceExtractionResult. Empty fields are null.
|
||||
*/
|
||||
export async function extractInvoiceFields(
|
||||
input: ExtractionInput
|
||||
rawInput: ExtractionInput
|
||||
): Promise<ExtractionOutput> {
|
||||
// Transcodes HEIC when possible and downscales oversized images; a no-op
|
||||
// for PDFs and normal-sized supported images.
|
||||
const input = await normalizeImageForExtraction(rawInput)
|
||||
|
||||
if (!SUPPORTED_MEDIA_TYPES.has(input.mimeType)) {
|
||||
return { data: emptyResult(), rawText: null }
|
||||
}
|
||||
|
||||
+16
-1
@@ -2797,7 +2797,22 @@
|
||||
"document_loading": "Fetching the document…",
|
||||
"document_load_failed": "The document is still stored but could not be shown right now.",
|
||||
"email_body_label": "Email content",
|
||||
"email_body_empty": "The email had no text."
|
||||
"email_body_empty": "The email had no text.",
|
||||
"doc_kind_label": "Type",
|
||||
"doc_kind_receipt": "Receipt",
|
||||
"doc_kind_supplier_invoice": "Supplier invoice",
|
||||
"doc_kind_government_letter": "Government letter",
|
||||
"doc_kind_other": "Other document",
|
||||
"payment_label": "Paid with",
|
||||
"payment_card": "Card",
|
||||
"payment_swish": "Swish",
|
||||
"payment_cash": "Cash",
|
||||
"payment_invoice": "Invoiced",
|
||||
"payment_other": "Other",
|
||||
"pages_partial_note": "Extracted from the first {analyzed} of {total} pages.",
|
||||
"heic_hint": "HEIC images cannot be AI-extracted yet. Upload the receipt as JPEG or PDF, or fill in the fields manually.",
|
||||
"skipped_hint": "AI extraction did not run for this document. You can link the document to a transaction or create a supplier invoice manually.",
|
||||
"retry_overwrite_confirm": "Re-running extraction overwrites the fields, including your own edits. Continue?"
|
||||
},
|
||||
"inbox_custom_domain": {
|
||||
"title": "Custom inbox domain",
|
||||
|
||||
+16
-1
@@ -2797,7 +2797,22 @@
|
||||
"document_loading": "Hämtar underlaget…",
|
||||
"document_load_failed": "Underlaget finns kvar men kunde inte visas just nu.",
|
||||
"email_body_label": "Mejlets innehåll",
|
||||
"email_body_empty": "Mejlet hade ingen text."
|
||||
"email_body_empty": "Mejlet hade ingen text.",
|
||||
"doc_kind_label": "Typ",
|
||||
"doc_kind_receipt": "Kvitto",
|
||||
"doc_kind_supplier_invoice": "Leverantörsfaktura",
|
||||
"doc_kind_government_letter": "Myndighetsbrev",
|
||||
"doc_kind_other": "Övrigt dokument",
|
||||
"payment_label": "Betalsätt",
|
||||
"payment_card": "Kort",
|
||||
"payment_swish": "Swish",
|
||||
"payment_cash": "Kontant",
|
||||
"payment_invoice": "Faktureras",
|
||||
"payment_other": "Annat",
|
||||
"pages_partial_note": "Tolkad från de första {analyzed} av {total} sidorna.",
|
||||
"heic_hint": "HEIC-bilder kan inte AI-tolkas ännu. Ladda upp kvittot som JPEG eller PDF, eller fyll i fälten manuellt.",
|
||||
"skipped_hint": "AI-tolkning kördes inte för det här dokumentet. Du kan koppla dokumentet till en transaktion eller skapa leverantörsfaktura manuellt.",
|
||||
"retry_overwrite_confirm": "Ny tolkning skriver över fälten, även ändringar du gjort själv. Fortsätta?"
|
||||
},
|
||||
"inbox_custom_domain": {
|
||||
"title": "Egen domän för inkorgen",
|
||||
|
||||
@@ -3628,7 +3628,32 @@ export interface IngestResult {
|
||||
|
||||
// ── Invoice extraction (used by invoice-inbox extension and core utils) ──
|
||||
|
||||
export type ExtractedDocumentKind =
|
||||
| 'receipt'
|
||||
| 'supplier_invoice'
|
||||
| 'government_letter'
|
||||
| 'other'
|
||||
export type ExtractedPaymentMethod = 'card' | 'swish' | 'cash' | 'invoice' | 'other'
|
||||
export type ExtractedMerchantCategory =
|
||||
| 'restaurant'
|
||||
| 'cafe'
|
||||
| 'taxi'
|
||||
| 'parking'
|
||||
| 'fuel'
|
||||
| 'grocery'
|
||||
| 'hotel'
|
||||
| 'other'
|
||||
export type ExtractedLegibility = 'good' | 'partial' | 'unreadable'
|
||||
|
||||
export interface InvoiceExtractionResult {
|
||||
// Classification fields (2026-08): optional because extractions stored
|
||||
// before they existed lack them. They route UI emphasis and clarifying
|
||||
// questions only: never bookings.
|
||||
documentKind?: ExtractedDocumentKind | null
|
||||
merchantCategory?: ExtractedMerchantCategory | null
|
||||
legibility?: ExtractedLegibility | null
|
||||
purchaseTime?: string | null
|
||||
payment?: { method: ExtractedPaymentMethod | null; cardLast4: string | null } | null
|
||||
supplier: {
|
||||
name: string | null
|
||||
orgNumber: string | null
|
||||
@@ -3654,10 +3679,15 @@ export interface InvoiceExtractionResult {
|
||||
subtotal: number | null
|
||||
vatAmount: number | null
|
||||
total: number | null
|
||||
// Öresavrundning line on Swedish receipts; negative when rounded down.
|
||||
roundingAmount?: number | null
|
||||
}
|
||||
vatBreakdown: VatBreakdownItem[]
|
||||
confidence: number
|
||||
suggestedTemplateId?: string
|
||||
// Set by the caller (not the model) when a long PDF was sliced before
|
||||
// extraction: fields were read from the first `analyzed` of `total` pages.
|
||||
pages?: { total: number; analyzed: number }
|
||||
}
|
||||
|
||||
export interface ExtractedInvoiceLineItem {
|
||||
|
||||
Reference in New Issue
Block a user