diff --git a/DECISIONS.md b/DECISIONS.md index 5b2223c9..a0520975 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -769,3 +769,6 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-08-03] kompletteringsregel_20 with a positive basis and zero acquisition cohorts is refused rather than computed: reducing over an empty cohort set would claim a full write-off the cohort evidence does not support (IL 18 kap. 17 §), so such periods require manual review instead of an automatic deduction. [2026-08-04] Balance-sheet synthetic result = complement of classes 1-2 (class 0/9/null rows included), not classes 3-8: a resultatavslut posted to 2099 without zeroing class 3-8 then self-cancels inside the residual instead of double-counting equity; mirrors balansrapport's residual definition (#1333) [2026-08-04] v1 settings PATCH writes directly (no staging), following the v1 customers precedent: REST callers are already scope-gated, staging is an MCP segregation-of-duties concept. +[2026-08-01] OCR quick-fix "bounded retry on transient Bedrock errors" dropped: @anthropic-ai/bedrock-sdk already retries 429/5xx twice by default (maxRetries=2, not overridden in extract-invoice-fields.ts); duplicating it would triple worst-case latency for no reliability gain. +[2026-08-01] Page-count gate (issue #553) changed from skip-extraction to slice-first-3-pages via pdf-lib: invoice data sits on page 1, and a >3-page supplier invoice getting ZERO fields was the worse failure mode. too_many_pages skip remains only for unsliceable (encrypted/malformed) PDFs; truncation recorded in extracted_data.pages. Side effect: client_opt_out now outranks too_many_pages in skip-reason priority (an opted-out caller never extracts regardless of length). +[2026-08-01] Extraction classification fields (documentKind/payment/merchantCategory/legibility) validate with .catch(null) instead of strict enums: a hallucinated label must degrade to unknown, not sink the whole document parse; amounts keep strict parsing on purpose. HEIC handling = attempt sharp transcode at runtime, fall through to today's empty-extraction when libvips lacks HEIF (prebuilt binaries exclude it for patent reasons), with a UI hint replacing the silence. diff --git a/components/extensions/general/InvoiceInboxWorkspace.tsx b/components/extensions/general/InvoiceInboxWorkspace.tsx index 522934bb..6fedf8d0 100644 --- a/components/extensions/general/InvoiceInboxWorkspace.tsx +++ b/components/extensions/general/InvoiceInboxWorkspace.tsx @@ -1128,6 +1128,7 @@ export default function InvoiceInboxWorkspace(_props: WorkspaceComponentProps) { {selected ? ( handleDelete(selected.id)} onBookDirect={() => setBookDirectOpen(true)} @@ -1741,6 +1742,7 @@ function EmptyPreview({ function FieldsRail({ item, + docMime, accountingMethod, onDelete, onBookDirect, @@ -1753,6 +1755,7 @@ function FieldsRail({ onRetryRequested, }: { item: InboxItem + docMime: string | null accountingMethod: AccountingMethod onDelete: () => void onBookDirect: () => void @@ -1787,6 +1790,9 @@ function FieldsRail({ !!extractedSupplierName const handleRetry = async () => { + // Retry overwrites extracted_data wholesale server-side, including any + // manual field edits: make the user opt into that loss explicitly. + if (hasAnyExtractedField(data) && !confirm(t('retry_overwrite_confirm'))) return setIsRetrying(true) try { const res = await fetch( @@ -1835,6 +1841,38 @@ function FieldsRail({ )} + {/* AI classification: what kind of document this is and how it was + paid. Read-only context above the editable fields; absent for + extractions from before the fields existed. */} + {(data?.documentKind || data?.payment?.method || data?.pages) && ( +
+ {data?.documentKind && ( +
+ {t('doc_kind_label')} + {t(`doc_kind_${data.documentKind}`)} +
+ )} + {data?.payment?.method && ( +
+ {t('payment_label')} + + {t(`payment_${data.payment.method}`)} + {data.payment.cardLast4 ? ` •• ${data.payment.cardLast4}` : ''} + {data.purchaseTime ? ` · ${data.purchaseTime}` : ''} + +
+ )} + {data?.pages && ( +
+ {t('pages_partial_note', { + analyzed: data.pages.analyzed, + total: data.pages.total, + })} +
+ )} +
+ )} + {item.error_message && (
@@ -1894,15 +1932,28 @@ function FieldsRail({ )} {/* Skipped-extraction hint: explains the empty fields and points the - user to the manual paths (transaction link or supplier invoice). */} + user to the manual paths (transaction link or supplier invoice). + Deliberately reason-agnostic: skip covers sandbox, BYO-extraction + and unsliceable PDFs, not just page count anymore. */} {item.extraction_skipped && !isResolved && (
- AI-tolkning skippades p.g.a. dokumentets storlek (fler än 3 sidor). - Du kan koppla dokumentet till en transaktion eller skapa - leverantörsfaktura manuellt. + {t('skipped_hint')}
)} + {/* HEIC: extraction ran but Bedrock cannot read the format, so every + field came back empty with no error. Tell the user why instead of + leaving a silently blank rail (iPhone photos default to HEIC). */} + {!item.extraction_skipped && + !isResolved && + hasAi && + (docMime === 'image/heic' || docMime === 'image/heif') && + !hasAnyExtractedField(data) && ( +
+ {t('heic_hint')} +
+ )} + {/* Extracted fields */}

diff --git a/extensions/general/invoice-inbox/__tests__/extract-invoice-fields.test.ts b/extensions/general/invoice-inbox/__tests__/extract-invoice-fields.test.ts index 5e70a12f..41be5c9c 100644 --- a/extensions/general/invoice-inbox/__tests__/extract-invoice-fields.test.ts +++ b/extensions/general/invoice-inbox/__tests__/extract-invoice-fields.test.ts @@ -12,6 +12,25 @@ vi.mock('@anthropic-ai/bedrock-sdk', () => { return { default: FakeBedrock } }) +// sharp is imported lazily by normalizeImageForExtraction. The default mock +// (no implementation → TypeError on .rotate()) mimics a build without HEIF +// support: normalization fails, the original buffer flows on. Individual +// tests install a working chain via sharpMock.mockImplementationOnce. +const sharpMock = vi.fn() +vi.mock('sharp', () => ({ + default: (...args: unknown[]) => sharpMock(...args), +})) + +function workingSharpChain(outputBuffer: Buffer) { + const chain = { + rotate: vi.fn(() => chain), + resize: vi.fn(() => chain), + jpeg: vi.fn(() => chain), + toBuffer: vi.fn().mockResolvedValue(outputBuffer), + } + return chain +} + const ORIG_AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID const ORIG_AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY @@ -182,6 +201,134 @@ describe('extractInvoiceFields', () => { expect(data.lineItems[0].accountSuggestion).toBeNull() }) + // ── Receipt-aware classification fields (2026-08) ────────── + + it('parses the classification fields when the model returns them', async () => { + mockCreate.mockReturnValueOnce( + aiResponse({ + ...VALID_RESULT, + documentKind: 'receipt', + merchantCategory: 'restaurant', + legibility: 'good', + purchaseTime: '12:41', + payment: { method: 'card', cardLast4: '1234' }, + totals: { ...VALID_RESULT.totals, roundingAmount: -0.25 }, + }) + ) + const { data } = await extractInvoiceFields({ + buffer: Buffer.from('%PDF'), + mimeType: 'application/pdf', + fileName: 'kvitto.pdf', + }) + expect(data.documentKind).toBe('receipt') + expect(data.merchantCategory).toBe('restaurant') + expect(data.legibility).toBe('good') + expect(data.purchaseTime).toBe('12:41') + expect(data.payment).toEqual({ method: 'card', cardLast4: '1234' }) + expect(data.totals.roundingAmount).toBe(-0.25) + }) + + it('still parses cached outputs from before the classification fields existed', async () => { + // VALID_RESULT has none of the new fields: the whole document must + // validate, not fall back to the empty result. + mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT)) + const { data } = await extractInvoiceFields({ + buffer: Buffer.from('%PDF'), + mimeType: 'application/pdf', + fileName: 'old.pdf', + }) + expect(data.supplier.name).toBe('Anthropic, PBC') + expect(data.documentKind).toBeUndefined() + }) + + it('degrades hallucinated classification values to null instead of failing the parse', async () => { + mockCreate.mockReturnValueOnce( + aiResponse({ + ...VALID_RESULT, + documentKind: 'parking_ticket', + merchantCategory: 'nightclub', + legibility: 'excellent', + purchaseTime: '25:99', + payment: { method: 'bitcoin', cardLast4: 'abcd' }, + totals: { ...VALID_RESULT.totals, roundingAmount: 'noll' }, + }) + ) + const { data } = await extractInvoiceFields({ + buffer: Buffer.from('%PDF'), + mimeType: 'application/pdf', + fileName: 'kvitto.pdf', + }) + // Amounts survived: the junk classification did not sink the document. + expect(data.totals.total).toBe(6.25) + expect(data.documentKind).toBeNull() + expect(data.merchantCategory).toBeNull() + expect(data.legibility).toBeNull() + expect(data.purchaseTime).toBeNull() + expect(data.payment).toEqual({ method: null, cardLast4: null }) + expect(data.totals.roundingAmount).toBeNull() + }) + + // ── Image normalization (HEIC transcode + oversized downscale) ── + + it('transcodes HEIC to JPEG and extracts when sharp can decode it', async () => { + const converted = Buffer.from('converted-jpeg-bytes') + sharpMock.mockImplementationOnce(() => workingSharpChain(converted)) + mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT)) + + const { data } = await extractInvoiceFields({ + buffer: Buffer.from('heic-bytes'), + mimeType: 'image/heic', + fileName: 'photo.heic', + }) + + expect(mockCreate).toHaveBeenCalledOnce() + const content = mockCreate.mock.calls[0][0].messages[0].content + expect(content[0].source.media_type).toBe('image/jpeg') + expect(content[0].source.data).toBe(converted.toString('base64')) + expect(data.supplier.name).toBe('Anthropic, PBC') + }) + + it('downscales oversized JPEGs before sending to Bedrock', async () => { + const converted = Buffer.from('downscaled-jpeg') + sharpMock.mockImplementationOnce(() => workingSharpChain(converted)) + mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT)) + + await extractInvoiceFields({ + buffer: Buffer.alloc(5 * 1024 * 1024, 1), + mimeType: 'image/jpeg', + fileName: 'big-photo.jpg', + }) + + expect(sharpMock).toHaveBeenCalledOnce() + const content = mockCreate.mock.calls[0][0].messages[0].content + expect(content[0].source.data).toBe(converted.toString('base64')) + }) + + it('keeps the original buffer when downscaling an oversized image fails', async () => { + // Default sharpMock throws: the original 5 MB buffer is still attempted. + mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT)) + + await extractInvoiceFields({ + buffer: Buffer.alloc(5 * 1024 * 1024, 1), + mimeType: 'image/jpeg', + fileName: 'big-photo.jpg', + }) + + expect(mockCreate).toHaveBeenCalledOnce() + const content = mockCreate.mock.calls[0][0].messages[0].content + expect(content[0].source.media_type).toBe('image/jpeg') + }) + + it('does not invoke sharp for normal-sized supported images', async () => { + mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT)) + await extractInvoiceFields({ + buffer: Buffer.from('JPEG'), + mimeType: 'image/jpeg', + fileName: 'photo.jpg', + }) + expect(sharpMock).not.toHaveBeenCalled() + }) + // Restore env vars so other test files aren't affected. afterAll(() => { if (ORIG_AWS_ACCESS_KEY_ID) process.env.AWS_ACCESS_KEY_ID = ORIG_AWS_ACCESS_KEY_ID diff --git a/extensions/general/invoice-inbox/__tests__/upload-page-count-gate.test.ts b/extensions/general/invoice-inbox/__tests__/upload-page-count-gate.test.ts index 5e9a3c38..9319ef57 100644 --- a/extensions/general/invoice-inbox/__tests__/upload-page-count-gate.test.ts +++ b/extensions/general/invoice-inbox/__tests__/upload-page-count-gate.test.ts @@ -121,20 +121,33 @@ beforeEach(() => { }) describe('POST /upload: page-count gate (issue #553)', () => { - it('skips extraction and marks the row as skipped when PDF has more than 3 pages', async () => { + it('slices long PDFs to the first 3 pages and extracts, instead of skipping', async () => { const captured: { row?: Record } = {} const supabase = makeSupabase(captured) + vi.mocked(extractInvoiceFields).mockResolvedValueOnce({ + data: emptyResult(), + rawText: 'ok', + }) const req = await makeUploadRequest(6) const res = await uploadRoute.handler(req, buildCtx(supabase)) - const { status, body } = await parseJsonResponse<{ data: Record }>(res) + const { status, body } = await parseJsonResponse<{ + data: { extracted_data?: { pages?: { total: number; analyzed: number } } } & Record + }>(res) expect(status).toBe(200) - expect(extractInvoiceFields).not.toHaveBeenCalled() - expect(captured.row?.extraction_skipped).toBe(true) - expect(body.data.extraction_skipped).toBe(true) - expect(body.data.skip_reason).toBe('too_many_pages') + expect(extractInvoiceFields).toHaveBeenCalledOnce() + // The buffer handed to Bedrock is the sliced copy, not the original. + const sentBuffer = vi.mocked(extractInvoiceFields).mock.calls[0][0].buffer + const sentPdf = await PDFDocument.load(sentBuffer) + expect(sentPdf.getPageCount()).toBe(3) + // The row is a normal extracted row: the truncation is recorded in + // extracted_data.pages rather than as a skip. + expect(captured.row?.extraction_skipped).toBe(false) + expect(body.data.extraction_skipped).toBe(false) + expect(body.data.skip_reason).toBeNull() expect(body.data.page_count).toBe(6) + expect(body.data.extracted_data?.pages).toEqual({ total: 6, analyzed: 3 }) }) it('runs extraction normally for PDFs at or below the page-count limit', async () => { diff --git a/extensions/general/invoice-inbox/index.ts b/extensions/general/invoice-inbox/index.ts index 1e8a1e6b..dd77fe47 100644 --- a/extensions/general/invoice-inbox/index.ts +++ b/extensions/general/invoice-inbox/index.ts @@ -90,6 +90,34 @@ async function countPdfPages(buffer: ArrayBuffer): Promise { } } +// Long PDFs used to skip extraction entirely (issue #553). Invoice data +// almost always sits on the first page(s), so instead we extract from a +// slim copy of the first MAX_PAGES_FOR_AUTO_EXTRACT pages and record the +// truncation in extracted_data.pages. Returns null when slicing fails +// (encrypted/malformed PDF) so the caller can fall back to the old skip. +async function slicePdfForExtraction( + buffer: ArrayBuffer, + maxPages: number +): Promise { + try { + const src = await PDFDocument.load(buffer, { updateMetadata: false }) + const dst = await PDFDocument.create() + const pages = await dst.copyPages( + src, + Array.from({ length: Math.min(maxPages, src.getPageCount()) }, (_, i) => i) + ) + for (const page of pages) dst.addPage(page) + const bytes = await dst.save() + // Copy into a fresh ArrayBuffer: Uint8Array.buffer is ArrayBufferLike + // (possibly SharedArrayBuffer-backed) and may span more than the view. + const out = new ArrayBuffer(bytes.byteLength) + new Uint8Array(out).set(bytes) + return out + } catch { + return null + } +} + // Sandbox companies (24h anonymous demo accounts) skip the Bedrock extraction // pipeline entirely. The document still uploads, the inbox row still lands, // and the user can fill the fields in by hand, but no Claude tokens are @@ -261,16 +289,26 @@ async function uploadAndExtract( // extraction path, so the document is still stored and can be filled in // manually. Highest priority (a hard paywall rule, not a heuristic). const hasAiEntitlement = await hasCapability(supabase, companyId, CAPABILITY.ai) - // Skip-reason priority: no-AI-entitlement > sandbox > page-count > client opt-out. + // Long PDFs are sliced to their first pages instead of skipped, but only + // when extraction would actually run: slicing after an entitlement/sandbox/ + // opt-out verdict would be wasted CPU. + const slicedBuffer = + gatedByPageCount && hasAiEntitlement && !sandbox && !opts.skipExtraction + ? await slicePdfForExtraction(file.buffer, MAX_PAGES_FOR_AUTO_EXTRACT) + : null + // Skip-reason priority: no-AI-entitlement > sandbox > client opt-out > + // page-count. Opt-out now outranks the page gate (an opted-out caller never + // extracts regardless of length), and too_many_pages only fires when the + // slice fallback also failed (encrypted/malformed PDF). const skipReason: 'no_ai_entitlement' | 'too_many_pages' | 'client_opt_out' | 'sandbox' | null = !hasAiEntitlement ? 'no_ai_entitlement' : sandbox ? 'sandbox' - : gatedByPageCount - ? 'too_many_pages' - : opts.skipExtraction - ? 'client_opt_out' + : opts.skipExtraction + ? 'client_opt_out' + : gatedByPageCount && slicedBuffer == null + ? 'too_many_pages' : null const skipExtraction = skipReason !== null @@ -282,10 +320,13 @@ async function uploadAndExtract( const { data: extracted, rawText } = skipExtraction ? { data: emptyResult(), rawText: null } : await extractInvoiceFields({ - buffer: Buffer.from(file.buffer), + buffer: Buffer.from(slicedBuffer ?? file.buffer), mimeType: file.type, fileName: file.name, }) + if (!skipExtraction && slicedBuffer != null && pageCount != null) { + extracted.pages = { total: pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT } + } // Supplier match by org-nr, then case-insensitive name (no AI fuzz). let matchedSupplierId: string | null = null @@ -843,9 +884,10 @@ export const invoiceInboxExtension: Extension = { upload_source: 'file_upload', }) - // Same page-count gate as /upload (issue #553): attaching a 6-page - // sales report to an existing inbox row should not block on Bedrock. - // Sandbox companies skip Bedrock unconditionally. + // Same page handling as /upload (issue #553): long PDFs extract + // from a slice of their first pages; the skip only remains for + // unsliceable (encrypted/malformed) PDFs. Sandbox companies skip + // Bedrock unconditionally. const pageCount = file.type === 'application/pdf' ? await countPdfPages(buffer) : null const gatedByPageCount = @@ -855,12 +897,16 @@ export const invoiceInboxExtension: Extension = { // skeleton; the attached document is still stored). Same paywall as // the shared upload path above. const hasAiEntitlement = await hasCapability(ctx.supabase, ctx.companyId, CAPABILITY.ai) + const slicedBuffer = + gatedByPageCount && hasAiEntitlement && !sandbox + ? await slicePdfForExtraction(buffer, MAX_PAGES_FOR_AUTO_EXTRACT) + : null const skipReason: 'no_ai_entitlement' | 'too_many_pages' | 'sandbox' | null = !hasAiEntitlement ? 'no_ai_entitlement' : sandbox ? 'sandbox' - : gatedByPageCount + : gatedByPageCount && slicedBuffer == null ? 'too_many_pages' : null const skipExtraction = skipReason !== null @@ -868,10 +914,13 @@ export const invoiceInboxExtension: Extension = { const { data: extracted } = skipExtraction ? { data: emptyResult() } : await extractInvoiceFields({ - buffer: Buffer.from(buffer), + buffer: Buffer.from(slicedBuffer ?? buffer), mimeType: file.type, fileName: file.name, }) + if (!skipExtraction && slicedBuffer != null && pageCount != null) { + extracted.pages = { total: pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT } + } const { error: linkError } = await ctx.supabase .from('invoice_inbox_items') diff --git a/extensions/general/invoice-inbox/lib/extract-invoice-fields.ts b/extensions/general/invoice-inbox/lib/extract-invoice-fields.ts index eb7ded11..f40ba648 100644 --- a/extensions/general/invoice-inbox/lib/extract-invoice-fields.ts +++ b/extensions/general/invoice-inbox/lib/extract-invoice-fields.ts @@ -54,7 +54,48 @@ export interface ExtractionOutput { rawText: string | null } +// Classification fields are nullable AND .catch(null): a hallucinated enum +// value must degrade to "unknown", never fail the whole document parse. The +// amount/date fields keep strict parsing on purpose: a malformed amount +// SHOULD reject the output rather than store garbage. +const DocumentKind = z + .enum(['receipt', 'supplier_invoice', 'government_letter', 'other']) + .nullable() + .catch(null) +const PaymentMethod = z + .enum(['card', 'swish', 'cash', 'invoice', 'other']) + .nullable() + .catch(null) +const MerchantCategory = z + .enum(['restaurant', 'cafe', 'taxi', 'parking', 'fuel', 'grocery', 'hotel', 'other']) + .nullable() + .catch(null) +const Legibility = z.enum(['good', 'partial', 'unreadable']).nullable().catch(null) + export const ExtractionSchema = z.object({ + // All optional: raw model outputs cached before these fields existed must + // still validate (same convention as servicePeriodStart/End below). These + // route UI emphasis and clarifying questions only; they never book anything. + documentKind: DocumentKind.optional(), + merchantCategory: MerchantCategory.optional(), + legibility: Legibility.optional(), + purchaseTime: z + .string() + .regex(/^([01]\d|2[0-3]):[0-5]\d$/) + .nullable() + .catch(null) + .optional(), + payment: z + .object({ + method: PaymentMethod, + // Length + digits-only, deliberately not the shared four-digit + // invariant from @/lib/invariants: this is the tail of a masked card + // number, not a BAS account and not a fiscal year. + cardLast4: z.string().length(4).regex(/^\d+$/).nullable().catch(null), + }) + .nullable() + .catch(null) + .optional(), supplier: z.object({ name: z.string().nullable(), orgNumber: z.string().nullable(), @@ -100,6 +141,9 @@ export const ExtractionSchema = z.object({ subtotal: z.number().nullable(), vatAmount: z.number().nullable(), total: z.number().nullable(), + // Öresavrundning line on Swedish receipts (can be negative). Optional so + // cached raw outputs from before the field still validate. + roundingAmount: z.number().nullable().catch(null).optional(), }), vatBreakdown: z.array( z.object({ @@ -131,6 +175,11 @@ const SYSTEM_PROMPT = `You extract invoice and receipt fields from a single docu Return ONLY a single JSON object that matches this schema exactly. No prose, no markdown fences, no commentary. { + "documentKind": "receipt" | "supplier_invoice" | "government_letter" | "other" | null, + "merchantCategory": "restaurant" | "cafe" | "taxi" | "parking" | "fuel" | "grocery" | "hotel" | "other" | null, + "legibility": "good" | "partial" | "unreadable", + "purchaseTime": string | null, // "HH:MM" 24h, receipts only + "payment": { "method": "card" | "swish" | "cash" | "invoice" | "other" | null, "cardLast4": string | null } | null, "supplier": { "name": string | null, "orgNumber": string | null, // 10 digits, no hyphen, only when issued by a Swedish entity @@ -161,7 +210,8 @@ Return ONLY a single JSON object that matches this schema exactly. No prose, no "totals": { "subtotal": number | null, // amount excluding VAT "vatAmount": number | null, // total VAT - "total": number | null // amount including VAT: what the buyer pays + "total": number | null, // amount including VAT: what the buyer actually pays + "roundingAmount": number | null // öresavrundning line, may be negative, e.g. -0.37 }, "vatBreakdown": [ { "rate": number, "base": number, "amount": number } // rate as percent integer, e.g. 25 for 25% @@ -172,6 +222,12 @@ VAT rate convention: BOTH lineItems[].vatRate AND vatBreakdown[].rate use the sa Rules: - Output JSON only. The first character must be '{' and the last must be '}'. +- documentKind: "receipt" = point-of-sale proof of a COMPLETED payment (kassakvitto, kortkvitto, taxi/parking slip, webshop order confirmation marked paid). "supplier_invoice" = a request for payment (has due date, OCR/payment reference, bankgiro, "Att betala senast"). "government_letter" = correspondence from a myndighet (Skatteverket, Bolagsverket, Försäkringskassan...). "other" = contracts, statements, reports. null only when truly indeterminate. +- merchantCategory: judge from the merchant name and line items (a receipt from "Prinsen" listing food and wine is "restaurant" even without the word). Use "other" when unsure. null for non-receipts. +- legibility: "good" = all key amounts and the merchant are readable. "partial" = some key fields are cut off, blurry, or unreadable. "unreadable" = the document is mostly illegible (too blurry/dark/small). Judge the IMAGE quality, not whether fields exist on the document. +- payment: only for documents that show how payment was made. "card" for kort/VISA/Mastercard; cardLast4 only when a masked card number like ****1234 is printed. "invoice" means the document says it will be billed separately. +- purchaseTime: the HH:MM time printed on a receipt. null when absent. +- Öresavrundning: Swedish receipts often show an "Avrundning"/"Öresavrundning" line. "total" is ALWAYS the amount actually paid AFTER rounding; put the rounding line in totals.roundingAmount (negative when rounded down). When present: subtotal + vatAmount + roundingAmount = total. - Currency: detect from the document (symbol $/€/kr or explicit code). Use the ISO 4217 code. Do NOT default to SEK if the document clearly shows another currency. - "total" is the amount the buyer must pay (look for "Att betala", "Total", "Amount paid", "Amount due", "Balance"). Prefer this over Subtotal. - Dates: convert any format to YYYY-MM-DD. If the document only shows month/year, leave null. @@ -186,6 +242,11 @@ Rules: export function emptyResult(): InvoiceExtractionResult { return { + documentKind: null, + merchantCategory: null, + legibility: null, + purchaseTime: null, + payment: null, supplier: { name: null, orgNumber: null, @@ -204,12 +265,64 @@ export function emptyResult(): InvoiceExtractionResult { servicePeriodEnd: null, }, lineItems: [], - totals: { subtotal: null, vatAmount: null, total: null }, + totals: { subtotal: null, vatAmount: null, total: null, roundingAmount: null }, vatBreakdown: [], confidence: 0, } } +// Anthropic rejects images above 5 MB (decoded bytes), and 12 MP phone photos +// routinely exceed that: before this step they errored out to an empty +// extraction. Downscaling to ≤2000px JPEG also cuts input tokens on every +// large image. HEIC/HEIF (iPhone default) is transcoded to JPEG when the +// local sharp/libvips build can decode it; prebuilt binaries usually cannot +// (patent licensing), in which case the caller falls through to the +// unsupported-type path exactly as before. +const IMAGE_DOWNSCALE_THRESHOLD_BYTES = 4 * 1024 * 1024 +const IMAGE_MAX_DIMENSION = 2000 + +async function normalizeImageForExtraction( + input: ExtractionInput +): Promise { + const isHeic = input.mimeType === 'image/heic' || input.mimeType === 'image/heif' + const isLargeSupportedImage = + input.mimeType.startsWith('image/') && + SUPPORTED_MEDIA_TYPES.has(input.mimeType) && + input.buffer.byteLength > IMAGE_DOWNSCALE_THRESHOLD_BYTES + if (!isHeic && !isLargeSupportedImage) return input + + try { + // Lazy import: sharp is a native module and only a fraction of + // extractions need it; loading it at module scope would tax every + // cold start of the extension route bundle. + const sharp = (await import('sharp')).default + const converted = await sharp(input.buffer) + // Apply the EXIF orientation before it is lost in re-encoding: + // phone photos are routinely stored rotated. + .rotate() + .resize({ + width: IMAGE_MAX_DIMENSION, + height: IMAGE_MAX_DIMENSION, + fit: 'inside', + withoutEnlargement: true, + }) + .jpeg({ quality: 80 }) + .toBuffer() + return { buffer: converted, mimeType: 'image/jpeg', fileName: input.fileName } + } catch (err) { + // HEIC without libheif lands here → caller hits the unsupported-type + // guard, same net behavior as before this step existed. For oversized + // JPEG/PNG the original buffer is still worth attempting. + log.warn('image normalization failed', { + file_name_hash: createHash('sha256').update(input.fileName).digest('hex').slice(0, 12), + mime_type: input.mimeType, + byte_length: input.buffer.byteLength, + error: err instanceof Error ? err.message : String(err), + }) + return input + } +} + function buildContent(input: ExtractionInput) { const base64 = input.buffer.toString('base64') if (input.mimeType === 'application/pdf') { @@ -240,8 +353,12 @@ function buildContent(input: ExtractionInput) { * always returns an InvoiceExtractionResult. Empty fields are null. */ export async function extractInvoiceFields( - input: ExtractionInput + rawInput: ExtractionInput ): Promise { + // Transcodes HEIC when possible and downscales oversized images; a no-op + // for PDFs and normal-sized supported images. + const input = await normalizeImageForExtraction(rawInput) + if (!SUPPORTED_MEDIA_TYPES.has(input.mimeType)) { return { data: emptyResult(), rawText: null } } diff --git a/messages/en.json b/messages/en.json index 47569d0e..91677012 100644 --- a/messages/en.json +++ b/messages/en.json @@ -2797,7 +2797,22 @@ "document_loading": "Fetching the document…", "document_load_failed": "The document is still stored but could not be shown right now.", "email_body_label": "Email content", - "email_body_empty": "The email had no text." + "email_body_empty": "The email had no text.", + "doc_kind_label": "Type", + "doc_kind_receipt": "Receipt", + "doc_kind_supplier_invoice": "Supplier invoice", + "doc_kind_government_letter": "Government letter", + "doc_kind_other": "Other document", + "payment_label": "Paid with", + "payment_card": "Card", + "payment_swish": "Swish", + "payment_cash": "Cash", + "payment_invoice": "Invoiced", + "payment_other": "Other", + "pages_partial_note": "Extracted from the first {analyzed} of {total} pages.", + "heic_hint": "HEIC images cannot be AI-extracted yet. Upload the receipt as JPEG or PDF, or fill in the fields manually.", + "skipped_hint": "AI extraction did not run for this document. You can link the document to a transaction or create a supplier invoice manually.", + "retry_overwrite_confirm": "Re-running extraction overwrites the fields, including your own edits. Continue?" }, "inbox_custom_domain": { "title": "Custom inbox domain", diff --git a/messages/sv.json b/messages/sv.json index 2ee96581..300ce381 100644 --- a/messages/sv.json +++ b/messages/sv.json @@ -2797,7 +2797,22 @@ "document_loading": "Hämtar underlaget…", "document_load_failed": "Underlaget finns kvar men kunde inte visas just nu.", "email_body_label": "Mejlets innehåll", - "email_body_empty": "Mejlet hade ingen text." + "email_body_empty": "Mejlet hade ingen text.", + "doc_kind_label": "Typ", + "doc_kind_receipt": "Kvitto", + "doc_kind_supplier_invoice": "Leverantörsfaktura", + "doc_kind_government_letter": "Myndighetsbrev", + "doc_kind_other": "Övrigt dokument", + "payment_label": "Betalsätt", + "payment_card": "Kort", + "payment_swish": "Swish", + "payment_cash": "Kontant", + "payment_invoice": "Faktureras", + "payment_other": "Annat", + "pages_partial_note": "Tolkad från de första {analyzed} av {total} sidorna.", + "heic_hint": "HEIC-bilder kan inte AI-tolkas ännu. Ladda upp kvittot som JPEG eller PDF, eller fyll i fälten manuellt.", + "skipped_hint": "AI-tolkning kördes inte för det här dokumentet. Du kan koppla dokumentet till en transaktion eller skapa leverantörsfaktura manuellt.", + "retry_overwrite_confirm": "Ny tolkning skriver över fälten, även ändringar du gjort själv. Fortsätta?" }, "inbox_custom_domain": { "title": "Egen domän för inkorgen", diff --git a/types/index.ts b/types/index.ts index 0bfda2a7..92f2f3ce 100644 --- a/types/index.ts +++ b/types/index.ts @@ -3628,7 +3628,32 @@ export interface IngestResult { // ── Invoice extraction (used by invoice-inbox extension and core utils) ── +export type ExtractedDocumentKind = + | 'receipt' + | 'supplier_invoice' + | 'government_letter' + | 'other' +export type ExtractedPaymentMethod = 'card' | 'swish' | 'cash' | 'invoice' | 'other' +export type ExtractedMerchantCategory = + | 'restaurant' + | 'cafe' + | 'taxi' + | 'parking' + | 'fuel' + | 'grocery' + | 'hotel' + | 'other' +export type ExtractedLegibility = 'good' | 'partial' | 'unreadable' + export interface InvoiceExtractionResult { + // Classification fields (2026-08): optional because extractions stored + // before they existed lack them. They route UI emphasis and clarifying + // questions only: never bookings. + documentKind?: ExtractedDocumentKind | null + merchantCategory?: ExtractedMerchantCategory | null + legibility?: ExtractedLegibility | null + purchaseTime?: string | null + payment?: { method: ExtractedPaymentMethod | null; cardLast4: string | null } | null supplier: { name: string | null orgNumber: string | null @@ -3654,10 +3679,15 @@ export interface InvoiceExtractionResult { subtotal: number | null vatAmount: number | null total: number | null + // Öresavrundning line on Swedish receipts; negative when rounded down. + roundingAmount?: number | null } vatBreakdown: VatBreakdownItem[] confidence: number suggestedTemplateId?: string + // Set by the caller (not the model) when a long PDF was sliced before + // extraction: fields were read from the first `analyzed` of `total` pages. + pages?: { total: number; analyzed: number } } export interface ExtractedInvoiceLineItem {