fix(invoice-inbox): read whole PDFs (last-page slice + truncation retry) (#2014)
* fix(invoice-inbox): read whole PDFs (last-page slice + truncation retry) PDF extraction read only part of well-structured PDFs, two confirmed mechanisms (21-day prod window: 49 sliced docs, 29 silent empties): - The auto-extract page budget was 3 (Bedrock-latency legacy, issue #553) and the slice kept only the first pages, so multi-page invoices lost the final page where totals, OCR and 'Att betala' sit. The budget is now 8 on pdf-native backends (Claude reads PDFs directly); the slice always keeps the last page. Rasterizing self-host backends keep the old budget of 3. - A max_tokens-truncated model answer was parsed as-is, failed, and became an all-null extraction with no trace. extractFromDocument now reports stop_reason max_tokens / finish_reason length as truncated; the extractor retries once at double AI_EXTRACTION_MAX_TOKENS and logs ai_extraction_truncated either way. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012xosyW53HUa9JoFiDayhSk * fix(invoice-inbox): sweep cutoff covers the slower two-call extraction Skeptic finding on #2014: the crash-recovery sweep flipped 'processing' rows to an empty skeleton after 2 minutes, but a deferred extraction can now legitimately run 3-5 minutes (8 native pages plus one truncation retry at a doubled token cap), so the sweep stole the row and the CAS discarded the worker's real result. Cutoff raised to 10 minutes. Also: pages_partial_note made period-agnostic (old rows were extracted from first-pages-only slices, so naming the last page was retroactively wrong for them), and two stale first-pages-only comments updated. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012xosyW53HUa9JoFiDayhSk * fix(invoice-inbox): keep the first extraction response when the retry throws CodeRabbit finding on #2014: a throttled/failed retry call bubbled to the outer catch before rawText was assigned, discarding a first response whose text may parse fine despite the truncation flag. The retry is now caught locally (logged as ai_extraction_retry_failed) and the first result flows on. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012xosyW53HUa9JoFiDayhSk --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Mattsson
co-authored by
Claude Fable 5
Claude Fable 5
parent
e8aa0670ca
commit
a1cafe495f
@@ -273,6 +273,80 @@ describe('extractInvoiceFields', () => {
|
||||
expect(extractJsonObject(oversized)).toBe(oversized)
|
||||
})
|
||||
|
||||
// ── max_tokens truncation retry (2026-08) ──────────────────
|
||||
// Line-item-heavy documents can blow the output cap; the truncated JSON
|
||||
// used to parse to nothing and look like an unreadable document.
|
||||
|
||||
it('retries once with a doubled cap when the output was truncated at max_tokens', async () => {
|
||||
mockCreate
|
||||
.mockResolvedValueOnce({
|
||||
content: [{ type: 'text', text: JSON.stringify(VALID_RESULT).slice(0, 40) }],
|
||||
stop_reason: 'max_tokens',
|
||||
})
|
||||
.mockReturnValueOnce(aiResponse(VALID_RESULT))
|
||||
const { data } = await extractInvoiceFields({
|
||||
buffer: Buffer.from('%PDF'),
|
||||
mimeType: 'application/pdf',
|
||||
fileName: 'many-line-items.pdf',
|
||||
})
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2)
|
||||
const firstMax = mockCreate.mock.calls[0][0].max_tokens
|
||||
expect(mockCreate.mock.calls[1][0].max_tokens).toBe(firstMax * 2)
|
||||
expect(data.totals.total).toBe(6.25)
|
||||
expect(data.confidence).toBe(1)
|
||||
})
|
||||
|
||||
it('falls back to the empty result when the retry is truncated too', async () => {
|
||||
mockCreate
|
||||
.mockResolvedValueOnce({
|
||||
content: [{ type: 'text', text: '{"lineItems":[{"desc' }],
|
||||
stop_reason: 'max_tokens',
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
content: [{ type: 'text', text: '{"lineItems":[{"description":"still cut' }],
|
||||
stop_reason: 'max_tokens',
|
||||
})
|
||||
const { data, rawText } = await extractInvoiceFields({
|
||||
buffer: Buffer.from('%PDF'),
|
||||
mimeType: 'application/pdf',
|
||||
fileName: 'many-line-items.pdf',
|
||||
})
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2)
|
||||
expect(rawText).toBe('{"lineItems":[{"description":"still cut')
|
||||
expect(data.totals.total).toBeNull()
|
||||
expect(data.confidence).toBe(0)
|
||||
})
|
||||
|
||||
it('keeps the first response when the retry call itself throws', async () => {
|
||||
// The truncated first answer happens to be complete valid JSON (the flag
|
||||
// can fire on the last token); a throttled retry must not discard it.
|
||||
mockCreate
|
||||
.mockResolvedValueOnce({
|
||||
content: [{ type: 'text', text: JSON.stringify(VALID_RESULT) }],
|
||||
stop_reason: 'max_tokens',
|
||||
})
|
||||
.mockRejectedValueOnce(new Error('throttled'))
|
||||
const { data, rawText } = await extractInvoiceFields({
|
||||
buffer: Buffer.from('%PDF'),
|
||||
mimeType: 'application/pdf',
|
||||
fileName: 'many-line-items.pdf',
|
||||
})
|
||||
expect(mockCreate).toHaveBeenCalledTimes(2)
|
||||
expect(rawText).toBe(JSON.stringify(VALID_RESULT))
|
||||
expect(data.totals.total).toBe(6.25)
|
||||
expect(data.confidence).toBe(1)
|
||||
})
|
||||
|
||||
it('does not retry when the answer completed under the cap', async () => {
|
||||
mockCreate.mockReturnValueOnce(aiResponse(VALID_RESULT))
|
||||
await extractInvoiceFields({
|
||||
buffer: Buffer.from('%PDF'),
|
||||
mimeType: 'application/pdf',
|
||||
fileName: 'kvitto.pdf',
|
||||
})
|
||||
expect(mockCreate).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('returns empty result when AI response fails schema validation', async () => {
|
||||
mockCreate.mockReturnValueOnce(
|
||||
aiResponse({ supplier: { name: 'X' } /* missing required keys */ })
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { PDFDocument } from 'pdf-lib'
|
||||
import {
|
||||
slicePdfForExtraction,
|
||||
maxPagesForAutoExtract,
|
||||
MAX_PAGES_FOR_AUTO_EXTRACT,
|
||||
MAX_PAGES_FOR_AUTO_EXTRACT_NATIVE,
|
||||
} from '@/extensions/general/invoice-inbox/lib/upload-and-extract'
|
||||
|
||||
// Pages get index-encoded widths (500 + i) so tests can assert exactly WHICH
|
||||
// source pages survived the slice, not just how many.
|
||||
async function makePdf(pageCount: number): Promise<ArrayBuffer> {
|
||||
const pdf = await PDFDocument.create()
|
||||
for (let i = 0; i < pageCount; i++) pdf.addPage([500 + i, 700])
|
||||
const bytes = await pdf.save()
|
||||
const out = new ArrayBuffer(bytes.byteLength)
|
||||
new Uint8Array(out).set(bytes)
|
||||
return out
|
||||
}
|
||||
|
||||
async function pageWidths(buffer: ArrayBuffer): Promise<number[]> {
|
||||
const pdf = await PDFDocument.load(buffer)
|
||||
return Array.from({ length: pdf.getPageCount() }, (_, i) =>
|
||||
Math.round(pdf.getPage(i).getSize().width)
|
||||
)
|
||||
}
|
||||
|
||||
describe('slicePdfForExtraction', () => {
|
||||
it('keeps the first maxPages-1 pages plus the LAST page (where totals sit)', async () => {
|
||||
const sliced = await slicePdfForExtraction(await makePdf(10), 8)
|
||||
expect(sliced).not.toBeNull()
|
||||
// Pages 0..6 plus page 9: widths 500..506 and 509.
|
||||
expect(await pageWidths(sliced!)).toEqual([500, 501, 502, 503, 504, 505, 506, 509])
|
||||
})
|
||||
|
||||
it('keeps the last page also on the old 3-page budget', async () => {
|
||||
const sliced = await slicePdfForExtraction(await makePdf(5), 3)
|
||||
expect(await pageWidths(sliced!)).toEqual([500, 501, 504])
|
||||
})
|
||||
|
||||
it('copies the document unchanged when it fits the budget', async () => {
|
||||
const sliced = await slicePdfForExtraction(await makePdf(3), 8)
|
||||
expect(await pageWidths(sliced!)).toEqual([500, 501, 502])
|
||||
})
|
||||
|
||||
it('returns null on an unparseable buffer', async () => {
|
||||
const garbage = new TextEncoder().encode('not a pdf').buffer as ArrayBuffer
|
||||
expect(await slicePdfForExtraction(garbage, 8)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('maxPagesForAutoExtract', () => {
|
||||
const ENV = [
|
||||
'AWS_ACCESS_KEY_ID',
|
||||
'AWS_SECRET_ACCESS_KEY',
|
||||
'ANTHROPIC_API_KEY',
|
||||
'AI_PROVIDER',
|
||||
'AI_BASE_URL',
|
||||
'AI_API_KEY',
|
||||
'AI_MODEL',
|
||||
'AI_PDF_MODE',
|
||||
] as const
|
||||
let saved: Record<string, string | undefined> = {}
|
||||
beforeEach(() => {
|
||||
saved = {}
|
||||
for (const k of ENV) {
|
||||
saved[k] = process.env[k]
|
||||
delete process.env[k]
|
||||
}
|
||||
})
|
||||
afterEach(() => {
|
||||
for (const k of ENV) {
|
||||
if (saved[k] === undefined) delete process.env[k]
|
||||
else process.env[k] = saved[k]
|
||||
}
|
||||
})
|
||||
|
||||
it('uses the higher budget when the backend reads PDFs natively (Claude)', () => {
|
||||
process.env.AWS_ACCESS_KEY_ID = 'AKIAEXAMPLE'
|
||||
process.env.AWS_SECRET_ACCESS_KEY = 'secret'
|
||||
expect(maxPagesForAutoExtract()).toBe(MAX_PAGES_FOR_AUTO_EXTRACT_NATIVE)
|
||||
})
|
||||
|
||||
it('keeps the conservative budget on a rasterizing OpenAI-compatible backend', () => {
|
||||
process.env.AI_PROVIDER = 'openai-compatible'
|
||||
process.env.AI_BASE_URL = 'http://localhost:8000/v1'
|
||||
process.env.AI_MODEL = 'some-model'
|
||||
expect(maxPagesForAutoExtract()).toBe(MAX_PAGES_FOR_AUTO_EXTRACT)
|
||||
})
|
||||
|
||||
it('follows AI_PDF_MODE=native on an OpenAI-compatible backend', () => {
|
||||
process.env.AI_PROVIDER = 'openai-compatible'
|
||||
process.env.AI_BASE_URL = 'http://localhost:8000/v1'
|
||||
process.env.AI_MODEL = 'some-model'
|
||||
process.env.AI_PDF_MODE = 'native'
|
||||
expect(maxPagesForAutoExtract()).toBe(MAX_PAGES_FOR_AUTO_EXTRACT_NATIVE)
|
||||
})
|
||||
})
|
||||
@@ -137,9 +137,12 @@ function buildCtx(supabase: unknown): ExtensionContext {
|
||||
} as ExtensionContext
|
||||
}
|
||||
|
||||
// The last page gets a distinct size so tests can assert the slice kept it:
|
||||
// slicePdfForExtraction takes the first pages PLUS the last (totals page).
|
||||
async function makePdfBuffer(pageCount: number): Promise<Uint8Array> {
|
||||
const pdf = await PDFDocument.create()
|
||||
for (let i = 0; i < pageCount; i++) pdf.addPage([612, 792])
|
||||
for (let i = 0; i < pageCount - 1; i++) pdf.addPage([612, 792])
|
||||
pdf.addPage([400, 600])
|
||||
return pdf.save()
|
||||
}
|
||||
|
||||
@@ -165,7 +168,7 @@ beforeEach(() => {
|
||||
})
|
||||
|
||||
describe('POST /upload: staged extraction + page-count gate (issue #553)', () => {
|
||||
it('defers long PDFs: responds processing, then slices to 3 pages and flips to received', async () => {
|
||||
it('defers long PDFs: responds processing, then slices to 8 pages (incl. the last) and flips to received', async () => {
|
||||
const captured: { row?: Record<string, unknown> } = {}
|
||||
const flip: FlipCapture = { filters: [] }
|
||||
const supabase = makeSupabase(captured, flip)
|
||||
@@ -175,7 +178,7 @@ describe('POST /upload: staged extraction + page-count gate (issue #553)', () =>
|
||||
rawText: 'ok',
|
||||
})
|
||||
|
||||
const req = await makeUploadRequest(6)
|
||||
const req = await makeUploadRequest(10)
|
||||
const res = await uploadRoute.handler(req, buildCtx(supabase))
|
||||
const { status, body } = await parseJsonResponse<{ data: Record<string, unknown> }>(res)
|
||||
|
||||
@@ -186,16 +189,20 @@ describe('POST /upload: staged extraction + page-count gate (issue #553)', () =>
|
||||
expect(body.data.extracted_data).toBeNull()
|
||||
expect(body.data.extraction_skipped).toBe(false)
|
||||
expect(body.data.skip_reason).toBeNull()
|
||||
expect(body.data.page_count).toBe(6)
|
||||
expect(body.data.page_count).toBe(10)
|
||||
expect(captured.row?.status).toBe('processing')
|
||||
expect(captured.row?.extracted_data).toBeNull()
|
||||
expect(captured.row?.extraction_skipped).toBe(false)
|
||||
|
||||
// The deferred worker extracts from the sliced copy, not the original.
|
||||
// The deferred worker extracts from the sliced copy, not the original:
|
||||
// the first 7 pages plus the LAST page (the distinct 400x600 one), where
|
||||
// invoice totals usually sit.
|
||||
await vi.waitFor(() => expect(extractInvoiceFields).toHaveBeenCalledOnce())
|
||||
const sentBuffer = vi.mocked(extractInvoiceFields).mock.calls[0][0].buffer
|
||||
const sentPdf = await PDFDocument.load(sentBuffer)
|
||||
expect(sentPdf.getPageCount()).toBe(3)
|
||||
expect(sentPdf.getPageCount()).toBe(8)
|
||||
const lastPage = sentPdf.getPage(7).getSize()
|
||||
expect(lastPage).toEqual({ width: 400, height: 600 })
|
||||
|
||||
// ...and CAS-flips the processing row to received, with the truncation
|
||||
// recorded in extracted_data.pages rather than as a skip.
|
||||
@@ -203,8 +210,8 @@ describe('POST /upload: staged extraction + page-count gate (issue #553)', () =>
|
||||
expect(flip.payload?.status).toBe('received')
|
||||
expect(flip.payload?.extraction_skipped).toBe(false)
|
||||
expect((flip.payload?.extracted_data as { pages?: unknown })?.pages).toEqual({
|
||||
total: 6,
|
||||
analyzed: 3,
|
||||
total: 10,
|
||||
analyzed: 8,
|
||||
})
|
||||
expect(flip.filters).toEqual([
|
||||
['id', 'inbox-1'],
|
||||
@@ -212,7 +219,7 @@ describe('POST /upload: staged extraction + page-count gate (issue #553)', () =>
|
||||
])
|
||||
})
|
||||
|
||||
it('defers PDFs at or below the page-count limit and extracts the full buffer', async () => {
|
||||
it('defers PDFs at or below the native page budget (6 pages) and extracts the full buffer', async () => {
|
||||
const captured: { row?: Record<string, unknown> } = {}
|
||||
const flip: FlipCapture = { filters: [] }
|
||||
const supabase = makeSupabase(captured, flip)
|
||||
@@ -222,7 +229,9 @@ describe('POST /upload: staged extraction + page-count gate (issue #553)', () =>
|
||||
rawText: 'ok',
|
||||
})
|
||||
|
||||
const req = await makeUploadRequest(2)
|
||||
// 6 pages gated on the old budget of 3; on the native budget of 8 the
|
||||
// whole document is read (this is the regression the raise fixes).
|
||||
const req = await makeUploadRequest(6)
|
||||
const res = await uploadRoute.handler(req, buildCtx(supabase))
|
||||
const { status, body } = await parseJsonResponse<{ data: Record<string, unknown> }>(res)
|
||||
|
||||
@@ -230,12 +239,12 @@ describe('POST /upload: staged extraction + page-count gate (issue #553)', () =>
|
||||
expect(body.data.status).toBe('processing')
|
||||
expect(body.data.extraction_skipped).toBe(false)
|
||||
expect(body.data.skip_reason).toBeNull()
|
||||
expect(body.data.page_count).toBe(2)
|
||||
expect(body.data.page_count).toBe(6)
|
||||
|
||||
await vi.waitFor(() => expect(extractInvoiceFields).toHaveBeenCalledOnce())
|
||||
const sentBuffer = vi.mocked(extractInvoiceFields).mock.calls[0][0].buffer
|
||||
const sentPdf = await PDFDocument.load(sentBuffer)
|
||||
expect(sentPdf.getPageCount()).toBe(2)
|
||||
expect(sentPdf.getPageCount()).toBe(6)
|
||||
|
||||
await vi.waitFor(() => expect(flip.payload).toBeDefined())
|
||||
expect(flip.payload?.status).toBe('received')
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
countPdfPages,
|
||||
slicePdfForExtraction,
|
||||
MAX_FILE_SIZE,
|
||||
MAX_PAGES_FOR_AUTO_EXTRACT,
|
||||
maxPagesForAutoExtract,
|
||||
UPLOAD_ALLOWED_MIME_TYPES,
|
||||
EMAIL_ALLOWED_MIME_TYPES,
|
||||
ensureHtmlDocument,
|
||||
@@ -758,13 +758,14 @@ export const invoiceInboxExtension: Extension = {
|
||||
})
|
||||
|
||||
// Same page handling as /upload (issue #553): long PDFs extract
|
||||
// from a slice of their first pages; the skip only remains for
|
||||
// unsliceable (encrypted/malformed) PDFs. Sandbox companies skip
|
||||
// Bedrock unconditionally.
|
||||
// from a slice (first pages + the last page); the skip only remains
|
||||
// for unsliceable (encrypted/malformed) PDFs. Sandbox companies
|
||||
// skip Bedrock unconditionally.
|
||||
const maxAutoExtractPages = maxPagesForAutoExtract()
|
||||
const pageCount =
|
||||
file.type === 'application/pdf' ? await countPdfPages(buffer) : null
|
||||
const gatedByPageCount =
|
||||
pageCount != null && pageCount > MAX_PAGES_FOR_AUTO_EXTRACT
|
||||
pageCount != null && pageCount > maxAutoExtractPages
|
||||
const sandbox = await isSandboxCompany(ctx.supabase, ctx.companyId)
|
||||
// Paid-tier gate: no `ai` capability → no Bedrock OCR (seed empty
|
||||
// skeleton; the attached document is still stored). Same paywall as
|
||||
@@ -772,7 +773,7 @@ export const invoiceInboxExtension: Extension = {
|
||||
const hasAiEntitlement = await hasCapability(ctx.supabase, ctx.companyId, CAPABILITY.ai)
|
||||
const slicedBuffer =
|
||||
gatedByPageCount && hasAiEntitlement && !sandbox
|
||||
? await slicePdfForExtraction(buffer, MAX_PAGES_FOR_AUTO_EXTRACT)
|
||||
? await slicePdfForExtraction(buffer, maxAutoExtractPages)
|
||||
: null
|
||||
const skipReason: 'no_ai_entitlement' | 'too_many_pages' | 'sandbox' | null =
|
||||
!hasAiEntitlement
|
||||
@@ -793,7 +794,7 @@ export const invoiceInboxExtension: Extension = {
|
||||
})
|
||||
const { data: extracted } = extraction
|
||||
if (!skipExtraction && slicedBuffer != null && pageCount != null) {
|
||||
extracted.pages = { total: pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT }
|
||||
extracted.pages = { total: pageCount, analyzed: maxAutoExtractPages }
|
||||
}
|
||||
await mirrorExtractionToDocument(doc.id, {
|
||||
data: extracted,
|
||||
|
||||
@@ -524,13 +524,14 @@ export async function extractInvoiceFields(
|
||||
let rawText: string | null = null
|
||||
let model: string | null = null
|
||||
try {
|
||||
const result = await service.extractFromDocument({
|
||||
const baseMaxTokens = readAiConfig().extractionMaxTokens
|
||||
const request = {
|
||||
document: toDocumentInput(input),
|
||||
system: SYSTEM_PROMPT,
|
||||
instruction: EXTRACTION_INSTRUCTION,
|
||||
maxTokens: readAiConfig().extractionMaxTokens,
|
||||
jsonSchema: EXTRACTION_JSON_SCHEMA,
|
||||
})
|
||||
}
|
||||
let result = await service.extractFromDocument({ ...request, maxTokens: baseMaxTokens })
|
||||
if (!result.ok) {
|
||||
// Not a failure: the deployment cannot read this document at all.
|
||||
// `ai_unconfigured` is the self-host "no key yet" case the 30 s
|
||||
@@ -538,6 +539,43 @@ export async function extractInvoiceFields(
|
||||
log.warn('AI extraction skipped', { file_name_hash: fileNameHash, reason: result.skipped })
|
||||
return { data: emptyResult(), rawText: null, skipped: result.skipped }
|
||||
}
|
||||
if (result.truncated) {
|
||||
// The output hit maxTokens mid-JSON (line-item-heavy documents). Left
|
||||
// alone this parsed to nothing and looked like an unreadable document;
|
||||
// one retry at double the cap recovers it. A second truncation falls
|
||||
// through to the normal parse path, which fails visibly in the log
|
||||
// below instead of silently.
|
||||
log.warn('ai_extraction_truncated', {
|
||||
file_name_hash: fileNameHash,
|
||||
max_tokens: baseMaxTokens,
|
||||
retrying: true,
|
||||
})
|
||||
// A retry that THROWS (throttle, network) must not sink the first
|
||||
// response: its text may still parse despite the truncation flag, and
|
||||
// the outer catch would otherwise return the empty skeleton with
|
||||
// rawText null. Swallow locally and continue with the first result.
|
||||
try {
|
||||
const retry = await service.extractFromDocument({
|
||||
...request,
|
||||
maxTokens: baseMaxTokens * 2,
|
||||
})
|
||||
if (retry.ok) {
|
||||
result = retry
|
||||
if (retry.truncated) {
|
||||
log.warn('ai_extraction_truncated', {
|
||||
file_name_hash: fileNameHash,
|
||||
max_tokens: baseMaxTokens * 2,
|
||||
retrying: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
} catch (retryErr) {
|
||||
log.warn('ai_extraction_retry_failed', {
|
||||
file_name_hash: fileNameHash,
|
||||
error: retryErr instanceof Error ? retryErr.message : String(retryErr),
|
||||
})
|
||||
}
|
||||
}
|
||||
rawText = result.text
|
||||
model = result.model
|
||||
|
||||
|
||||
@@ -21,11 +21,16 @@ import { emptyResult } from './extract-invoice-fields'
|
||||
const log = createLogger('invoice-inbox/sweep')
|
||||
|
||||
/**
|
||||
* A deferred extraction is one Bedrock call (the WhatsApp cron budgets
|
||||
* 10-60s for the same call). Two minutes of silence means no live worker
|
||||
* can still deliver a flip that beats the sweep by enough to matter.
|
||||
* A deferred extraction is now up to TWO Bedrock calls on up to 8 native PDF
|
||||
* pages: the base call can spend its full output budget (~90-150s at 8192
|
||||
* tokens) before the truncation retry doubles the cap and runs again. A live
|
||||
* worker can therefore be legitimately silent for several minutes; a cutoff
|
||||
* shorter than its worst case makes the sweep steal the row and discard the
|
||||
* worker's real result via the status CAS. Ten minutes clears the two-call
|
||||
* worst case with margin while still flipping genuinely crashed rows well
|
||||
* before anyone files a support mail about a stuck item.
|
||||
*/
|
||||
export const PROCESSING_STUCK_MS = 2 * 60 * 1000
|
||||
export const PROCESSING_STUCK_MS = 10 * 60 * 1000
|
||||
const BATCH = 50
|
||||
|
||||
export interface InboxSweepSummary {
|
||||
|
||||
@@ -42,12 +42,24 @@ export function sanitiseMime(raw: string | null | undefined): string {
|
||||
|
||||
export const MAX_FILE_SIZE = 10 * 1024 * 1024
|
||||
|
||||
// AI extraction is tuned for single-page receipts/invoices. Documents above
|
||||
// this page count tend to be sales reports, bank statements, or contracts:
|
||||
// Bedrock churns for minutes and still extracts nothing useful (issue #553).
|
||||
// Above the limit we skip extraction entirely; the document still lands in
|
||||
// the inbox and can be attached to a transaction or converted manually.
|
||||
// Page budget for auto-extraction. Documents far above it tend to be sales
|
||||
// reports, bank statements, or contracts (issue #553); those are sliced (see
|
||||
// slicePdfForExtraction) rather than read whole. The budget depends on how
|
||||
// the backend reads PDFs:
|
||||
// - native (Claude on hosted): the model reads PDF bytes directly and
|
||||
// handles long documents fine; 8 pages covers real multi-page invoices
|
||||
// whose totals sit on a later page without opening the door to 100-page
|
||||
// statements.
|
||||
// - rasterize (OpenAI-compatible self-host): every page becomes a PNG in
|
||||
// the prompt, so the old conservative budget of 3 stays.
|
||||
export const MAX_PAGES_FOR_AUTO_EXTRACT = 3
|
||||
export const MAX_PAGES_FOR_AUTO_EXTRACT_NATIVE = 8
|
||||
|
||||
export function maxPagesForAutoExtract(): number {
|
||||
return getAiStatus().capabilities.pdfNative
|
||||
? MAX_PAGES_FOR_AUTO_EXTRACT_NATIVE
|
||||
: MAX_PAGES_FOR_AUTO_EXTRACT
|
||||
}
|
||||
|
||||
// Returns the page count for a PDF buffer, or null if the buffer isn't a
|
||||
// parseable PDF. Errors fall through so callers can treat "unknown" the same
|
||||
@@ -61,11 +73,13 @@ export async function countPdfPages(buffer: ArrayBuffer): Promise<number | null>
|
||||
}
|
||||
}
|
||||
|
||||
// Long PDFs used to skip extraction entirely (issue #553). Invoice data
|
||||
// almost always sits on the first page(s), so instead we extract from a
|
||||
// slim copy of the first MAX_PAGES_FOR_AUTO_EXTRACT pages and record the
|
||||
// truncation in extracted_data.pages. Returns null when slicing fails
|
||||
// (encrypted/malformed PDF) so the caller can fall back to the old skip.
|
||||
// Long PDFs used to skip extraction entirely (issue #553). Instead we extract
|
||||
// from a slim copy and record the truncation in extracted_data.pages. The
|
||||
// slice is the first maxPages-1 pages PLUS the last page: on multi-page
|
||||
// invoices the totals, OCR number and "Att betala" routinely sit on the final
|
||||
// page, and a first-pages-only slice read everything except the amounts.
|
||||
// Returns null when slicing fails (encrypted/malformed PDF) so the caller can
|
||||
// fall back to the old skip.
|
||||
export async function slicePdfForExtraction(
|
||||
buffer: ArrayBuffer,
|
||||
maxPages: number
|
||||
@@ -73,10 +87,12 @@ export async function slicePdfForExtraction(
|
||||
try {
|
||||
const src = await PDFDocument.load(buffer, { updateMetadata: false })
|
||||
const dst = await PDFDocument.create()
|
||||
const pages = await dst.copyPages(
|
||||
src,
|
||||
Array.from({ length: Math.min(maxPages, src.getPageCount()) }, (_, i) => i)
|
||||
)
|
||||
const total = src.getPageCount()
|
||||
const indices =
|
||||
total > maxPages && maxPages >= 2
|
||||
? [...Array.from({ length: maxPages - 1 }, (_, i) => i), total - 1]
|
||||
: Array.from({ length: Math.min(maxPages, total) }, (_, i) => i)
|
||||
const pages = await dst.copyPages(src, indices)
|
||||
for (const page of pages) dst.addPage(page)
|
||||
const bytes = await dst.save()
|
||||
// Copy into a fresh ArrayBuffer: Uint8Array.buffer is ArrayBufferLike
|
||||
@@ -335,16 +351,18 @@ export async function uploadAndExtract(
|
||||
console.error('[invoice-inbox] Failed to append DocumentIngested:', err)
|
||||
}
|
||||
|
||||
// Page-count gate (issue #553): PDFs above MAX_PAGES_FOR_AUTO_EXTRACT
|
||||
// skip extraction. Bedrock would otherwise block the upload response for
|
||||
// minutes on a 6-page sales report and return nothing useful. Images and
|
||||
// non-PDFs are never gated (single-page by definition). countPdfPages
|
||||
// returns null on malformed PDFs: we treat null as "not gated" and fall
|
||||
// through to the existing extraction path so today's behavior is preserved.
|
||||
// Page-count gate (issue #553): PDFs above the auto-extract page budget
|
||||
// are sliced before extraction. Bedrock would otherwise block the upload
|
||||
// response for minutes on a long sales report and return nothing useful.
|
||||
// Images and non-PDFs are never gated (single-page by definition).
|
||||
// countPdfPages returns null on malformed PDFs: we treat null as "not
|
||||
// gated" and fall through to the existing extraction path so today's
|
||||
// behavior is preserved.
|
||||
const maxAutoExtractPages = maxPagesForAutoExtract()
|
||||
const pageCount =
|
||||
file.type === 'application/pdf' ? await countPdfPages(file.buffer) : null
|
||||
const gatedByPageCount =
|
||||
pageCount != null && pageCount > MAX_PAGES_FOR_AUTO_EXTRACT
|
||||
pageCount != null && pageCount > maxAutoExtractPages
|
||||
const sandbox = await isSandboxCompany(supabase, companyId)
|
||||
// Paid-tier gate: AI document OCR (Bedrock, via extractInvoiceFields) is the
|
||||
// `ai` capability. A company without it (free/manual tier) must never trigger
|
||||
@@ -419,6 +437,7 @@ export async function uploadAndExtract(
|
||||
file,
|
||||
pageCount,
|
||||
gatedByPageCount,
|
||||
maxAutoExtractPages,
|
||||
})
|
||||
|
||||
return {
|
||||
@@ -434,12 +453,12 @@ export async function uploadAndExtract(
|
||||
}
|
||||
}
|
||||
|
||||
// Long PDFs are sliced to their first pages instead of skipped, but only
|
||||
// when extraction would actually run: slicing after an entitlement/sandbox/
|
||||
// opt-out verdict would be wasted CPU.
|
||||
// Long PDFs are sliced (first pages + the last page) instead of skipped,
|
||||
// but only when extraction would actually run: slicing after an
|
||||
// entitlement/sandbox/opt-out verdict would be wasted CPU.
|
||||
const slicedBuffer =
|
||||
gatedByPageCount && syncSkipReason === null
|
||||
? await slicePdfForExtraction(file.buffer, MAX_PAGES_FOR_AUTO_EXTRACT)
|
||||
? await slicePdfForExtraction(file.buffer, maxAutoExtractPages)
|
||||
: null
|
||||
// Skip-reason priority: no-AI-entitlement > sandbox > client opt-out >
|
||||
// page-count. Opt-out outranks the page gate (an opted-out caller never
|
||||
@@ -464,7 +483,7 @@ export async function uploadAndExtract(
|
||||
})
|
||||
const { data: extracted, rawText } = extraction
|
||||
if (!skipExtraction && slicedBuffer != null && pageCount != null) {
|
||||
extracted.pages = { total: pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT }
|
||||
extracted.pages = { total: pageCount, analyzed: maxAutoExtractPages }
|
||||
}
|
||||
|
||||
// Supplier match by org-nr, then VAT number, then case-insensitive name
|
||||
@@ -563,6 +582,9 @@ interface DeferredExtractionJob {
|
||||
file: { name: string; buffer: ArrayBuffer; type: string }
|
||||
pageCount: number | null
|
||||
gatedByPageCount: boolean
|
||||
/** Snapshot of maxPagesForAutoExtract() from the request, so the gate
|
||||
* verdict and the slice/stamp below cannot disagree. */
|
||||
maxAutoExtractPages: number
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -588,7 +610,7 @@ function scheduleDeferredExtraction(job: DeferredExtractionJob): void {
|
||||
let skipReason: string | null = null
|
||||
try {
|
||||
const slicedBuffer = job.gatedByPageCount
|
||||
? await slicePdfForExtraction(job.file.buffer, MAX_PAGES_FOR_AUTO_EXTRACT)
|
||||
? await slicePdfForExtraction(job.file.buffer, job.maxAutoExtractPages)
|
||||
: null
|
||||
if (job.gatedByPageCount && slicedBuffer == null) {
|
||||
extractionSkipped = true
|
||||
@@ -610,7 +632,7 @@ function scheduleDeferredExtraction(job: DeferredExtractionJob): void {
|
||||
skipReason = result.skipped
|
||||
}
|
||||
if (slicedBuffer != null && job.pageCount != null) {
|
||||
extracted.pages = { total: job.pageCount, analyzed: MAX_PAGES_FOR_AUTO_EXTRACT }
|
||||
extracted.pages = { total: job.pageCount, analyzed: job.maxAutoExtractPages }
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
|
||||
Reference in New Issue
Block a user