From 9a25672dcb1745239fd6fdb14a1b2eab11a7ee1b Mon Sep 17 00:00:00 2001 From: Jakob Wennberg Date: Thu, 3 Sep 2026 17:16:50 +0200 Subject: [PATCH] fix(import): stop the per-file model pass and parallelize underlag-to-verifikat attach (#2188) (#2229) Linking migrated underlag to verifikat by filename ran one request per file, strictly in sequence, and each request awaited a vision-model extraction through document.uploaded even though the file lands on an already-posted verifikat. A few hundred files took ten-plus minutes in the foreground. - The attach route passes extractionOwner: 'none' to uploadDocument: the booking is already known, so the model pass bought nothing. Same opt-out the provider underlag sweep took in #1783. - The wizard runs the attach step through mapWithConcurrency with a pool of 4 instead of one-after-another; the per-file counter still ticks and the outcome list keeps plan order. The per-file re-plan (planPermitsAttach) still costs two DB round trips per file; it is bounded now that the pool overlaps them, and left as is. Closes #2188 Claude-Session: https://claude.ai/code/session_01QPQLwHNEiQfiCNLSMzXMiQ Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- .../documents/attach/__tests__/route.test.ts | 2 ++ app/api/import/documents/attach/route.ts | 7 ++++ components/import/UnderlagImportWizard.tsx | 33 +++++++++++++------ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/app/api/import/documents/attach/__tests__/route.test.ts b/app/api/import/documents/attach/__tests__/route.test.ts index 1a46eaa9..be2fdcb5 100644 --- a/app/api/import/documents/attach/__tests__/route.test.ts +++ b/app/api/import/documents/attach/__tests__/route.test.ts @@ -312,6 +312,8 @@ describe('POST /api/import/documents/attach', () => { // key is the entry, not the bytes. idempotency_key: TARGET_ID, upload_source: 'file_upload', + // The verifikat is already booked: no per-file model pass (#2188). + extractionOwner: 'none', }) }) diff --git a/app/api/import/documents/attach/route.ts b/app/api/import/documents/attach/route.ts index 6f5f0b30..f7903315 100644 --- a/app/api/import/documents/attach/route.ts +++ b/app/api/import/documents/attach/route.ts @@ -177,6 +177,13 @@ export const POST = withRouteContext( { upload_source: 'file_upload', journal_entry_id: journalEntryId, + // The file lands on a posted verifikat by construction, so the + // booking is already known and a model pass per file buys nothing. + // Run inline through document.uploaded (the bus awaits its + // handlers), that pass was the bulk of a 10-minute foreground wait + // on a few hundred migrated files (#2188). Same opt-out as the + // provider underlag sweep (#1783), for the same reason. + extractionOwner: 'none', // Scope the deterministic id to the target verifikat: the same // receipt may legitimately back several verifikat, so content alone // must not dedupe across them. diff --git a/components/import/UnderlagImportWizard.tsx b/components/import/UnderlagImportWizard.tsx index fae57072..47c691b7 100644 --- a/components/import/UnderlagImportWizard.tsx +++ b/components/import/UnderlagImportWizard.tsx @@ -16,6 +16,7 @@ import { useDestructiveConfirm, } from '@/components/ui/destructive-confirm-dialog' import { FyPicker } from '@/components/common/FyPicker' +import { mapWithConcurrency } from '@/lib/concurrency' import { getErrorMessage } from '@/lib/errors/get-error-message' import { cn, formatDate } from '@/lib/utils' import { FileUp, Loader2 } from 'lucide-react' @@ -49,6 +50,14 @@ type Step = 'select' | 'review' | 'result' const ACCEPTED_TYPES = 'application/pdf,image/jpeg,image/png,image/webp' +/** + * Parallel attach requests during the final step. Same size as the + * transactions page's batch actions: enough that a 300-file migration + * finishes in a few minutes instead of a coffee break, small enough that a + * laptop on hotel wifi does not choke on in-flight uploads. + */ +const ATTACH_CONCURRENCY = 4 + /** Message keys per status, spelled out so next-intl keeps checking them. */ const STATUS_KEY: Record = { matched: 'underlag_status_matched', @@ -273,13 +282,15 @@ export default function UnderlagImportWizard() { setIsLoading(true) setAttached(0) - const results: AttachOutcome[] = [] + let results: AttachOutcome[] = [] try { - // Sequential on purpose: hundreds of uploads in parallel would swamp the - // browser and the storage bucket, and a visible one-by-one count is what - // makes a long migration legible. - for (const row of selectedRows) { + // A small worker pool, not one-after-another: each attach is a handful + // of serialized round trips (auth, company, plan, storage, insert), so + // a few hundred files took ten-plus minutes in sequence (#2188). The + // pool keeps the storage bucket and the browser bounded, the per-file + // counter still ticks, and mapWithConcurrency preserves plan order. + results = await mapWithConcurrency(selectedRows, ATTACH_CONCURRENCY, async (row) => { const formData = new FormData() formData.append('file', row.file) formData.append('journal_entry_id', row.targetId as string) @@ -288,6 +299,7 @@ export default function UnderlagImportWizard() { formData.append('fiscal_period_id', plan.fiscal_period_id) if (row.manual) formData.append('override', 'true') + let outcome: AttachOutcome try { const res = await fetch('/api/import/documents/attach', { method: 'POST', @@ -295,20 +307,21 @@ export default function UnderlagImportWizard() { }) if (!res.ok) { const data = await res.json().catch(() => null) - results.push({ + outcome = { file_name: row.file_name, ok: false, message: getErrorMessage(data, { statusCode: res.status }), - }) + } } else { - results.push({ file_name: row.file_name, ok: true }) + outcome = { file_name: row.file_name, ok: true } } } catch (err) { - results.push({ file_name: row.file_name, ok: false, message: getErrorMessage(err) }) + outcome = { file_name: row.file_name, ok: false, message: getErrorMessage(err) } } setAttached((n) => n + 1) - } + return outcome + }) } finally { // Whatever happens above, the wizard must not stay stuck "loading": // that state also freezes the year picker.