fix(import): stop the per-file model pass and parallelize underlag-to-verifikat attach (#2188) (#2229)
Linking migrated underlag to verifikat by filename ran one request per file, strictly in sequence, and each request awaited a vision-model extraction through document.uploaded even though the file lands on an already-posted verifikat. A few hundred files took ten-plus minutes in the foreground. - The attach route passes extractionOwner: 'none' to uploadDocument: the booking is already known, so the model pass bought nothing. Same opt-out the provider underlag sweep took in #1783. - The wizard runs the attach step through mapWithConcurrency with a pool of 4 instead of one-after-another; the per-file counter still ticks and the outcome list keeps plan order. The per-file re-plan (planPermitsAttach) still costs two DB round trips per file; it is bounded now that the pool overlaps them, and left as is. Closes #2188 Claude-Session: https://claude.ai/code/session_01QPQLwHNEiQfiCNLSMzXMiQ Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jakob Wennberg
Claude Fable 5.1
parent
bc5da12372
commit
9a25672dcb
@@ -312,6 +312,8 @@ describe('POST /api/import/documents/attach', () => {
|
||||
// key is the entry, not the bytes.
|
||||
idempotency_key: TARGET_ID,
|
||||
upload_source: 'file_upload',
|
||||
// The verifikat is already booked: no per-file model pass (#2188).
|
||||
extractionOwner: 'none',
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -177,6 +177,13 @@ export const POST = withRouteContext(
|
||||
{
|
||||
upload_source: 'file_upload',
|
||||
journal_entry_id: journalEntryId,
|
||||
// The file lands on a posted verifikat by construction, so the
|
||||
// booking is already known and a model pass per file buys nothing.
|
||||
// Run inline through document.uploaded (the bus awaits its
|
||||
// handlers), that pass was the bulk of a 10-minute foreground wait
|
||||
// on a few hundred migrated files (#2188). Same opt-out as the
|
||||
// provider underlag sweep (#1783), for the same reason.
|
||||
extractionOwner: 'none',
|
||||
// Scope the deterministic id to the target verifikat: the same
|
||||
// receipt may legitimately back several verifikat, so content alone
|
||||
// must not dedupe across them.
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
useDestructiveConfirm,
|
||||
} from '@/components/ui/destructive-confirm-dialog'
|
||||
import { FyPicker } from '@/components/common/FyPicker'
|
||||
import { mapWithConcurrency } from '@/lib/concurrency'
|
||||
import { getErrorMessage } from '@/lib/errors/get-error-message'
|
||||
import { cn, formatDate } from '@/lib/utils'
|
||||
import { FileUp, Loader2 } from 'lucide-react'
|
||||
@@ -49,6 +50,14 @@ type Step = 'select' | 'review' | 'result'
|
||||
|
||||
const ACCEPTED_TYPES = 'application/pdf,image/jpeg,image/png,image/webp'
|
||||
|
||||
/**
|
||||
* Parallel attach requests during the final step. Same size as the
|
||||
* transactions page's batch actions: enough that a 300-file migration
|
||||
* finishes in a few minutes instead of a coffee break, small enough that a
|
||||
* laptop on hotel wifi does not choke on in-flight uploads.
|
||||
*/
|
||||
const ATTACH_CONCURRENCY = 4
|
||||
|
||||
/** Message keys per status, spelled out so next-intl keeps checking them. */
|
||||
const STATUS_KEY: Record<UnderlagPlanStatus, string> = {
|
||||
matched: 'underlag_status_matched',
|
||||
@@ -273,13 +282,15 @@ export default function UnderlagImportWizard() {
|
||||
|
||||
setIsLoading(true)
|
||||
setAttached(0)
|
||||
const results: AttachOutcome[] = []
|
||||
let results: AttachOutcome[] = []
|
||||
|
||||
try {
|
||||
// Sequential on purpose: hundreds of uploads in parallel would swamp the
|
||||
// browser and the storage bucket, and a visible one-by-one count is what
|
||||
// makes a long migration legible.
|
||||
for (const row of selectedRows) {
|
||||
// A small worker pool, not one-after-another: each attach is a handful
|
||||
// of serialized round trips (auth, company, plan, storage, insert), so
|
||||
// a few hundred files took ten-plus minutes in sequence (#2188). The
|
||||
// pool keeps the storage bucket and the browser bounded, the per-file
|
||||
// counter still ticks, and mapWithConcurrency preserves plan order.
|
||||
results = await mapWithConcurrency(selectedRows, ATTACH_CONCURRENCY, async (row) => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', row.file)
|
||||
formData.append('journal_entry_id', row.targetId as string)
|
||||
@@ -288,6 +299,7 @@ export default function UnderlagImportWizard() {
|
||||
formData.append('fiscal_period_id', plan.fiscal_period_id)
|
||||
if (row.manual) formData.append('override', 'true')
|
||||
|
||||
let outcome: AttachOutcome
|
||||
try {
|
||||
const res = await fetch('/api/import/documents/attach', {
|
||||
method: 'POST',
|
||||
@@ -295,20 +307,21 @@ export default function UnderlagImportWizard() {
|
||||
})
|
||||
if (!res.ok) {
|
||||
const data = await res.json().catch(() => null)
|
||||
results.push({
|
||||
outcome = {
|
||||
file_name: row.file_name,
|
||||
ok: false,
|
||||
message: getErrorMessage(data, { statusCode: res.status }),
|
||||
})
|
||||
}
|
||||
} else {
|
||||
results.push({ file_name: row.file_name, ok: true })
|
||||
outcome = { file_name: row.file_name, ok: true }
|
||||
}
|
||||
} catch (err) {
|
||||
results.push({ file_name: row.file_name, ok: false, message: getErrorMessage(err) })
|
||||
outcome = { file_name: row.file_name, ok: false, message: getErrorMessage(err) }
|
||||
}
|
||||
|
||||
setAttached((n) => n + 1)
|
||||
}
|
||||
return outcome
|
||||
})
|
||||
} finally {
|
||||
// Whatever happens above, the wizard must not stay stuck "loading":
|
||||
// that state also freezes the year picker.
|
||||
|
||||
Reference in New Issue
Block a user