fix(migration): resumable underlag import without inline extraction + same-origin MCP storage URLs (#1783)
* fix(migration): resumable underlag import without inline extraction, same-origin MCP storage URLs The Fortnox underlag import ran every file's AI extraction inline inside one request and hit the hosted 300 s function limit after ~17 of 113 files (twice on 2026-08-21); the UI showed the generic "underlagen kunde inte importeras" although the files it did reach were linked. The import now works in time-budgeted slices with a stable cursor (the UI loops until the server reports the end and shows "x av y") and opts out of extraction (extractionOwner 'none', stamped skipped:opted_out): every file is linked to its posted verifikat on arrival, so the booking is already known. MCP signed Storage URLs (upload_url, signed_url, download_url) are served through a same-origin proxy, /api/storage/[...path], because Claude Desktop's sandbox only reaches the MCP host and blocked the PUT to <project>.supabase.co. The signed token stays the only credential; the proxy forwards only signed documents-bucket paths to our own Storage host and is a no-op rewrite when NEXT_PUBLIC_APP_URL is unset. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm * fix(mcp): keep the storage-proxy note out of the size-capped tool descriptions The per-tool 280-char cap and the tools/list payload ceiling both tripped on the two sentences added to gnubok_create_document_upload and gnubok_get_document_content; the why now lives in a code comment. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm * fix(review): id cursor, stall = error, capped upload body, encoded dot segments Review follow-ups on #1783: - the import cursor is the last handled provider attachment id, not an index, so a file Fortnox adds or removes mid-sweep shifts nothing - a partial answer whose cursor does not advance (or the round guard) is reported as ARCIM_DOCUMENT_IMPORT_STALLED instead of "complete"; the slices already landed stay reported and the retry button resumes - the storage proxy reads the PUT body as a capped stream instead of buffering an unbounded payload before measuring it - object paths are rejected when any segment decodes to "." or ".." (or holds a separator), and the URL fetch() would actually request is re-checked against the allowlist after normalisation - download_url description no longer claims a direct Storage URL Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Claude Fable 5
Jakob Wennberg
Claude Fable 5
Jakob Wennberg
parent
0531576807
commit
524d9978f1
@@ -37,6 +37,7 @@ import {
|
||||
parseArcimDocumentOAuthResume,
|
||||
PROVIDER_DOCUMENT_SCOPES_UNAVAILABLE,
|
||||
requestArcimDocumentImport,
|
||||
runArcimDocumentImportToCompletion,
|
||||
resolveArcimDocumentFollowUpProvider,
|
||||
watchArcimOAuthPopup,
|
||||
type ArcimDocumentImportProblem,
|
||||
@@ -1477,10 +1478,26 @@ function DocumentImportFollowUp({
|
||||
? t('ext_arcim_documents_importing')
|
||||
: t('ext_arcim_documents_reconnecting')
|
||||
|
||||
// Running totals arrive after each server slice of a real import; the
|
||||
// dry-run result that sits in state while the first slice runs is not
|
||||
// progress, so it stays silent.
|
||||
const progress =
|
||||
state.phase === 'importing' && state.result && !state.result.dryRun && state.result.total > 0
|
||||
? state.result
|
||||
: null
|
||||
|
||||
return (
|
||||
<section className="space-y-3" aria-live="polite">
|
||||
{title}
|
||||
<SpinnerLine>{label}</SpinnerLine>
|
||||
{progress && (
|
||||
<p className="text-sm tabular-nums text-muted-foreground">
|
||||
{t('ext_arcim_documents_import_progress', {
|
||||
done: progress.scanned,
|
||||
total: progress.total,
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
@@ -2316,7 +2333,12 @@ export default function ArcimMigrationWorkspace({
|
||||
const runDocumentImport = useCallback(async (currentConsentId: string) => {
|
||||
dispatchDocumentImport({ type: 'import-started' })
|
||||
try {
|
||||
const result = await requestArcimDocumentImport(currentConsentId, false)
|
||||
// One route call per time-budgeted slice; the helper loops until the
|
||||
// server reports the end and feeds running totals back for the UI.
|
||||
const result = await runArcimDocumentImportToCompletion(currentConsentId, {
|
||||
onProgress: (progress) =>
|
||||
dispatchDocumentImport({ type: 'import-progress', result: progress }),
|
||||
})
|
||||
dispatchDocumentImport({ type: 'import-succeeded', result })
|
||||
} catch (documentError) {
|
||||
dispatchDocumentImport({
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
ARCIM_DOCUMENT_IMPORT_STALLED,
|
||||
ARCIM_DOCUMENT_OAUTH_RESUME_KEY,
|
||||
INITIAL_ARCIM_DOCUMENT_IMPORT_STATE,
|
||||
PROVIDER_DOCUMENT_SCOPES_REQUIRED,
|
||||
@@ -8,8 +9,10 @@ import {
|
||||
arcimDocumentImportReducer,
|
||||
documentOAuthProblemFromReason,
|
||||
parseArcimDocumentOAuthResume,
|
||||
mergeArcimDocumentImportResults,
|
||||
requestArcimDocumentImport,
|
||||
resolveArcimDocumentFollowUpProvider,
|
||||
runArcimDocumentImportToCompletion,
|
||||
watchArcimOAuthPopup,
|
||||
type ArcimDocumentImportResult,
|
||||
} from '../arcim-document-import-flow'
|
||||
@@ -26,10 +29,20 @@ function result(
|
||||
failed: 0,
|
||||
dryRun: true,
|
||||
unmatchedSamples: [],
|
||||
total: 7,
|
||||
partial: false,
|
||||
nextCursor: null,
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
function jsonResponse(body: unknown, status = 200): Response {
|
||||
return new Response(JSON.stringify(body), {
|
||||
status,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
}
|
||||
|
||||
describe('Fortnox document follow-up state', () => {
|
||||
it('offers the prompt after a successful Fortnox migration using the honest found count', () => {
|
||||
const discovering = arcimDocumentImportReducer(
|
||||
@@ -298,3 +311,123 @@ describe('document scope OAuth recovery', () => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
})
|
||||
|
||||
describe('resumable import (one server slice per call)', () => {
|
||||
it('accepts an older server answer without resume fields as one complete slice', async () => {
|
||||
const { total: _t, partial: _p, nextCursor: _c, ...legacy } = result({ scanned: 9 })
|
||||
const fetcher = vi.fn().mockResolvedValue(jsonResponse({ success: true, result: legacy }))
|
||||
|
||||
const normalized = await requestArcimDocumentImport('consent-1', true, fetcher)
|
||||
|
||||
expect(normalized).toMatchObject({ total: 9, partial: false, nextCursor: null })
|
||||
expect(JSON.parse((fetcher.mock.calls[0][1] as RequestInit).body as string)).toEqual({
|
||||
consentId: 'consent-1',
|
||||
dryRun: true,
|
||||
})
|
||||
})
|
||||
|
||||
it('sends the cursor only when resuming', async () => {
|
||||
const fetcher = vi.fn().mockResolvedValue(jsonResponse({ success: true, result: result() }))
|
||||
|
||||
await requestArcimDocumentImport('consent-1', false, fetcher, 'file-42')
|
||||
|
||||
expect(JSON.parse((fetcher.mock.calls[0][1] as RequestInit).body as string)).toEqual({
|
||||
consentId: 'consent-1',
|
||||
dryRun: false,
|
||||
cursor: 'file-42',
|
||||
})
|
||||
})
|
||||
|
||||
it('sums every slice and keeps the latest total', () => {
|
||||
const merged = mergeArcimDocumentImportResults(
|
||||
result({ scanned: 17, linked: 15, skipped: 1, unmatched: 1, failed: 0, total: 113, unmatchedSamples: [{ uploadId: 'u1', voucher: 'A1', date: '2026-01-01' }] }),
|
||||
result({ scanned: 20, linked: 18, skipped: 0, unmatched: 1, failed: 1, total: 113, partial: true, nextCursor: 'file-37', unmatchedSamples: [{ uploadId: 'u2', voucher: 'A2', date: '2026-01-02' }] }),
|
||||
)
|
||||
|
||||
expect(merged).toMatchObject({
|
||||
scanned: 37,
|
||||
linked: 33,
|
||||
skipped: 1,
|
||||
unmatched: 2,
|
||||
failed: 1,
|
||||
total: 113,
|
||||
partial: true,
|
||||
nextCursor: 'file-37',
|
||||
})
|
||||
expect(merged.unmatchedSamples).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('loops until the server reports the end, passing the cursor back and reporting running totals', async () => {
|
||||
const fetcher = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({ success: true, result: result({ dryRun: false, scanned: 17, linked: 17, total: 40, partial: true, nextCursor: 'file-17' }) }),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({ success: true, result: result({ dryRun: false, scanned: 17, linked: 16, skipped: 1, total: 40, partial: true, nextCursor: 'file-34' }) }),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({ success: true, result: result({ dryRun: false, scanned: 6, linked: 6, total: 40 }) }),
|
||||
)
|
||||
const onProgress = vi.fn()
|
||||
|
||||
const final = await runArcimDocumentImportToCompletion('consent-1', { fetcher, onProgress })
|
||||
|
||||
expect(fetcher).toHaveBeenCalledTimes(3)
|
||||
const bodies = fetcher.mock.calls.map((call) => JSON.parse((call[1] as RequestInit).body as string))
|
||||
expect(bodies).toEqual([
|
||||
{ consentId: 'consent-1', dryRun: false },
|
||||
{ consentId: 'consent-1', dryRun: false, cursor: 'file-17' },
|
||||
{ consentId: 'consent-1', dryRun: false, cursor: 'file-34' },
|
||||
])
|
||||
expect(onProgress).toHaveBeenCalledTimes(2)
|
||||
expect(onProgress.mock.calls[1][0]).toMatchObject({ scanned: 34, linked: 33, skipped: 1, total: 40 })
|
||||
expect(final).toMatchObject({ scanned: 40, linked: 39, skipped: 1, total: 40, partial: false, nextCursor: null })
|
||||
})
|
||||
|
||||
it('fails (does not report completion) when a server hands back a cursor that does not advance', async () => {
|
||||
const fetcher = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(jsonResponse({ success: true, result: result({ dryRun: false, scanned: 5, total: 10, partial: true, nextCursor: 'file-5' }) }))
|
||||
.mockResolvedValueOnce(jsonResponse({ success: true, result: result({ dryRun: false, scanned: 0, total: 10, partial: true, nextCursor: 'file-5' }) }))
|
||||
const onProgress = vi.fn()
|
||||
|
||||
await expect(
|
||||
runArcimDocumentImportToCompletion('consent-1', { fetcher, onProgress }),
|
||||
).rejects.toMatchObject({ problem: { code: ARCIM_DOCUMENT_IMPORT_STALLED } })
|
||||
|
||||
expect(fetcher).toHaveBeenCalledTimes(2)
|
||||
// The slice that did land was reported, so the UI keeps honest totals.
|
||||
expect(onProgress).toHaveBeenCalledTimes(1)
|
||||
expect(onProgress.mock.calls[0][0]).toMatchObject({ scanned: 5, total: 10 })
|
||||
})
|
||||
|
||||
it('fails when a partial answer carries no cursor at all', async () => {
|
||||
const fetcher = vi.fn().mockResolvedValue(
|
||||
jsonResponse({ success: true, result: result({ dryRun: false, scanned: 5, total: 10, partial: true, nextCursor: null }) }),
|
||||
)
|
||||
|
||||
await expect(runArcimDocumentImportToCompletion('consent-1', { fetcher })).rejects.toMatchObject({
|
||||
problem: { code: ARCIM_DOCUMENT_IMPORT_STALLED },
|
||||
})
|
||||
expect(fetcher).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('keeps the import phase while recording running totals, then completes with the honest total', () => {
|
||||
const importing = arcimDocumentImportReducer(
|
||||
{ phase: 'offered', found: 40, result: result({ dryRun: true, total: 40 }), problem: null },
|
||||
{ type: 'import-started' },
|
||||
)
|
||||
const progressed = arcimDocumentImportReducer(importing, {
|
||||
type: 'import-progress',
|
||||
result: result({ dryRun: false, scanned: 17, linked: 17, total: 40, partial: true, nextCursor: 'file-17' }),
|
||||
})
|
||||
expect(progressed).toMatchObject({ phase: 'importing', result: { scanned: 17, total: 40 } })
|
||||
|
||||
const complete = arcimDocumentImportReducer(progressed, {
|
||||
type: 'import-succeeded',
|
||||
result: result({ dryRun: false, scanned: 40, linked: 39, skipped: 1, total: 40 }),
|
||||
})
|
||||
expect(complete).toMatchObject({ phase: 'complete', found: 40, result: { linked: 39, skipped: 1 } })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -63,6 +63,42 @@ export interface ArcimDocumentImportResult {
|
||||
failed: number
|
||||
dryRun: boolean
|
||||
unmatchedSamples: { uploadId: string; voucher: string; date: string }[]
|
||||
/** Attachments in the provider's whole list (not just this call). */
|
||||
total: number
|
||||
/** The server stopped at its time budget; continue after nextCursor. */
|
||||
partial: boolean
|
||||
/** Provider id of the last handled attachment, or null when complete. */
|
||||
nextCursor: string | null
|
||||
}
|
||||
|
||||
const MAX_UNMATCHED_SAMPLES = 20
|
||||
|
||||
/**
|
||||
* Fold one server call into the running totals of a multi-call import. The
|
||||
* route processes a slice per call (hosted function limit), so the numbers
|
||||
* the user sees must be the sum of every slice, not the last one.
|
||||
*/
|
||||
export function mergeArcimDocumentImportResults(
|
||||
accumulated: ArcimDocumentImportResult | null,
|
||||
next: ArcimDocumentImportResult,
|
||||
): ArcimDocumentImportResult {
|
||||
if (!accumulated) return next
|
||||
return {
|
||||
provider: next.provider,
|
||||
dryRun: next.dryRun,
|
||||
scanned: accumulated.scanned + next.scanned,
|
||||
linked: accumulated.linked + next.linked,
|
||||
skipped: accumulated.skipped + next.skipped,
|
||||
unmatched: accumulated.unmatched + next.unmatched,
|
||||
failed: accumulated.failed + next.failed,
|
||||
unmatchedSamples: [...accumulated.unmatchedSamples, ...next.unmatchedSamples].slice(
|
||||
0,
|
||||
MAX_UNMATCHED_SAMPLES,
|
||||
),
|
||||
total: next.total,
|
||||
partial: next.partial,
|
||||
nextCursor: next.nextCursor,
|
||||
}
|
||||
}
|
||||
|
||||
export interface ArcimDocumentImportProblem {
|
||||
@@ -133,6 +169,7 @@ export type ArcimDocumentImportAction =
|
||||
| { type: 'discovery-failed'; problem: ArcimDocumentImportProblem }
|
||||
| { type: 'dismissed' }
|
||||
| { type: 'import-started' }
|
||||
| { type: 'import-progress'; result: ArcimDocumentImportResult }
|
||||
| { type: 'import-succeeded'; result: ArcimDocumentImportResult }
|
||||
| { type: 'import-failed'; problem: ArcimDocumentImportProblem }
|
||||
| { type: 'reconnect-started' }
|
||||
@@ -195,10 +232,14 @@ export function arcimDocumentImportReducer(
|
||||
return { ...state, phase: 'dismissed', problem: null }
|
||||
case 'import-started':
|
||||
return { ...state, phase: 'importing', problem: null }
|
||||
case 'import-progress':
|
||||
// Running totals after each server slice; the phase stays 'importing'
|
||||
// so the UI keeps its spinner and shows "x av y".
|
||||
return { ...state, phase: 'importing', result: action.result, problem: null }
|
||||
case 'import-succeeded':
|
||||
return {
|
||||
phase: 'complete',
|
||||
found: state.found || action.result.scanned,
|
||||
found: state.found || action.result.total || action.result.scanned,
|
||||
result: action.result,
|
||||
problem: null,
|
||||
}
|
||||
@@ -242,7 +283,13 @@ function problemFromPayload(payload: unknown): ArcimDocumentImportProblem {
|
||||
}
|
||||
}
|
||||
|
||||
function isDocumentImportResult(value: unknown): value is ArcimDocumentImportResult {
|
||||
type WireDocumentImportResult = Omit<
|
||||
ArcimDocumentImportResult,
|
||||
'total' | 'partial' | 'nextCursor'
|
||||
> &
|
||||
Partial<Pick<ArcimDocumentImportResult, 'total' | 'partial' | 'nextCursor'>>
|
||||
|
||||
function isDocumentImportResult(value: unknown): value is WireDocumentImportResult {
|
||||
if (!value || typeof value !== 'object') return false
|
||||
const result = value as Partial<ArcimDocumentImportResult>
|
||||
return (
|
||||
@@ -257,16 +304,37 @@ function isDocumentImportResult(value: unknown): value is ArcimDocumentImportRes
|
||||
)
|
||||
}
|
||||
|
||||
/** Call the existing POST route for both discovery and the actual import. */
|
||||
/** Older servers answer without the resume fields: a single complete slice. */
|
||||
function normalizeDocumentImportResult(
|
||||
result: WireDocumentImportResult,
|
||||
): ArcimDocumentImportResult {
|
||||
return {
|
||||
...result,
|
||||
total: typeof result.total === 'number' ? result.total : result.scanned,
|
||||
partial: result.partial === true,
|
||||
nextCursor:
|
||||
typeof result.nextCursor === 'string' && result.nextCursor.length > 0 && result.partial === true
|
||||
? result.nextCursor
|
||||
: null,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Call the existing POST route for both discovery and the actual import.
|
||||
* `cursor` resumes a partial import; omit it for discovery and the first slice.
|
||||
*/
|
||||
export async function requestArcimDocumentImport(
|
||||
consentId: string,
|
||||
dryRun: boolean,
|
||||
fetcher: typeof fetch = fetch,
|
||||
cursor?: string,
|
||||
): Promise<ArcimDocumentImportResult> {
|
||||
const response = await fetcher(ARCIM_DOCUMENT_IMPORT_ENDPOINT, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ consentId, dryRun }),
|
||||
body: JSON.stringify(
|
||||
cursor === undefined ? { consentId, dryRun } : { consentId, dryRun, cursor },
|
||||
),
|
||||
})
|
||||
const payload = await response.json().catch(() => null)
|
||||
|
||||
@@ -283,5 +351,50 @@ export async function requestArcimDocumentImport(
|
||||
})
|
||||
}
|
||||
|
||||
return result
|
||||
return normalizeDocumentImportResult(result)
|
||||
}
|
||||
|
||||
/** Far above any real list: a guard against a server that never completes. */
|
||||
const MAX_IMPORT_ROUNDS = 500
|
||||
|
||||
export const ARCIM_DOCUMENT_IMPORT_STALLED = 'ARCIM_DOCUMENT_IMPORT_STALLED'
|
||||
|
||||
/**
|
||||
* Run the real import to completion: the route works through one time-budgeted
|
||||
* slice per call and hands back a cursor, so this loops until the server says
|
||||
* it reached the end, reporting running totals after every slice. A thrown
|
||||
* request error aborts the loop; the slices already archived stay linked and
|
||||
* the next attempt skips them by hash. A server that keeps answering "partial"
|
||||
* without advancing (or past the round guard) is reported as a failure, never
|
||||
* as a completed import: the retry button resumes safely.
|
||||
*/
|
||||
export async function runArcimDocumentImportToCompletion(
|
||||
consentId: string,
|
||||
options: {
|
||||
fetcher?: typeof fetch
|
||||
onProgress?: (accumulated: ArcimDocumentImportResult) => void
|
||||
} = {},
|
||||
): Promise<ArcimDocumentImportResult> {
|
||||
const fetcher = options.fetcher ?? fetch
|
||||
let accumulated: ArcimDocumentImportResult | null = null
|
||||
let cursor: string | undefined
|
||||
|
||||
for (let round = 0; round < MAX_IMPORT_ROUNDS; round++) {
|
||||
const slice = await requestArcimDocumentImport(consentId, false, fetcher, cursor)
|
||||
accumulated = mergeArcimDocumentImportResults(accumulated, slice)
|
||||
if (!slice.partial) {
|
||||
return { ...accumulated, partial: false, nextCursor: null }
|
||||
}
|
||||
if (slice.nextCursor === null || (cursor !== undefined && slice.nextCursor <= cursor)) {
|
||||
break
|
||||
}
|
||||
cursor = slice.nextCursor
|
||||
options.onProgress?.(accumulated)
|
||||
}
|
||||
|
||||
throw new ArcimDocumentImportRequestError({
|
||||
code: ARCIM_DOCUMENT_IMPORT_STALLED,
|
||||
requestId: null,
|
||||
reconnectRequired: false,
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user