fix(migration): resumable underlag import without inline extraction + same-origin MCP storage URLs (#1783)

* fix(migration): resumable underlag import without inline extraction, same-origin MCP storage URLs The Fortnox underlag import ran every file's AI extraction inline inside one request and hit the hosted 300 s function limit after ~17 of 113 files (twice on 2026-08-21); the UI showed the generic "underlagen kunde inte importeras" although the files it did reach were linked. The import now works in time-budgeted slices with a stable cursor (the UI loops until the server reports the end and shows "x av y") and opts out of extraction (extractionOwner 'none', stamped skipped:opted_out): every file is linked to its posted verifikat on arrival, so the booking is already known. MCP signed Storage URLs (upload_url, signed_url, download_url) are served through a same-origin proxy, /api/storage/[...path], because Claude Desktop's sandbox only reaches the MCP host and blocked the PUT to <project>.supabase.co. The signed token stays the only credential; the proxy forwards only signed documents-bucket paths to our own Storage host and is a no-op rewrite when NEXT_PUBLIC_APP_URL is unset. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm * fix(mcp): keep the storage-proxy note out of the size-capped tool descriptions The per-tool 280-char cap and the tools/list payload ceiling both tripped on the two sentences added to gnubok_create_document_upload and gnubok_get_document_content; the why now lives in a code comment. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm * fix(review): id cursor, stall = error, capped upload body, encoded dot segments Review follow-ups on #1783: - the import cursor is the last handled provider attachment id, not an index, so a file Fortnox adds or removes mid-sweep shifts nothing - a partial answer whose cursor does not advance (or the round guard) is reported as ARCIM_DOCUMENT_IMPORT_STALLED instead of "complete"; the slices already landed stay reported and the retry button resumes - the storage proxy reads the PUT body as a capped stream instead of buffering an unbounded payload before measuring it - object paths are rejected when any segment decodes to "." or ".." (or holds a separator), and the URL fetch() would actually request is re-checked against the allowlist after normalisation - download_url description no longer claims a direct Storage URL Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-21 15:28:37 +02:00
parent 0531576807
commit 524d9978f1
21 changed files with 1326 additions and 20 deletions
@@ -1,5 +1,6 @@
 import { describe, expect, it, vi } from 'vitest'
 import {
+  ARCIM_DOCUMENT_IMPORT_STALLED,
  ARCIM_DOCUMENT_OAUTH_RESUME_KEY,
  INITIAL_ARCIM_DOCUMENT_IMPORT_STATE,
  PROVIDER_DOCUMENT_SCOPES_REQUIRED,
@@ -8,8 +9,10 @@ import {
  arcimDocumentImportReducer,
  documentOAuthProblemFromReason,
  parseArcimDocumentOAuthResume,
+  mergeArcimDocumentImportResults,
  requestArcimDocumentImport,
  resolveArcimDocumentFollowUpProvider,
+  runArcimDocumentImportToCompletion,
  watchArcimOAuthPopup,
  type ArcimDocumentImportResult,
 } from '../arcim-document-import-flow'
@@ -26,10 +29,20 @@ function result(
    failed: 0,
    dryRun: true,
    unmatchedSamples: [],
+    total: 7,
+    partial: false,
+    nextCursor: null,
    ...overrides,
  }
 }

+function jsonResponse(body: unknown, status = 200): Response {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: { 'Content-Type': 'application/json' },
+  })
+}
+
 describe('Fortnox document follow-up state', () => {
  it('offers the prompt after a successful Fortnox migration using the honest found count', () => {
    const discovering = arcimDocumentImportReducer(
@@ -298,3 +311,123 @@ describe('document scope OAuth recovery', () => {
    vi.useRealTimers()
  })
 })
+
+describe('resumable import (one server slice per call)', () => {
+  it('accepts an older server answer without resume fields as one complete slice', async () => {
+    const { total: _t, partial: _p, nextCursor: _c, ...legacy } = result({ scanned: 9 })
+    const fetcher = vi.fn().mockResolvedValue(jsonResponse({ success: true, result: legacy }))
+
+    const normalized = await requestArcimDocumentImport('consent-1', true, fetcher)
+
+    expect(normalized).toMatchObject({ total: 9, partial: false, nextCursor: null })
+    expect(JSON.parse((fetcher.mock.calls[0][1] as RequestInit).body as string)).toEqual({
+      consentId: 'consent-1',
+      dryRun: true,
+    })
+  })
+
+  it('sends the cursor only when resuming', async () => {
+    const fetcher = vi.fn().mockResolvedValue(jsonResponse({ success: true, result: result() }))
+
+    await requestArcimDocumentImport('consent-1', false, fetcher, 'file-42')
+
+    expect(JSON.parse((fetcher.mock.calls[0][1] as RequestInit).body as string)).toEqual({
+      consentId: 'consent-1',
+      dryRun: false,
+      cursor: 'file-42',
+    })
+  })
+
+  it('sums every slice and keeps the latest total', () => {
+    const merged = mergeArcimDocumentImportResults(
+      result({ scanned: 17, linked: 15, skipped: 1, unmatched: 1, failed: 0, total: 113, unmatchedSamples: [{ uploadId: 'u1', voucher: 'A1', date: '2026-01-01' }] }),
+      result({ scanned: 20, linked: 18, skipped: 0, unmatched: 1, failed: 1, total: 113, partial: true, nextCursor: 'file-37', unmatchedSamples: [{ uploadId: 'u2', voucher: 'A2', date: '2026-01-02' }] }),
+    )
+
+    expect(merged).toMatchObject({
+      scanned: 37,
+      linked: 33,
+      skipped: 1,
+      unmatched: 2,
+      failed: 1,
+      total: 113,
+      partial: true,
+      nextCursor: 'file-37',
+    })
+    expect(merged.unmatchedSamples).toHaveLength(2)
+  })
+
+  it('loops until the server reports the end, passing the cursor back and reporting running totals', async () => {
+    const fetcher = vi
+      .fn()
+      .mockResolvedValueOnce(
+        jsonResponse({ success: true, result: result({ dryRun: false, scanned: 17, linked: 17, total: 40, partial: true, nextCursor: 'file-17' }) }),
+      )
+      .mockResolvedValueOnce(
+        jsonResponse({ success: true, result: result({ dryRun: false, scanned: 17, linked: 16, skipped: 1, total: 40, partial: true, nextCursor: 'file-34' }) }),
+      )
+      .mockResolvedValueOnce(
+        jsonResponse({ success: true, result: result({ dryRun: false, scanned: 6, linked: 6, total: 40 }) }),
+      )
+    const onProgress = vi.fn()
+
+    const final = await runArcimDocumentImportToCompletion('consent-1', { fetcher, onProgress })
+
+    expect(fetcher).toHaveBeenCalledTimes(3)
+    const bodies = fetcher.mock.calls.map((call) => JSON.parse((call[1] as RequestInit).body as string))
+    expect(bodies).toEqual([
+      { consentId: 'consent-1', dryRun: false },
+      { consentId: 'consent-1', dryRun: false, cursor: 'file-17' },
+      { consentId: 'consent-1', dryRun: false, cursor: 'file-34' },
+    ])
+    expect(onProgress).toHaveBeenCalledTimes(2)
+    expect(onProgress.mock.calls[1][0]).toMatchObject({ scanned: 34, linked: 33, skipped: 1, total: 40 })
+    expect(final).toMatchObject({ scanned: 40, linked: 39, skipped: 1, total: 40, partial: false, nextCursor: null })
+  })
+
+  it('fails (does not report completion) when a server hands back a cursor that does not advance', async () => {
+    const fetcher = vi
+      .fn()
+      .mockResolvedValueOnce(jsonResponse({ success: true, result: result({ dryRun: false, scanned: 5, total: 10, partial: true, nextCursor: 'file-5' }) }))
+      .mockResolvedValueOnce(jsonResponse({ success: true, result: result({ dryRun: false, scanned: 0, total: 10, partial: true, nextCursor: 'file-5' }) }))
+    const onProgress = vi.fn()
+
+    await expect(
+      runArcimDocumentImportToCompletion('consent-1', { fetcher, onProgress }),
+    ).rejects.toMatchObject({ problem: { code: ARCIM_DOCUMENT_IMPORT_STALLED } })
+
+    expect(fetcher).toHaveBeenCalledTimes(2)
+    // The slice that did land was reported, so the UI keeps honest totals.
+    expect(onProgress).toHaveBeenCalledTimes(1)
+    expect(onProgress.mock.calls[0][0]).toMatchObject({ scanned: 5, total: 10 })
+  })
+
+  it('fails when a partial answer carries no cursor at all', async () => {
+    const fetcher = vi.fn().mockResolvedValue(
+      jsonResponse({ success: true, result: result({ dryRun: false, scanned: 5, total: 10, partial: true, nextCursor: null }) }),
+    )
+
+    await expect(runArcimDocumentImportToCompletion('consent-1', { fetcher })).rejects.toMatchObject({
+      problem: { code: ARCIM_DOCUMENT_IMPORT_STALLED },
+    })
+    expect(fetcher).toHaveBeenCalledTimes(1)
+  })
+
+  it('keeps the import phase while recording running totals, then completes with the honest total', () => {
+    const importing = arcimDocumentImportReducer(
+      { phase: 'offered', found: 40, result: result({ dryRun: true, total: 40 }), problem: null },
+      { type: 'import-started' },
+    )
+    const progressed = arcimDocumentImportReducer(importing, {
+      type: 'import-progress',
+      result: result({ dryRun: false, scanned: 17, linked: 17, total: 40, partial: true, nextCursor: 'file-17' }),
+    })
+    expect(progressed).toMatchObject({ phase: 'importing', result: { scanned: 17, total: 40 } })
+
+    const complete = arcimDocumentImportReducer(progressed, {
+      type: 'import-succeeded',
+      result: result({ dryRun: false, scanned: 40, linked: 39, skipped: 1, total: 40 }),
+    })
+    expect(complete).toMatchObject({ phase: 'complete', found: 40, result: { linked: 39, skipped: 1 } })
+  })
+})