fix(migration): resumable underlag import without inline extraction + same-origin MCP storage URLs (#1783)

* fix(migration): resumable underlag import without inline extraction, same-origin MCP storage URLs

The Fortnox underlag import ran every file's AI extraction inline inside
one request and hit the hosted 300 s function limit after ~17 of 113 files
(twice on 2026-08-21); the UI showed the generic "underlagen kunde inte
importeras" although the files it did reach were linked. The import now
works in time-budgeted slices with a stable cursor (the UI loops until the
server reports the end and shows "x av y") and opts out of extraction
(extractionOwner 'none', stamped skipped:opted_out): every file is linked
to its posted verifikat on arrival, so the booking is already known.

MCP signed Storage URLs (upload_url, signed_url, download_url) are served
through a same-origin proxy, /api/storage/[...path], because Claude
Desktop's sandbox only reaches the MCP host and blocked the PUT to
<project>.supabase.co. The signed token stays the only credential; the
proxy forwards only signed documents-bucket paths to our own Storage host
and is a no-op rewrite when NEXT_PUBLIC_APP_URL is unset.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm

* fix(mcp): keep the storage-proxy note out of the size-capped tool descriptions

The per-tool 280-char cap and the tools/list payload ceiling both tripped on
the two sentences added to gnubok_create_document_upload and
gnubok_get_document_content; the why now lives in a code comment.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm

* fix(review): id cursor, stall = error, capped upload body, encoded dot segments

Review follow-ups on #1783:
- the import cursor is the last handled provider attachment id, not an
  index, so a file Fortnox adds or removes mid-sweep shifts nothing
- a partial answer whose cursor does not advance (or the round guard) is
  reported as ARCIM_DOCUMENT_IMPORT_STALLED instead of "complete"; the
  slices already landed stay reported and the retry button resumes
- the storage proxy reads the PUT body as a capped stream instead of
  buffering an unbounded payload before measuring it
- object paths are rejected when any segment decodes to "." or ".." (or
  holds a separator), and the URL fetch() would actually request is
  re-checked against the allowlist after normalisation
- download_url description no longer claims a direct Storage URL

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013YoZ8iboyTj221axW6Gdtm

---------

Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
2026-08-21 15:28:37 +02:00
committed by GitHub
co-authored by Claude Fable 5 Jakob Wennberg
parent 0531576807
commit 524d9978f1
21 changed files with 1326 additions and 20 deletions
@@ -152,6 +152,45 @@ describe('POST /import-documents', () => {
)
})
it('passes a non-empty string cursor through and restarts from the top for anything else', async () => {
;(importProviderDocuments as Mock).mockResolvedValue({
provider: 'fortnox',
scanned: 1,
linked: 1,
skipped: 0,
unmatched: 0,
failed: 0,
dryRun: false,
unmatchedSamples: [],
total: 113,
partial: true,
nextCursor: 'file-18',
})
const withCursor = createMockRequest(
'http://localhost/api/extensions/ext/arcim-migration/import-documents',
{ method: 'POST', body: { consentId: 'consent-1', dryRun: false, cursor: 'file-17' } },
)
const { status, body } = await parseJsonResponse<{
result: { partial: boolean; nextCursor: string | null }
}>(await handler(withCursor, buildContext()))
expect(status).toBe(200)
expect(body.result).toMatchObject({ partial: true, nextCursor: 'file-18' })
expect(importProviderDocuments).toHaveBeenLastCalledWith(
expect.objectContaining({ consentId: 'consent-1', dryRun: false, cursor: 'file-17' }),
)
const garbage = createMockRequest(
'http://localhost/api/extensions/ext/arcim-migration/import-documents',
{ method: 'POST', body: { consentId: 'consent-1', cursor: 17 } },
)
await handler(garbage, buildContext())
expect(importProviderDocuments).toHaveBeenLastCalledWith(
expect.objectContaining({ cursor: null }),
)
})
it('asks the user to reconnect only once the connect request carries the scopes', async () => {
fortnoxOAuth.documentScopesApproved = true
;(importProviderDocuments as Mock).mockRejectedValue(
@@ -177,11 +177,16 @@ describe('importProviderDocuments', () => {
expect(userId).toBe(USER)
expect(companyId).toBe(COMPANY)
expect(file).toMatchObject({ name: 'Kvitto.pdf', type: 'application/pdf' })
// extractionOwner 'none': the file arrives linked to its posted
// verifikat, so no paid model pass (it was the inline extraction that
// blew the hosted function budget on a 113-file import).
expect(metadata).toEqual({
upload_source: 'api',
journal_entry_id: 'je-1',
idempotency_key: 'je-1',
extractionOwner: 'none',
})
expect(result).toMatchObject({ total: 1, partial: false, nextCursor: null })
})
it('skips a receipt already archived on the same verifikat (sha256 + journal entry idempotency)', async () => {
@@ -268,6 +273,7 @@ describe('importProviderDocuments', () => {
upload_source: 'api',
journal_entry_id: 'je-1',
idempotency_key: 'je-1',
extractionOwner: 'none',
})
})
@@ -648,4 +654,123 @@ describe('importProviderDocuments', () => {
const [, , , file] = mockUpload.mock.calls[0]
expect(file).toMatchObject({ name: 'Kvitto.jpg', type: 'image/jpeg' })
})
describe('time budget and cursor (resumable sweep)', () => {
const THREE_CONNECTIONS: FortnoxFileConnection[] = [
{ ...FORTNOX_CONNECTION, fileId: 'file-c', name: 'c.pdf' },
{ ...FORTNOX_CONNECTION, fileId: 'file-a', name: 'a.pdf' },
{ ...FORTNOX_CONNECTION, fileId: 'file-b', name: 'b.pdf' },
]
it('stops after the attachment in flight once the budget is spent and hands back a cursor', async () => {
const supabase = wireFortnox({ connections: THREE_CONNECTIONS })
// Each download "costs" 100 ms of wall clock; the budget allows one.
let clock = 0
mockDownloadFortnoxArchiveFile.mockImplementation(async () => {
clock += 100
return { bytes: bytesOf('FORTNOX-PDF-' + clock), contentType: 'application/pdf' }
})
const first = await importProviderDocuments({
supabase,
companyId: COMPANY,
userId: USER,
consentId: 'c1',
timeBudgetMs: 50,
now: () => clock,
})
expect(first).toMatchObject({
total: 3,
scanned: 1,
linked: 1,
partial: true,
nextCursor: 'file-a',
})
// Sorted by provider id, so the first slice is file-a, not file-c.
expect(mockUpload.mock.calls[0][3]).toMatchObject({ name: 'a.pdf' })
const second = await importProviderDocuments({
supabase,
companyId: COMPANY,
userId: USER,
consentId: 'c1',
cursor: first.nextCursor,
timeBudgetMs: 1_000_000,
now: () => clock,
})
expect(second).toMatchObject({ total: 3, scanned: 2, linked: 2, partial: false, nextCursor: null })
expect(mockUpload.mock.calls.map((call) => (call[3] as { name: string }).name)).toEqual([
'a.pdf',
'b.pdf',
'c.pdf',
])
})
it('always processes at least one attachment per call even when the budget is already spent', async () => {
const supabase = wireFortnox({ connections: THREE_CONNECTIONS })
const result = await importProviderDocuments({
supabase,
companyId: COMPANY,
userId: USER,
consentId: 'c1',
timeBudgetMs: 0,
now: () => 0,
})
expect(result).toMatchObject({ scanned: 1, linked: 1, partial: true, nextCursor: 'file-a' })
})
it('treats a cursor past the last id as a complete, empty slice', async () => {
const supabase = wireFortnox({ connections: THREE_CONNECTIONS })
const result = await importProviderDocuments({
supabase,
companyId: COMPANY,
userId: USER,
consentId: 'c1',
cursor: 'file-z',
})
expect(result).toMatchObject({ total: 3, scanned: 0, linked: 0, partial: false, nextCursor: null })
expect(mockUpload).not.toHaveBeenCalled()
})
it('resumes after the last handled id even when the provider list changed in between', async () => {
// Slice 1 handled file-a. Before slice 2, Fortnox gained a file sorting
// BEFORE the resume point (file-0) and lost file-b. An index cursor
// would now skip one; an id cursor continues with everything > file-a.
const supabase = wireFortnox({
connections: [
{ ...FORTNOX_CONNECTION, fileId: 'file-0', name: '0.pdf' },
{ ...FORTNOX_CONNECTION, fileId: 'file-a', name: 'a.pdf' },
{ ...FORTNOX_CONNECTION, fileId: 'file-c', name: 'c.pdf' },
{ ...FORTNOX_CONNECTION, fileId: 'file-d', name: 'd.pdf' },
],
})
// Distinct bytes per file: the default mock returns the same bytes for
// every download, and identical content on the same verifikat is
// (correctly) deduped, which is not what this test is about.
mockDownloadFortnoxArchiveFile.mockImplementation(async (_client, _token, id) => ({
bytes: bytesOf('PDF-' + id),
contentType: 'application/pdf',
}))
const result = await importProviderDocuments({
supabase,
companyId: COMPANY,
userId: USER,
consentId: 'c1',
cursor: 'file-a',
})
expect(result).toMatchObject({ total: 4, scanned: 2, linked: 2, partial: false })
expect(mockUpload.mock.calls.map((call) => (call[3] as { name: string }).name)).toEqual([
'c.pdf',
'd.pdf',
])
})
})
})
+19 -1
View File
@@ -1363,10 +1363,23 @@ export const arcimMigrationExtension: Extension = {
let consentId: string | undefined
let dryRun = false
let cursor: string | null = null
try {
const body = (await request.json()) as { consentId?: string; dryRun?: boolean }
const body = (await request.json()) as {
consentId?: string
dryRun?: boolean
cursor?: string
}
consentId = body?.consentId
dryRun = body?.dryRun === true
// Resume point from a previous partial call (the last handled
// provider attachment id, see import-documents.ts); anything else
// restarts from the top, which is always safe: already-archived
// receipts are skipped by hash.
cursor =
typeof body?.cursor === 'string' && body.cursor.length > 0 && body.cursor.length <= 256
? body.cursor
: null
} catch {
// empty/invalid body: consentId check below rejects it
}
@@ -1382,15 +1395,20 @@ export const arcimMigrationExtension: Extension = {
userId: user.id,
consentId,
dryRun,
cursor,
})
log.info('arcim import-documents completed', {
companyId,
dryRun,
cursor,
total: result.total,
scanned: result.scanned,
linked: result.linked,
skipped: result.skipped,
unmatched: result.unmatched,
failed: result.failed,
partial: result.partial,
nextCursor: result.nextCursor,
})
return NextResponse.json({ success: true, dryRun, result })
} catch (error) {
@@ -24,6 +24,22 @@
* critical path: provider document APIs are rate-limited and a full sweep can
* issue hundreds of download calls. Fortnox requires archive and connectfile
* scopes; existing consents must reconnect before this import can run.
*
* Resumable: one call works through the provider's attachment list (sorted
* by provider id so the order is stable between calls) until `timeBudgetMs`
* is spent, then returns `partial: true` with `nextCursor` = the id of the
* last attachment it handled. The caller loops, passing the cursor back,
* until `partial` is false. The cursor is an id, not an index, so a file the
* provider adds or removes mid-sweep shifts nothing: the next slice simply
* continues after the last handled id in sorted order. A 113-file Fortnox
* import used to run every file inline inside one request and hit the
* hosted 300 s function limit after ~17 files (2026-08-21); the UI then
* showed a generic failure even though the files it did reach were linked.
*
* No AI extraction: every file here is linked to a posted verifikat on
* arrival, so the booking is already known. The upload opts out
* (`extractionOwner: 'none'`); it was the per-file model pass that made the
* sweep blow its budget in the first place.
*/
import type { SupabaseClient } from '@supabase/supabase-js'
@@ -78,8 +94,20 @@ export interface ImportDocumentsOptions {
consentId: string
/** Resolve + report what would be attached without downloading or writing. */
dryRun?: boolean
/** Resume after this provider attachment id (sorted order); omit to start from the top. */
cursor?: string | null
/**
* Wall-clock budget for this call. Once spent, the sweep stops after the
* attachment in flight and reports `nextCursor`. Default leaves headroom
* under the hosted 300 s function limit for listing + the final response.
*/
timeBudgetMs?: number
/** Clock, injectable for tests. */
now?: () => number
}
export const DEFAULT_IMPORT_DOCUMENTS_TIME_BUDGET_MS = 200_000
export interface ImportDocumentsResult {
provider: string
/** Provider attachments linked to a voucher that were considered. */
@@ -95,6 +123,12 @@ export interface ImportDocumentsResult {
dryRun: boolean
/** A few unmatched voucher labels, to aid diagnosis without dumping all. */
unmatchedSamples: { uploadId: string; voucher: string; date: string }[]
/** Provider attachments linked to a voucher in total (all calls). */
total: number
/** True when the time budget ran out before the end of the list. */
partial: boolean
/** Last handled attachment id; pass back as `cursor` to continue. Null when complete. */
nextCursor: string | null
}
interface ProviderAttachment {
@@ -232,7 +266,17 @@ function fortnoxSource(
export async function importProviderDocuments(
opts: ImportDocumentsOptions,
): Promise<ImportDocumentsResult> {
const { supabase, companyId, userId, consentId, dryRun = false } = opts
const {
supabase,
companyId,
userId,
consentId,
dryRun = false,
cursor = null,
timeBudgetMs = DEFAULT_IMPORT_DOCUMENTS_TIME_BUDGET_MS,
now = Date.now,
} = opts
const startedAt = now()
let resolved = await resolveConsent(companyId, consentId)
const provider = resolved.consent.provider as string
@@ -246,6 +290,9 @@ export async function importProviderDocuments(
failed: 0,
dryRun,
unmatchedSamples: [],
total: 0,
partial: false,
nextCursor: null,
}
// Unsupported providers are a no-op rather than an error
@@ -263,7 +310,7 @@ export async function importProviderDocuments(
: fortnoxSource(fortnoxClient as FortnoxClient, resolved.accessToken)
// ── Bulk reads (one round of paged requests each, no per-item N+1) ──
const [attachments, periods, vouchers, existingAttachments] = await Promise.all([
const [listedAttachments, periods, vouchers, existingAttachments] = await Promise.all([
source().list(),
fetchFiscalPeriods(supabase, companyId),
fetchSourceRefVouchers(supabase, companyId),
@@ -280,6 +327,15 @@ export async function importProviderDocuments(
),
])
// Stable order across calls: a provider may page its listing differently
// from one call to the next, and the cursor is "continue after this id".
const attachments = [...listedAttachments].sort((a, b) =>
a.id < b.id ? -1 : a.id > b.id ? 1 : 0,
)
result.total = attachments.length
const firstAfterCursor = cursor ? attachments.findIndex((a) => a.id > cursor) : 0
const startIndex = firstAfterCursor === -1 ? attachments.length : firstAfterCursor
// Index gnubok verifikat by (period, series, number) for in-memory resolution.
const voucherIndex = buildVoucherIndex(vouchers)
@@ -302,7 +358,16 @@ export async function importProviderDocuments(
let refreshedAfterUnauthorized = false
for (const attachment of attachments) {
for (let index = startIndex; index < attachments.length; index++) {
// Always make progress: at least one attachment per call, then stop at
// the budget so the route answers well inside the function limit and the
// caller resumes after the last handled id.
if (index > startIndex && now() - startedAt >= timeBudgetMs) {
result.partial = true
result.nextCursor = attachments[index - 1].id
break
}
const attachment = attachments[index]
result.scanned++
const ref = attachment.ref
@@ -363,6 +428,8 @@ export async function importProviderDocuments(
upload_source: 'api',
journal_entry_id: journalEntryId,
idempotency_key: journalEntryId,
// Already booked: the verifikat is the booking. No model pass.
extractionOwner: 'none',
},
)
@@ -405,11 +472,16 @@ export async function importProviderDocuments(
log.info('document import complete', {
companyId,
dryRun,
cursor,
total: result.total,
scanned: result.scanned,
linked: result.linked,
skipped: result.skipped,
unmatched: result.unmatched,
failed: result.failed,
partial: result.partial,
nextCursor: result.nextCursor,
elapsedMs: now() - startedAt,
})
return result
@@ -64,6 +64,13 @@ beforeEach(() => {
describe('document-extraction handler', () => {
// THE dedupe: inbox-owned documents are extracted (and mirrored) by the
// inbox itself. The handler used to race it and pay a second model call.
it('stamps and skips when the uploader opted out (already-booked provider underlag)', async () => {
await handler(payload({ extractionOwner: 'none' }))
expect(extractMock).not.toHaveBeenCalled()
expect(lastStamp()).toBe('skipped:opted_out')
})
it('yields entirely when the inbox owns extraction', async () => {
await handler(payload({ extractionOwner: 'invoice-inbox' }))
expect(supabase.from).not.toHaveBeenCalled()
@@ -7,6 +7,7 @@ import { CAPABILITY } from '@/lib/entitlements/keys'
import { createLogger } from '@/lib/logger'
import { createServiceClient } from '@/lib/supabase/server'
import type { DocumentAttachment } from '@/types'
import type { DocumentExtractionOwner } from '@/lib/events/types'
const log = createLogger('document-extraction')
@@ -62,9 +63,17 @@ export const documentExtractionExtension: Extension = {
document: DocumentAttachment
userId: string
companyId: string
extractionOwner?: 'invoice-inbox'
extractionOwner?: DocumentExtractionOwner
}
if (extractionOwner === 'invoice-inbox') return
if (extractionOwner === 'none') {
// The uploader already holds the booking (provider underlag import
// links each file to its posted verifikat): a model pass would cost
// a paid call per file and, run inline, blew the 300 s route budget
// on a 113-file Fortnox import (2026-08-21). Stamp so nothing polls.
await stamp(createServiceClient(), document.id, 'skipped:opted_out')
return
}
await extractAndPersist(document, companyId)
},
},
@@ -1,4 +1,4 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { makeDocumentAttachment } from '@/tests/helpers'
import { TOOL_SCOPE_MAP } from '@/lib/auth/api-keys'
import { MCP_TOOL_CAPABILITY_MAP } from '@/lib/entitlements/keys'
@@ -51,6 +51,10 @@ function makeQueryBuilder(result: { data: unknown; error: unknown }) {
}
describe('MCP model-free document upload tools', () => {
afterEach(() => {
vi.unstubAllEnvs()
})
beforeEach(() => {
vi.clearAllMocks()
mocks.createPendingDocumentUpload.mockResolvedValue({
@@ -76,6 +80,32 @@ describe('MCP model-free document upload tools', () => {
})
})
// Claude Desktop's sandbox only reaches the MCP host: a signed URL on
// <project>.supabase.co was refused there (2026-08-21), so the tool hands
// out the same-origin /api/storage proxy URL instead.
it('serves the upload URL from the app origin when the signed URL points at our Storage host', async () => {
vi.stubEnv('NEXT_PUBLIC_SUPABASE_URL', 'https://pwxtzglxptnnvjrpixpg.supabase.co')
vi.stubEnv('NEXT_PUBLIC_APP_URL', 'https://app.accounted.se')
mocks.createPendingDocumentUpload.mockResolvedValue({
uploadId,
signedUrl:
'https://pwxtzglxptnnvjrpixpg.supabase.co/storage/v1/object/upload/sign/documents/co/user/pending/up/invoice.pdf?token=signed',
expiresAt: '2026-08-03T12:00:00.000Z',
})
const result = await findTool('gnubok_create_document_upload').execute(
{ file_name: 'invoice.pdf' },
companyId,
userId,
{} as never,
)
expect(result).toMatchObject({
upload_url:
'https://app.accounted.se/api/storage/upload/sign/documents/co/user/pending/up/invoice.pdf?token=signed',
})
})
it('returns an unauthenticated PUT URL without accepting file bytes', async () => {
const tool = findTool('gnubok_create_document_upload')
const result = await tool.execute(
+7 -4
View File
@@ -209,6 +209,7 @@ import {
uploadDocument,
MAX_DOCUMENT_SIZE,
} from '@/lib/core/documents/document-service'
import { toSameOriginStorageUrl } from '@/lib/core/documents/storage-proxy'
import { extractInvoiceFields, ExtractionSchema as InvoiceExtractionSchema, AgentExtractionSchema } from '@/extensions/general/invoice-inbox/lib/extract-invoice-fields'
import { mirrorExtractionToDocument } from '@/extensions/general/invoice-inbox/lib/mirror-extraction'
// Skatteverket filing tools (PR5). Cross-extension lib import, same sanctioned
@@ -9715,9 +9716,11 @@ export const tools: McpTool[] = [
uploadId,
fileName,
)
// Served from the app origin: agent sandboxes (Claude Desktop) only
// reach the MCP host, not <project>.supabase.co. See storage-proxy.ts.
return {
upload_id: reservation.uploadId,
upload_url: reservation.signedUrl,
upload_url: toSameOriginStorageUrl(reservation.signedUrl),
expires_at: reservation.expiresAt,
}
},
@@ -10664,7 +10667,7 @@ export const tools: McpTool[] = [
file_name: doc.file_name,
mime_type: doc.mime_type,
size_bytes: doc.file_size_bytes,
signed_url: signed.signedUrl,
signed_url: toSameOriginStorageUrl(signed.signedUrl),
expires_at: expiresAt,
}
},
@@ -13574,7 +13577,7 @@ export const tools: McpTool[] = [
type: 'object',
additionalProperties: false,
properties: {
download_url: { type: ['string', 'null'], description: 'Signed Supabase Storage URL, valid 1 hour; null when estimate_only=true. Restricted-egress proxies may 403; needs a network with Supabase egress.' },
download_url: { type: ['string', 'null'], description: 'Signed download URL, valid 1 hour, on the app origin (direct Storage URL only when NEXT_PUBLIC_APP_URL is unset); null when estimate_only=true.' },
storage_path: { type: ['string', 'null'] },
file_name: { type: 'string' },
size_bytes: { type: 'number' },
@@ -13679,7 +13682,7 @@ export const tools: McpTool[] = [
const expiresAt = new Date(Date.now() + SIGNED_URL_TTL_SECONDS * 1000).toISOString()
return {
download_url: signed.signedUrl,
download_url: toSameOriginStorageUrl(signed.signedUrl),
storage_path: storagePath,
file_name: fileName,
size_bytes: zipBuffer.byteLength,