feat(import): stream real Arcim migration progress as NDJSON (#1485)
* feat(import): stream real Arcim migration progress as NDJSON The /migrate route ran the orchestrator to completion and answered with one JSON blob, so the wizard faked its progress bar: a hardcoded 55% anchor and a static step label for a phase that can take minutes. The orchestrator has had a real onProgress channel (eight emit points with Swedish step labels and anchors) since it was written; the route just never passed it. Now a request with Accept: application/x-ndjson gets a streamed response: one line per orchestrator progress event, then a terminal done line with the results or an error line carrying the same structured envelope the JSON path returns (the 200 status is already committed once the stream opens). Callers without the header keep the original single-JSON contract, so pre-deploy tabs and the existing error-mapping tests are untouched. The wizard opts in, drives MigratingStep from the real labels and anchors (mapped onto the 55-100 slice of the wizard bar), and treats a dropped connection as unconfirmed rather than failed, since the migration keeps running server-side and a blind retry could double-import. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * docs: record the opt-in NDJSON streaming decision for /migrate Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Claude Fable 5
Jakob Wennberg
Claude Fable 5
Jakob Wennberg
parent
4ccebd3645
commit
1d01010928
@@ -0,0 +1,161 @@
|
||||
import { describe, it, expect, beforeEach, vi, type Mock } from 'vitest'
|
||||
import { createMockSupabase } from '@/tests/helpers'
|
||||
import type { ExtensionContext } from '@/lib/extensions/types'
|
||||
|
||||
/**
|
||||
* Locks the /migrate NDJSON streaming contract (opt-in via
|
||||
* `Accept: application/x-ndjson`):
|
||||
*
|
||||
* - each orchestrator onProgress call becomes one `progress` line with the
|
||||
* real step label and anchor (the wizard's fake 55→100 jump is dead)
|
||||
* - success ends with a terminal `done` line carrying the results, after
|
||||
* acceptConsent ran
|
||||
* - an orchestrator failure ends with a terminal `error` line carrying the
|
||||
* SAME structured envelope the JSON path answers with (the 200 status is
|
||||
* already committed when the stream opens)
|
||||
* - a request WITHOUT the Accept header keeps the original JSON contract
|
||||
*/
|
||||
|
||||
vi.mock('../lib/migration-orchestrator', () => ({
|
||||
executeMigration: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('../lib/provider-client', () => ({
|
||||
createConsent: vi.fn(),
|
||||
getConsent: vi.fn(),
|
||||
listConsents: vi.fn(),
|
||||
generateOtc: vi.fn(),
|
||||
consumeOAuthState: vi.fn(),
|
||||
getAuthUrl: vi.fn(),
|
||||
exchangeAuthToken: vi.fn(),
|
||||
submitProviderToken: vi.fn(),
|
||||
acceptConsent: vi.fn().mockResolvedValue(undefined),
|
||||
deleteConsent: vi.fn(),
|
||||
resolveConsent: vi.fn(),
|
||||
fetchCompanyInfoDirect: vi.fn(),
|
||||
ProviderTokenInvalidError: class ProviderTokenInvalidError extends Error {},
|
||||
ConsentNotFoundError: class ConsentNotFoundError extends Error {},
|
||||
}))
|
||||
|
||||
import { arcimMigrationExtension } from '../index'
|
||||
import { executeMigration } from '../lib/migration-orchestrator'
|
||||
import { getConsent, acceptConsent } from '../lib/provider-client'
|
||||
|
||||
const migrateRoute = (arcimMigrationExtension.apiRoutes ?? []).find(
|
||||
(r) => r.method === 'POST' && r.path === '/migrate',
|
||||
)!
|
||||
|
||||
type RouteHandler = (request: Request, ctx?: ExtensionContext) => Promise<Response>
|
||||
const handler = migrateRoute.handler as RouteHandler
|
||||
|
||||
function buildCtx(): ExtensionContext {
|
||||
const { supabase, mockResult } = createMockSupabase()
|
||||
// The SIE guard awaits `from('sie_imports').select(..,{count,head}).eq().eq()`.
|
||||
mockResult({ count: 1 })
|
||||
;(supabase as unknown as { auth: unknown }).auth = {
|
||||
getUser: vi.fn().mockResolvedValue({ data: { user: { id: 'user-1' } } }),
|
||||
}
|
||||
return { supabase, companyId: 'company-1' } as unknown as ExtensionContext
|
||||
}
|
||||
|
||||
function streamRequest() {
|
||||
return new Request('http://localhost/api/extensions/ext/arcim-migration/migrate', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', Accept: 'application/x-ndjson' },
|
||||
body: JSON.stringify({ consentId: 'consent-1' }),
|
||||
})
|
||||
}
|
||||
|
||||
async function readNdjson(res: Response): Promise<Record<string, unknown>[]> {
|
||||
const text = await res.text()
|
||||
return text
|
||||
.split('\n')
|
||||
.filter((line) => line.trim())
|
||||
.map((line) => JSON.parse(line))
|
||||
}
|
||||
|
||||
describe('POST /migrate: NDJSON streaming', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
;(getConsent as Mock).mockResolvedValue({ id: 'consent-1', status: 1, provider: 'visma' })
|
||||
})
|
||||
|
||||
it('streams one progress line per orchestrator event, then a terminal done line with results', async () => {
|
||||
const results = {
|
||||
customers: { total: 5, imported: 5, updated: 0, skipped: 0, skipReasons: {} },
|
||||
stepErrors: [],
|
||||
}
|
||||
;(executeMigration as Mock).mockImplementation(
|
||||
async (opts: { onProgress?: (p: { status: string; currentStep?: string; progress: number }) => void }) => {
|
||||
opts.onProgress?.({ status: 'fetching', currentStep: 'Ansluter till Visma eEkonomi...', progress: 5 })
|
||||
opts.onProgress?.({ status: 'importing', currentStep: 'Importerar kunder...', progress: 20 })
|
||||
opts.onProgress?.({ status: 'completed', currentStep: 'Klart!', progress: 100 })
|
||||
return results
|
||||
},
|
||||
)
|
||||
|
||||
const res = await handler(streamRequest(), buildCtx())
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.headers.get('content-type')).toContain('application/x-ndjson')
|
||||
|
||||
const events = await readNdjson(res)
|
||||
expect(events).toHaveLength(4)
|
||||
expect(events[0]).toEqual({
|
||||
kind: 'progress',
|
||||
status: 'fetching',
|
||||
currentStep: 'Ansluter till Visma eEkonomi...',
|
||||
progress: 5,
|
||||
})
|
||||
expect(events[1]).toMatchObject({ kind: 'progress', currentStep: 'Importerar kunder...', progress: 20 })
|
||||
expect(events[3]).toEqual({ kind: 'done', success: true, results })
|
||||
expect(acceptConsent).toHaveBeenCalledWith('consent-1')
|
||||
})
|
||||
|
||||
it('ends with a terminal error line carrying the structured envelope when the orchestrator fails', async () => {
|
||||
const vismaError = new Error('Visma API error: 403 Forbidden') as Error & {
|
||||
statusCode: number
|
||||
body: string
|
||||
}
|
||||
vismaError.statusCode = 403
|
||||
vismaError.body =
|
||||
'{"ErrorCode":4002,"DeveloperErrorMessage":"ForbiddenRequestException - No access to module: api_standard","ErrorId":"x","Errors":[]}'
|
||||
;(executeMigration as Mock).mockImplementation(
|
||||
async (opts: { onProgress?: (p: { status: string; progress: number }) => void }) => {
|
||||
opts.onProgress?.({ status: 'fetching', progress: 5 })
|
||||
throw vismaError
|
||||
},
|
||||
)
|
||||
|
||||
const res = await handler(streamRequest(), buildCtx())
|
||||
expect(res.status).toBe(200)
|
||||
|
||||
const events = await readNdjson(res)
|
||||
const terminal = events[events.length - 1] as {
|
||||
kind: string
|
||||
error: { code: string; message: string }
|
||||
}
|
||||
expect(terminal.kind).toBe('error')
|
||||
expect(terminal.error.code).toBe('PROVIDER_API_MODULE_INACTIVE')
|
||||
expect(terminal.error.message).toContain('Appar och tillägg')
|
||||
expect(acceptConsent).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps the single-JSON contract when the Accept header is absent', async () => {
|
||||
const results = { customers: { total: 1, imported: 1, updated: 0, skipped: 0, skipReasons: {} } }
|
||||
;(executeMigration as Mock).mockResolvedValue(results)
|
||||
|
||||
const res = await handler(
|
||||
new Request('http://localhost/api/extensions/ext/arcim-migration/migrate', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ consentId: 'consent-1' }),
|
||||
}),
|
||||
buildCtx(),
|
||||
)
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.headers.get('content-type')).toContain('application/json')
|
||||
expect(await res.json()).toEqual({ success: true, results })
|
||||
})
|
||||
})
|
||||
@@ -129,6 +129,27 @@ async function buildArcimOAuthUrl(consentId: string, provider: ArcimProvider): P
|
||||
return url
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a failed /migrate run to its structured error response. Shared by the
|
||||
* JSON path (returned as-is) and the NDJSON path (body re-sent as the
|
||||
* terminal `error` event, since the stream's 200 status is already
|
||||
* committed). Consent missing or owned by another company: same 404 either way.
|
||||
*/
|
||||
function migrateFailureResponse(error: unknown, consentId: string): NextResponse {
|
||||
if (error instanceof ConsentNotFoundError) {
|
||||
return errorResponseFromCode('PROVIDER_CONSENT_NOT_FOUND', moduleLog, {
|
||||
details: { consentId },
|
||||
})
|
||||
}
|
||||
const classified = classifyProviderError(error)
|
||||
return errorResponseFromCode(classified ?? 'PROVIDER_MIGRATE_FAILED', moduleLog, {
|
||||
details: {
|
||||
reason: error instanceof Error ? error.message : 'unknown',
|
||||
classified: classified ?? 'unclassified',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider Migration extension
|
||||
*
|
||||
@@ -1093,7 +1114,7 @@ export const arcimMigrationExtension: Extension = {
|
||||
|
||||
log.info(`Starting migration for user ${user.id} from ${consent.provider}`)
|
||||
|
||||
const results = await executeMigration({
|
||||
const migrationOptions = {
|
||||
consentId,
|
||||
companyId,
|
||||
userId: user.id,
|
||||
@@ -1104,7 +1125,59 @@ export const arcimMigrationExtension: Extension = {
|
||||
importSalesInvoices,
|
||||
importSupplierInvoices,
|
||||
reconcileVouchers,
|
||||
})
|
||||
}
|
||||
|
||||
// Streaming mode (the migration wizard opts in via Accept): one
|
||||
// NDJSON line per orchestrator progress event, then a terminal
|
||||
// `done` or `error` line. Errors after the stream opens cannot
|
||||
// change the HTTP status, so the terminal `error` line carries the
|
||||
// same structured envelope the JSON path answers with. Callers
|
||||
// without the Accept header keep the original JSON contract.
|
||||
if ((request.headers.get('accept') ?? '').includes('application/x-ndjson')) {
|
||||
const encoder = new TextEncoder()
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
async start(controller) {
|
||||
const send = (event: Record<string, unknown>) => {
|
||||
try {
|
||||
controller.enqueue(encoder.encode(JSON.stringify(event) + '\n'))
|
||||
} catch {
|
||||
// Reader cancelled (tab closed, navigation). The migration
|
||||
// keeps running server-side; we just stop narrating.
|
||||
}
|
||||
}
|
||||
try {
|
||||
const results = await executeMigration({
|
||||
...migrationOptions,
|
||||
onProgress: (p) => send({
|
||||
kind: 'progress',
|
||||
status: p.status,
|
||||
currentStep: p.currentStep,
|
||||
progress: p.progress,
|
||||
}),
|
||||
})
|
||||
log.info('Migration completed:', results)
|
||||
// Mark consent as fully accepted now that data has been imported
|
||||
await acceptConsent(consentId)
|
||||
send({ kind: 'done', success: true, results })
|
||||
} catch (error) {
|
||||
log.error('arcim migration failed', error as Error)
|
||||
const envelope = await migrateFailureResponse(error, consentId).json()
|
||||
send({ kind: 'error', ...envelope })
|
||||
} finally {
|
||||
controller.close()
|
||||
}
|
||||
},
|
||||
})
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
'Content-Type': 'application/x-ndjson; charset=utf-8',
|
||||
'Cache-Control': 'no-store',
|
||||
'X-Accel-Buffering': 'no',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const results = await executeMigration(migrationOptions)
|
||||
|
||||
log.info('Migration completed:', results)
|
||||
|
||||
@@ -1114,19 +1187,7 @@ export const arcimMigrationExtension: Extension = {
|
||||
return NextResponse.json({ success: true, results })
|
||||
} catch (error) {
|
||||
log.error('arcim migration failed', error as Error)
|
||||
// Consent missing or owned by another company: same 404 either way.
|
||||
if (error instanceof ConsentNotFoundError) {
|
||||
return errorResponseFromCode('PROVIDER_CONSENT_NOT_FOUND', moduleLog, {
|
||||
details: { consentId },
|
||||
})
|
||||
}
|
||||
const classified = classifyProviderError(error)
|
||||
return errorResponseFromCode(classified ?? 'PROVIDER_MIGRATE_FAILED', moduleLog, {
|
||||
details: {
|
||||
reason: error instanceof Error ? error.message : 'unknown',
|
||||
classified: classified ?? 'unclassified',
|
||||
},
|
||||
})
|
||||
return migrateFailureResponse(error, consentId)
|
||||
}
|
||||
},
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user