/** * Per-provider SIE-over-API fetcher. * * Providers that expose their general ledger as a SIE export over the API get * the "Fortnox-grade" migration experience: the wizard pulls the GL itself * instead of requiring a manual SIE upload. Everything downstream (parsing, * validation, account mapping, replace-mode import) is provider-agnostic: * this module's only job is to produce raw SIE file contents per fiscal year. * * Shared by /preview (latest year, for stats) and /sie-data (all years). */ import { FortnoxClient } from '@/lib/providers/fortnox/client' import { BrioxClient } from '@/lib/providers/briox/client' import { BjornLundenClient } from '@/lib/providers/bjornlunden/client' import { WintClient } from '@/lib/providers/wint/client' import { buildWintSieFile, deriveIbByYear, mapWintAccountForSie, mapWintVoucherForSie, type WintSieVoucher, type WintSieYear, } from '@/lib/providers/wint/sie-builder' import type { ProviderName } from '@/lib/providers/types' import { detectEncoding, decodeBuffer } from '@/lib/import/sie-parser' import { createLogger } from '@/lib/logger' const log = createLogger('extensions/arcim-migration/sie-fetcher') /** * The DEFAULT selection of fiscal years: the current year and the two before * it, keyed on each fiscal year's start year. A default, not a cap (issue * #2211 / #2238): the wizard lets the user add older years, and * fetchProviderSieFiles takes an explicit `years` selection. The default is * the cost bound: every selected year is one SIE export fetched and parsed * inside the single /sie-data invocation (hosted function limit 300 s), so * an older history is the user's own wait, chosen in the preview step. * Derived at call time (not a module constant) so the window rolls forward * automatically at new year without a code change. */ export function getAllowedFiscalYears(now: Date = new Date()): Set { const currentYear = now.getFullYear() return new Set([currentYear - 2, currentYear - 1, currentYear]) } /** * The most fiscal years one import run may select. The bound is the single * /sie-data invocation that fetches and parses one SIE export per selected * year: hosted function limit 300 s; one export call is 15 s per attempt * (FETCH_TIMEOUT_MS in lib/providers/fortnox/client.ts), 3 attempts with * 1 s and 2 s backoff between them (lib/providers/retry.ts defaults, capped * at 30 s), so a year that times out on every attempt costs 15 + 1 + 15 + * 2 + 15 = 48 s. Six such years are 288 s, which leaves the remaining 12 s * for the /financialyears listing, parsing and the response. Older years * beyond the cap go in a second run. Enforced server-side in /sie-data and * mirrored by the preview step's picker (which reads it from /preview). */ export const MAX_SELECTED_FISCAL_YEARS = 6 /** * Thrown by fetchProviderSieFiles when an explicit selection names a year * the source does not have: raised right after the year listing, before any * SIE export is fetched, so an unknown year never costs a provider export. */ export class FiscalYearSelectionError extends Error { constructor(public readonly unknownYears: number[]) { super(`Fiscal years not found at the provider: ${unknownYears.join(', ')}`) this.name = 'FiscalYearSelectionError' } } export interface ProviderSieFile { fiscalYear: number rawContent: string } export interface ProviderSieFetchResult { files: ProviderSieFile[] /** * Every fiscal year available at the provider within the selection (the * default window, or the explicit `years`): also populated when latestOnly * fetched just one file, so /preview can show the full year list without a * second round-trip. */ availableYears: number[] /** * Every fiscal year the source has, oldest first, with the provider's own * bounds and whether it is in the default selection. The preview step * renders these as the year picker, so no year can be left out silently. */ sourceYears: SourceFiscalYear[] /** * Allowed years whose export failed (or came back empty). Callers MUST * surface these to the user: silently importing e.g. 2024+2026 without 2025 * breaks IB/UB continuity between the years without anyone noticing. */ failedYears: { year: number; error: string }[] /** * Source fiscal years that were NOT part of this fetch (outside the * selection), oldest first. Until issue #2211 these were never mentioned: * the result step names them so nobody believes the books are complete. */ omittedYears: SourceFiscalYear[] } /** * A fiscal year as the source reports it. Bounds are the provider's own * (ISO yyyy-mm-dd) so a broken year can be named as "2022-09-01 till * 2023-12-31" rather than as a calendar year that is wrong for it; null when * the provider reported none. `year` (the start year) is the key the whole * import uses for a fiscal year, and what `fetchProviderSieFiles` selects on. */ export interface SourceFiscalYear { year: number fromDate: string | null toDate: string | null /** True when the year falls in the default selection (getAllowedFiscalYears). */ inDefaultSelection: boolean } // Singleton clients (they hold rate limiters) const fortnoxClient = new FortnoxClient() const brioxClient = new BrioxClient() const bjornLundenClient = new BjornLundenClient() const wintClient = new WintClient() /** * True when the provider's API can serve the GL as SIE (no manual upload). * WINT qualifies even though its v1 API has no SIE endpoint: the ledger is * fetched voucher-by-voucher and RENDERED as SIE on our side (Tier A; see * lib/providers/wint/sie-builder.ts). Downstream the file is indistinguishable * from a provider export and goes through the same parse/validate/import path. */ export function providerSupportsSie(provider: ProviderName): boolean { return provider === 'fortnox' || provider === 'briox' || provider === 'bjornlunden' || provider === 'wint' } interface FiscalYearRef { id: string | number year: number /** Period bounds: required by BL, whose export URL is date-ranged. */ fromDate?: string toDate?: string } /** * Fetch SIE type-4 exports from the provider, one file per selected fiscal * year (oldest first). The selection is `opts.years` (start years, as the * preview step's picker sends them) or, when absent, the default window. * Years whose export fails do not block the rest of the migration, but they * are reported in `failedYears` so the caller can warn the user before * importing a gap (IB/UB continuity). Years at the source outside the * selection are reported in `omittedYears`; the full list is `sourceYears`. */ export async function fetchProviderSieFiles( provider: ProviderName, accessToken: string, providerCompanyId: string | undefined, opts?: { latestOnly?: boolean; years?: number[] }, ): Promise { const fetcher = getSieFetcher(provider, providerCompanyId, opts?.years) if (!fetcher) { throw new Error(`Provider ${provider} does not support SIE over API`) } const defaultFiscalYears = getAllowedFiscalYears() const selectedFiscalYears = opts?.years ? new Set(opts.years) : defaultFiscalYears const byYear = (a: FiscalYearRef, b: FiscalYearRef) => a.year - b.year || (a.fromDate ?? '').localeCompare(b.fromDate ?? '') const allYears = (await fetcher.listYears(accessToken)).sort(byYear) if (opts?.years) { // Reject an unknown year here, before the first export: the listing is // the only provider call made so far. const known = new Set(allYears.map((fy) => fy.year)) const unknownYears = opts.years.filter((y) => !known.has(y)) if (unknownYears.length > 0) throw new FiscalYearSelectionError(unknownYears) } const allowedYears = allYears.filter((fy) => selectedFiscalYears.has(fy.year)) const availableYears = allowedYears.map((fy) => fy.year) const toFetch = opts?.latestOnly ? allowedYears.slice(-1) : allowedYears // Both derived from the year list already fetched: naming every source // year, and the ones left out, costs no extra provider call. const describe = (fy: FiscalYearRef): SourceFiscalYear => ({ year: fy.year, fromDate: fy.fromDate ?? null, toDate: fy.toDate ?? null, inDefaultSelection: defaultFiscalYears.has(fy.year), }) const sourceYears = allYears.map(describe) const omittedYears = allYears.filter((fy) => !selectedFiscalYears.has(fy.year)).map(describe) const files: ProviderSieFile[] = [] const failedYears: { year: number; error: string }[] = [] for (const fy of toFetch) { try { const rawContent = await fetcher.fetchSie(accessToken, fy) if (rawContent) { files.push({ fiscalYear: fy.year, rawContent }) } else { failedYears.push({ year: fy.year, error: 'Provider returned an empty SIE export' }) } } catch (err) { const reason = err instanceof Error ? err.message : String(err) log.warn(`Failed to fetch SIE for ${provider} fiscal year ${fy.year} (id ${fy.id})`, { reason, }) failedYears.push({ year: fy.year, error: reason }) } } return { files, availableYears, sourceYears, failedYears, omittedYears } } interface SieFetcher { listYears(accessToken: string): Promise fetchSie(accessToken: string, fy: FiscalYearRef): Promise } function getSieFetcher( provider: ProviderName, providerCompanyId: string | undefined, selectedYears?: number[], ): SieFetcher | null { if (provider === 'fortnox') { return { async listYears(accessToken) { const fyResponse = await fortnoxClient.get>( accessToken, '/financialyears', ) const years = (fyResponse['FinancialYears'] as Record[] | undefined) ?? [] return years.map((fy) => ({ id: fy['Id'] as number, year: new Date(fy['FromDate'] as string).getFullYear(), fromDate: typeof fy['FromDate'] === 'string' ? fy['FromDate'] : undefined, toDate: typeof fy['ToDate'] === 'string' ? fy['ToDate'] : undefined, })) }, async fetchSie(accessToken, fy) { // Fortnox normally serves the SIE body as UTF-8, but endpoint variants // have been seen answering CP437 (the SIE spec encoding): a blind // response.text() would turn å/ä/ö into U+FFFD irrecoverably. Fetch // raw bytes and detect-decode like the Briox/BL paths. const buffer = await fortnoxClient.getBytes(accessToken, `/sie/4?financialyear=${fy.id}`) return decodeBuffer(buffer, detectEncoding(buffer)) }, } } if (provider === 'briox') { return { async listYears(accessToken) { const years = await brioxClient.listFinancialYears(accessToken) return years.map((fy) => ({ id: fy.id, year: new Date(fy.fromdate).getFullYear(), fromDate: fy.fromdate, toDate: fy.todate, })) }, async fetchSie(accessToken, fy) { // Briox serves SIE as an octet-stream whose encoding varies // (CP437/Windows-1252/UTF-8): fetch bytes and detect-decode. const buffer = await brioxClient.getBytes(accessToken, `/sie/${fy.id}/4`) return decodeBuffer(buffer, detectEncoding(buffer)) }, } } if (provider === 'wint') { // The SIE files are rendered from voucher data, and opening balances for // years before WINT's current fiscal year are derived by walking the // voucher deltas backward from the /api/Account Ib anchor. That walk // needs every year's vouchers at once, so the first fetchSie call builds // a shared context (this fetcher object lives for exactly one // fetchProviderSieFiles invocation: the closure is the right lifetime). interface WintSieContext { companyName: string orgNumber?: string accounts: ReturnType[] /** Every year the source has, unfiltered: what listYears reports. */ allYears: (WintSieYear & { id: number })[] /** The allowed subset: the years that render as SIE. */ years: (WintSieYear & { id: number })[] vouchersByYear: Map ibByYear: Map> fetchErrors: Map } let contextPromise: Promise | null = null const loadContext = (accessToken: string): Promise => { contextPromise ??= (async () => { const company = await wintClient.get>(accessToken, '/api/Auth') const rawYears = (company['FinancialYears'] as Record[] | undefined) ?? [] // The same selection fetchProviderSieFiles applies: the explicit // years when given, else the default window. const allowed = selectedYears ? new Set(selectedYears) : getAllowedFiscalYears() const allYears = rawYears .map((fy) => ({ id: Number(fy['Id']), year: new Date((fy['Start'] as string) ?? '').getFullYear(), start: ((fy['Start'] as string) ?? '').slice(0, 10), end: ((fy['End'] as string) ?? '').slice(0, 10), })) .sort((a, b) => a.year - b.year) const years = allYears.filter((fy) => allowed.has(fy.year)) const accountsRaw = await wintClient.getPaginated>( accessToken, '/api/Account', ) const accounts = accountsRaw.map(mapWintAccountForSie).filter((a) => a.accountNumber) // The Ib anchor is WINT's ACTIVE fiscal year: selected from the // UNFILTERED year list, so an active year outside the allowed import // window can never be silently swapped for the latest allowed year // (that would attach the anchor balances to the wrong year). When the // anchor lies outside the window, its vouchers are still fetched below // so the derivation chain stays complete; only allowed years render. const today = new Date().toISOString().slice(0, 10) const anchor = allYears.find((fy) => fy.start <= today && today <= fy.end) ?? allYears[allYears.length - 1] const chainYears = new Map(years.map((fy) => [fy.year, fy])) if (anchor) { const lo = Math.min(anchor.year, ...years.map((fy) => fy.year)) const hi = Math.max(anchor.year, ...years.map((fy) => fy.year)) for (const fy of allYears) { if (fy.year >= lo && fy.year <= hi) chainYears.set(fy.year, fy) } } // A single year's fetch failure must not sink the whole migration: // the year is simply absent from vouchersByYear, deriveIbByYear stops // at the hole, and the affected years fail loudly in fetchSie while // the years on the anchor's side of the hole still render. const vouchersByYear = new Map() const fetchErrors = new Map() for (const fy of [...chainYears.values()].sort((a, b) => a.year - b.year)) { try { const raw = await wintClient.getPaginated>( accessToken, `/api/Voucher?BookingDateStart=${fy.start}&BookingDateEnd=${fy.end}&IncludeTransactions=true`, ) vouchersByYear.set(fy.year, raw.map(mapWintVoucherForSie)) } catch (err) { const reason = err instanceof Error ? err.message : String(err) log.warn(`WINT voucher fetch failed for fiscal year ${fy.year}`, { reason }) fetchErrors.set(fy.year, reason) } } const anchorIb = new Map() for (const account of accounts) { if (account.ib != null && account.ib !== 0) anchorIb.set(account.accountNumber, account.ib) } const ibByYear = anchor ? deriveIbByYear(anchor.year, anchorIb, vouchersByYear, years.map((fy) => fy.year)) : new Map>() return { companyName: (company['Name'] as string) ?? 'Okänt företag', orgNumber: (company['Org'] as string | undefined) || undefined, accounts, allYears, years, vouchersByYear, ibByYear, fetchErrors, } })() return contextPromise } return { async listYears(accessToken) { // The unfiltered list: fetchProviderSieFiles applies the window // itself and needs the years outside it to name what is left out. const context = await loadContext(accessToken) return context.allYears.map((fy) => ({ id: fy.id, year: fy.year, fromDate: fy.start, toDate: fy.end, })) }, async fetchSie(accessToken, fy) { const context = await loadContext(accessToken) const yearRef = context.years.find((y) => y.year === fy.year) const vouchers = context.vouchersByYear.get(fy.year) const ibByAccount = context.ibByYear.get(fy.year) if (!yearRef || !vouchers) { const reason = context.fetchErrors.get(fy.year) throw new Error( reason ? `WINT voucher fetch failed for fiscal year ${fy.year}: ${reason}` : `WINT returned no ledger data for fiscal year ${fy.year}`, ) } if (!ibByAccount) { // A hole in the voucher chain between this year and the Ib anchor // year: opening balances cannot be established. Failing the year is // better than importing broken IB/UB continuity. throw new Error( `Opening balances for ${fy.year} could not be derived from WINT's ledger data`, ) } const previousYear = context.years.find((y) => y.year === fy.year - 1) return buildWintSieFile({ companyName: context.companyName, orgNumber: context.orgNumber, programVersion: '1.0', generatedDate: new Date().toISOString().slice(0, 10), year: yearRef, previousYear, accounts: context.accounts, vouchers, ibByAccount, }) }, } } if (provider === 'bjornlunden') { // providerCompanyId carries the per-company User-Key header value. const userKey = providerCompanyId if (!userKey) { throw new Error('Björn Lundén requires a company User-Key: reconnect the provider') } return { async listYears(accessToken) { const years = await bjornLundenClient.listFinancialYears(accessToken, userKey) return years.map((fy) => ({ id: fy.id ?? fy.entityId, year: new Date(fy.fromDate).getFullYear(), fromDate: fy.fromDate, toDate: fy.toDate, })) }, async fetchSie(accessToken, fy) { // BL's export is date-ranged rather than year-id based. Sandbox- // verified: the body is RAW SIE bytes (CP437, Content-Type // text/vnd.sie-gruppen.si) even though the swagger declares a base64 // string: decodeSieBytes handles both shapes. const buffer = await bjornLundenClient.getBytes( accessToken, userKey, `/sie/export/${fy.fromDate}/${fy.toDate}`, ) return decodeSieBytes(buffer) }, } } return null } /** * Decode a SIE payload that may arrive either as raw SIE bytes or as a * base64 string (optionally JSON-quoted). BL's swagger declares base64 but * the live API sends raw CP437: handle both so a future API change doesn't * silently break the import. */ function decodeSieBytes(buffer: ArrayBuffer): string { const direct = decodeBuffer(buffer, detectEncoding(buffer)) if (looksLikeSie(direct)) return direct const candidate = direct.trim().replace(/^"|"$/g, '') if (/^[A-Za-z0-9+/=\s]+$/.test(candidate)) { try { const bytes = Buffer.from(candidate, 'base64') const ab = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer const decoded = decodeBuffer(ab, detectEncoding(ab)) if (looksLikeSie(decoded)) return decoded } catch { // fall through to returning the direct decode } } // Neither shape matched: return the direct decode and let the SIE parser // produce its own diagnostics instead of failing silently here. return direct } /** SIE files start with a #-record (#FLAGGA per spec; be lenient about order). */ function looksLikeSie(text: string): boolean { return text.trimStart().startsWith('#') }