accounted/extensions/general/arcim-migration/lib/sie-fetcher.ts

/**
 * Per-provider SIE-over-API fetcher.
 *
 * Providers that expose their general ledger as a SIE export over the API get
 * the "Fortnox-grade" migration experience: the wizard pulls the GL itself
 * instead of requiring a manual SIE upload. Everything downstream (parsing,
 * validation, account mapping, replace-mode import) is provider-agnostic:
 * this module's only job is to produce raw SIE file contents per fiscal year.
 *
 * Shared by /preview (latest year, for stats) and /sie-data (all years).
 */

import { FortnoxClient } from '@/lib/providers/fortnox/client'
import { BrioxClient } from '@/lib/providers/briox/client'
import { BjornLundenClient } from '@/lib/providers/bjornlunden/client'
import { WintClient } from '@/lib/providers/wint/client'
import {
  buildWintSieFile,
  deriveIbByYear,
  mapWintAccountForSie,
  mapWintVoucherForSie,
  type WintSieVoucher,
  type WintSieYear,
} from '@/lib/providers/wint/sie-builder'
import type { ProviderName } from '@/lib/providers/types'
import { detectEncoding, decodeBuffer } from '@/lib/import/sie-parser'
import { createLogger } from '@/lib/logger'

const log = createLogger('extensions/arcim-migration/sie-fetcher')

/**
 * Fiscal years we support importing: the current year and the two before it.
 * Derived at call time (not a module constant) so the window rolls forward
 * automatically at new year without a code change.
 */
export function getAllowedFiscalYears(now: Date = new Date()): Set<number> {
  const currentYear = now.getFullYear()
  return new Set([currentYear - 2, currentYear - 1, currentYear])
}

export interface ProviderSieFile {
  fiscalYear: number
  rawContent: string
}

export interface ProviderSieFetchResult {
  files: ProviderSieFile[]
  /**
   * Every fiscal year available at the provider within the allowed window:
   * also populated when latestOnly fetched just one file, so /preview can show
   * the full year list without a second round-trip.
   */
  availableYears: number[]
  /**
   * Allowed years whose export failed (or came back empty). Callers MUST
   * surface these to the user: silently importing e.g. 2024+2026 without 2025
   * breaks IB/UB continuity between the years without anyone noticing.
   */
  failedYears: { year: number; error: string }[]
}

// Singleton clients (they hold rate limiters)
const fortnoxClient = new FortnoxClient()
const brioxClient = new BrioxClient()
const bjornLundenClient = new BjornLundenClient()
const wintClient = new WintClient()

/**
 * True when the provider's API can serve the GL as SIE (no manual upload).
 * WINT qualifies even though its v1 API has no SIE endpoint: the ledger is
 * fetched voucher-by-voucher and RENDERED as SIE on our side (Tier A; see
 * lib/providers/wint/sie-builder.ts). Downstream the file is indistinguishable
 * from a provider export and goes through the same parse/validate/import path.
 */
export function providerSupportsSie(provider: ProviderName): boolean {
  return provider === 'fortnox' || provider === 'briox' || provider === 'bjornlunden' || provider === 'wint'
}

interface FiscalYearRef {
  id: string | number
  year: number
  /** Period bounds: required by BL, whose export URL is date-ranged. */
  fromDate?: string
  toDate?: string
}

/**
 * Fetch SIE type-4 exports from the provider, one file per allowed fiscal
 * year (oldest first). Years whose export fails do not block the rest of the
 * migration, but they are reported in `failedYears` so the caller can warn
 * the user before importing a gap (IB/UB continuity).
 */
export async function fetchProviderSieFiles(
  provider: ProviderName,
  accessToken: string,
  providerCompanyId: string | undefined,
  opts?: { latestOnly?: boolean },
): Promise<ProviderSieFetchResult> {
  const fetcher = getSieFetcher(provider, providerCompanyId)
  if (!fetcher) {
    throw new Error(`Provider ${provider} does not support SIE over API`)
  }

  const allowedFiscalYears = getAllowedFiscalYears()
  const allYears = await fetcher.listYears(accessToken)
  const allowedYears = allYears
    .filter((fy) => allowedFiscalYears.has(fy.year))
    .sort((a, b) => a.year - b.year)

  const availableYears = allowedYears.map((fy) => fy.year)
  const toFetch = opts?.latestOnly ? allowedYears.slice(-1) : allowedYears

  const files: ProviderSieFile[] = []
  const failedYears: { year: number; error: string }[] = []
  for (const fy of toFetch) {
    try {
      const rawContent = await fetcher.fetchSie(accessToken, fy)
      if (rawContent) {
        files.push({ fiscalYear: fy.year, rawContent })
      } else {
        failedYears.push({ year: fy.year, error: 'Provider returned an empty SIE export' })
      }
    } catch (err) {
      const reason = err instanceof Error ? err.message : String(err)
      log.warn(`Failed to fetch SIE for ${provider} fiscal year ${fy.year} (id ${fy.id})`, {
        reason,
      })
      failedYears.push({ year: fy.year, error: reason })
    }
  }

  return { files, availableYears, failedYears }
}

interface SieFetcher {
  listYears(accessToken: string): Promise<FiscalYearRef[]>
  fetchSie(accessToken: string, fy: FiscalYearRef): Promise<string>
}

function getSieFetcher(
  provider: ProviderName,
  providerCompanyId: string | undefined,
): SieFetcher | null {
  if (provider === 'fortnox') {
    return {
      async listYears(accessToken) {
        const fyResponse = await fortnoxClient.get<Record<string, unknown>>(
          accessToken,
          '/financialyears',
        )
        const years = (fyResponse['FinancialYears'] as Record<string, unknown>[] | undefined) ?? []
        return years.map((fy) => ({
          id: fy['Id'] as number,
          year: new Date(fy['FromDate'] as string).getFullYear(),
        }))
      },
      async fetchSie(accessToken, fy) {
        // Fortnox normally serves the SIE body as UTF-8, but endpoint variants
        // have been seen answering CP437 (the SIE spec encoding): a blind
        // response.text() would turn å/ä/ö into U+FFFD irrecoverably. Fetch
        // raw bytes and detect-decode like the Briox/BL paths.
        const buffer = await fortnoxClient.getBytes(accessToken, `/sie/4?financialyear=${fy.id}`)
        return decodeBuffer(buffer, detectEncoding(buffer))
      },
    }
  }

  if (provider === 'briox') {
    return {
      async listYears(accessToken) {
        const years = await brioxClient.listFinancialYears(accessToken)
        return years.map((fy) => ({
          id: fy.id,
          year: new Date(fy.fromdate).getFullYear(),
        }))
      },
      async fetchSie(accessToken, fy) {
        // Briox serves SIE as an octet-stream whose encoding varies
        // (CP437/Windows-1252/UTF-8): fetch bytes and detect-decode.
        const buffer = await brioxClient.getBytes(accessToken, `/sie/${fy.id}/4`)
        return decodeBuffer(buffer, detectEncoding(buffer))
      },
    }
  }

  if (provider === 'wint') {
    // The SIE files are rendered from voucher data, and opening balances for
    // years before WINT's current fiscal year are derived by walking the
    // voucher deltas backward from the /api/Account Ib anchor. That walk
    // needs every year's vouchers at once, so the first fetchSie call builds
    // a shared context (this fetcher object lives for exactly one
    // fetchProviderSieFiles invocation: the closure is the right lifetime).
    interface WintSieContext {
      companyName: string
      orgNumber?: string
      accounts: ReturnType<typeof mapWintAccountForSie>[]
      years: (WintSieYear & { id: number })[]
      vouchersByYear: Map<number, WintSieVoucher[]>
      ibByYear: Map<number, Map<string, number>>
      fetchErrors: Map<number, string>
    }
    let contextPromise: Promise<WintSieContext> | null = null

    const loadContext = (accessToken: string): Promise<WintSieContext> => {
      contextPromise ??= (async () => {
        const company = await wintClient.get<Record<string, unknown>>(accessToken, '/api/Auth')
        const rawYears = (company['FinancialYears'] as Record<string, unknown>[] | undefined) ?? []
        const allowed = getAllowedFiscalYears()
        const allYears = rawYears
          .map((fy) => ({
            id: Number(fy['Id']),
            year: new Date((fy['Start'] as string) ?? '').getFullYear(),
            start: ((fy['Start'] as string) ?? '').slice(0, 10),
            end: ((fy['End'] as string) ?? '').slice(0, 10),
          }))
          .sort((a, b) => a.year - b.year)
        const years = allYears.filter((fy) => allowed.has(fy.year))

        const accountsRaw = await wintClient.getPaginated<Record<string, unknown>>(
          accessToken,
          '/api/Account',
        )
        const accounts = accountsRaw.map(mapWintAccountForSie).filter((a) => a.accountNumber)

        // The Ib anchor is WINT's ACTIVE fiscal year: selected from the
        // UNFILTERED year list, so an active year outside the allowed import
        // window can never be silently swapped for the latest allowed year
        // (that would attach the anchor balances to the wrong year). When the
        // anchor lies outside the window, its vouchers are still fetched below
        // so the derivation chain stays complete; only allowed years render.
        const today = new Date().toISOString().slice(0, 10)
        const anchor =
          allYears.find((fy) => fy.start <= today && today <= fy.end) ?? allYears[allYears.length - 1]

        const chainYears = new Map(years.map((fy) => [fy.year, fy]))
        if (anchor) {
          const lo = Math.min(anchor.year, ...years.map((fy) => fy.year))
          const hi = Math.max(anchor.year, ...years.map((fy) => fy.year))
          for (const fy of allYears) {
            if (fy.year >= lo && fy.year <= hi) chainYears.set(fy.year, fy)
          }
        }

        // A single year's fetch failure must not sink the whole migration:
        // the year is simply absent from vouchersByYear, deriveIbByYear stops
        // at the hole, and the affected years fail loudly in fetchSie while
        // the years on the anchor's side of the hole still render.
        const vouchersByYear = new Map<number, WintSieVoucher[]>()
        const fetchErrors = new Map<number, string>()
        for (const fy of [...chainYears.values()].sort((a, b) => a.year - b.year)) {
          try {
            const raw = await wintClient.getPaginated<Record<string, unknown>>(
              accessToken,
              `/api/Voucher?BookingDateStart=${fy.start}&BookingDateEnd=${fy.end}&IncludeTransactions=true`,
            )
            vouchersByYear.set(fy.year, raw.map(mapWintVoucherForSie))
          } catch (err) {
            const reason = err instanceof Error ? err.message : String(err)
            log.warn(`WINT voucher fetch failed for fiscal year ${fy.year}`, { reason })
            fetchErrors.set(fy.year, reason)
          }
        }

        const anchorIb = new Map<string, number>()
        for (const account of accounts) {
          if (account.ib != null && account.ib !== 0) anchorIb.set(account.accountNumber, account.ib)
        }
        const ibByYear = anchor
          ? deriveIbByYear(anchor.year, anchorIb, vouchersByYear, years.map((fy) => fy.year))
          : new Map<number, Map<string, number>>()

        return {
          companyName: (company['Name'] as string) ?? 'Okänt företag',
          orgNumber: (company['Org'] as string | undefined) || undefined,
          accounts,
          years,
          vouchersByYear,
          ibByYear,
          fetchErrors,
        }
      })()
      return contextPromise
    }

    return {
      async listYears(accessToken) {
        const context = await loadContext(accessToken)
        return context.years.map((fy) => ({
          id: fy.id,
          year: fy.year,
          fromDate: fy.start,
          toDate: fy.end,
        }))
      },
      async fetchSie(accessToken, fy) {
        const context = await loadContext(accessToken)
        const yearRef = context.years.find((y) => y.year === fy.year)
        const vouchers = context.vouchersByYear.get(fy.year)
        const ibByAccount = context.ibByYear.get(fy.year)
        if (!yearRef || !vouchers) {
          const reason = context.fetchErrors.get(fy.year)
          throw new Error(
            reason
              ? `WINT voucher fetch failed for fiscal year ${fy.year}: ${reason}`
              : `WINT returned no ledger data for fiscal year ${fy.year}`,
          )
        }
        if (!ibByAccount) {
          // A hole in the voucher chain between this year and the Ib anchor
          // year: opening balances cannot be established. Failing the year is
          // better than importing broken IB/UB continuity.
          throw new Error(
            `Opening balances for ${fy.year} could not be derived from WINT's ledger data`,
          )
        }
        const previousYear = context.years.find((y) => y.year === fy.year - 1)
        return buildWintSieFile({
          companyName: context.companyName,
          orgNumber: context.orgNumber,
          programVersion: '1.0',
          generatedDate: new Date().toISOString().slice(0, 10),
          year: yearRef,
          previousYear,
          accounts: context.accounts,
          vouchers,
          ibByAccount,
        })
      },
    }
  }

  if (provider === 'bjornlunden') {
    // providerCompanyId carries the per-company User-Key header value.
    const userKey = providerCompanyId
    if (!userKey) {
      throw new Error('Björn Lundén requires a company User-Key: reconnect the provider')
    }
    return {
      async listYears(accessToken) {
        const years = await bjornLundenClient.listFinancialYears(accessToken, userKey)
        return years.map((fy) => ({
          id: fy.id ?? fy.entityId,
          year: new Date(fy.fromDate).getFullYear(),
          fromDate: fy.fromDate,
          toDate: fy.toDate,
        }))
      },
      async fetchSie(accessToken, fy) {
        // BL's export is date-ranged rather than year-id based. Sandbox-
        // verified: the body is RAW SIE bytes (CP437, Content-Type
        // text/vnd.sie-gruppen.si) even though the swagger declares a base64
        // string: decodeSieBytes handles both shapes.
        const buffer = await bjornLundenClient.getBytes(
          accessToken,
          userKey,
          `/sie/export/${fy.fromDate}/${fy.toDate}`,
        )
        return decodeSieBytes(buffer)
      },
    }
  }

  return null
}

/**
 * Decode a SIE payload that may arrive either as raw SIE bytes or as a
 * base64 string (optionally JSON-quoted). BL's swagger declares base64 but
 * the live API sends raw CP437: handle both so a future API change doesn't
 * silently break the import.
 */
function decodeSieBytes(buffer: ArrayBuffer): string {
  const direct = decodeBuffer(buffer, detectEncoding(buffer))
  if (looksLikeSie(direct)) return direct

  const candidate = direct.trim().replace(/^"|"$/g, '')
  if (/^[A-Za-z0-9+/=\s]+$/.test(candidate)) {
    try {
      const bytes = Buffer.from(candidate, 'base64')
      const ab = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer
      const decoded = decodeBuffer(ab, detectEncoding(ab))
      if (looksLikeSie(decoded)) return decoded
    } catch {
      // fall through to returning the direct decode
    }
  }

  // Neither shape matched: return the direct decode and let the SIE parser
  // produce its own diagnostics instead of failing silently here.
  return direct
}

/** SIE files start with a #-record (#FLAGGA per spec; be lenient about order). */
function looksLikeSie(text: string): boolean {
  return text.trimStart().startsWith('#')
}