fix(import): migration preview and theater read all fiscal years, not just the newest (#1614)
The /sie-data route parsed only the newest fiscal year's SIE file for the import preview and the returned SIEData.parsed. Mid-year provider exports have few or zero vouchers in the newest year, so the first real Fortnox migration (3 fiscal years, 4153 vouchers) previewed "0 verifikationer" and drew an almost-empty migration theater while the import itself landed all 4153 vouchers from the older files. - New mergeParsedSIEFiles (lib/import/sie-merge.ts): pure, browser-clean whole-dataset merge (accounts union first-wins, vouchers concatenated, fiscal years union oldest-first re-indexed newest=0, balances and issues concatenated, dimensions deduped), with unit tests. - /sie-data parses each file exactly once, builds the preview from the merged parse and returns parsed: merged; response shape unchanged. Validation stays newest-file-only so no previously accepted dataset is newly rejected. - /preview drops latestOnly and computes sieStats from the merged parse: the connect step's "Hittade X konton och Y verifikationer" line renders from THESE stats, so this is where the founder-visible count was lying. - The migration theater spreads its account waves across ~10s and births an additional wave on each real step label during the SIE phase (progress <= 55), through a shared rate-limited gate, so the canvas keeps performing over a multi-minute run. Narration labels and progress remain the wizard's real values; reduced motion unchanged. Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Jakob Wennberg
Claude Fable 5
Jakob Wennberg
Claude Fable 5
parent
ea85f4c084
commit
b556475b01
@@ -0,0 +1,175 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { parseSIEFile } from '../sie-parser'
|
||||
import { mergeParsedSIEFiles } from '../sie-merge'
|
||||
import { generateImportPreview } from '../sie-import'
|
||||
import { buildTheaterModel } from '../theater-model'
|
||||
import type { AccountMapping } from '../types'
|
||||
|
||||
/**
|
||||
* One provider-style SIE 4 export per fiscal year, like the files
|
||||
* fetchProviderSieFiles returns (oldest first).
|
||||
*/
|
||||
function sieYear(opts: {
|
||||
year: number
|
||||
companyName?: string
|
||||
konto?: string[]
|
||||
vouchers?: string[][]
|
||||
extra?: string[]
|
||||
}): string {
|
||||
const { year, companyName = 'One Punkt Com AB', konto = [], vouchers = [], extra = [] } = opts
|
||||
return [
|
||||
'#FLAGGA 0',
|
||||
'#SIETYP 4',
|
||||
`#FNAMN "${companyName}"`,
|
||||
'#ORGNR 5566778899',
|
||||
`#RAR 0 ${year}0101 ${year}1231`,
|
||||
`#RAR -1 ${year - 1}0101 ${year - 1}1231`,
|
||||
'#KONTO 1930 "Företagskonto"',
|
||||
'#KONTO 2641 "Ingående moms"',
|
||||
...konto,
|
||||
...extra,
|
||||
...vouchers.flat(),
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
function voucher(num: number, date: string, description: string, account: string, amount = 100): string[] {
|
||||
return [
|
||||
`#VER A ${num} ${date} "${description}"`,
|
||||
'{',
|
||||
`#TRANS ${account} {} ${amount}.00`,
|
||||
`#TRANS 1930 {} -${amount}.00`,
|
||||
'}',
|
||||
]
|
||||
}
|
||||
|
||||
describe('mergeParsedSIEFiles', () => {
|
||||
it('throws on an empty list and passes a single file through unchanged', () => {
|
||||
expect(() => mergeParsedSIEFiles([])).toThrow()
|
||||
const parsed = parseSIEFile(sieYear({ year: 2026 }))
|
||||
expect(mergeParsedSIEFiles([parsed])).toBe(parsed)
|
||||
})
|
||||
|
||||
it('merges distinct years: vouchers concatenated, fiscal years union oldest first', () => {
|
||||
const y2024 = parseSIEFile(
|
||||
sieYear({
|
||||
year: 2024,
|
||||
konto: ['#KONTO 4010 "Material"'],
|
||||
vouchers: [voucher(1, '20240315', 'Bolagsverket', '4010'), voucher(2, '20240601', 'Loopia', '4010')],
|
||||
}),
|
||||
)
|
||||
const y2025 = parseSIEFile(
|
||||
sieYear({
|
||||
year: 2025,
|
||||
konto: ['#KONTO 3041 "Försäljning tjänster"'],
|
||||
vouchers: [voucher(1, '20250210', 'Kund AB', '3041', 250)],
|
||||
}),
|
||||
)
|
||||
const y2026 = parseSIEFile(sieYear({ year: 2026 }))
|
||||
|
||||
const merged = mergeParsedSIEFiles([y2024, y2025, y2026])
|
||||
|
||||
expect(merged.vouchers).toHaveLength(3)
|
||||
expect(merged.stats.totalVouchers).toBe(3)
|
||||
expect(merged.stats.totalTransactionLines).toBe(6)
|
||||
// Union of #RAR periods: 2023 (from y2024's #RAR -1) through 2026,
|
||||
// oldest first, newest re-indexed to yearIndex 0.
|
||||
expect(merged.header.fiscalYears.map((fy) => fy.start)).toEqual([
|
||||
'2023-01-01',
|
||||
'2024-01-01',
|
||||
'2025-01-01',
|
||||
'2026-01-01',
|
||||
])
|
||||
expect(merged.header.fiscalYears.map((fy) => fy.yearIndex)).toEqual([-3, -2, -1, 0])
|
||||
expect(merged.stats.fiscalYearStart).toBe('2026-01-01')
|
||||
expect(merged.stats.fiscalYearEnd).toBe('2026-12-31')
|
||||
// Accounts are a union across files.
|
||||
expect(merged.accounts.map((a) => a.number).sort()).toEqual(['1930', '2641', '3041', '4010'])
|
||||
expect(merged.stats.totalAccounts).toBe(4)
|
||||
expect(merged.header.companyName).toBe('One Punkt Com AB')
|
||||
expect(merged.header.orgNumber).toBe('5566778899')
|
||||
})
|
||||
|
||||
it('dedupes duplicate account numbers: first name wins, later files fill missing metadata', () => {
|
||||
const older = parseSIEFile(
|
||||
sieYear({ year: 2024, konto: ['#KONTO 4010 "Material och varor"'] }),
|
||||
)
|
||||
const newer = parseSIEFile(
|
||||
sieYear({
|
||||
year: 2025,
|
||||
konto: ['#KONTO 4010 "Inköp material"'],
|
||||
extra: ['#SRU 4010 7512'],
|
||||
}),
|
||||
)
|
||||
|
||||
const merged = mergeParsedSIEFiles([older, newer])
|
||||
|
||||
const acc = merged.accounts.filter((a) => a.number === '4010')
|
||||
expect(acc).toHaveLength(1)
|
||||
expect(acc[0].name).toBe('Material och varor')
|
||||
// The older file had no #SRU for 4010: the newer file's code fills it in.
|
||||
expect(acc[0].sruCode).toBe('7512')
|
||||
})
|
||||
|
||||
it('an empty newest file plus a rich older file yields a rich merge', () => {
|
||||
const rich = parseSIEFile(
|
||||
sieYear({
|
||||
year: 2025,
|
||||
konto: ['#KONTO 4010 "Material"', '#KONTO 3041 "Försäljning tjänster"'],
|
||||
vouchers: [
|
||||
voucher(1, '20250110', 'Loopia', '4010'),
|
||||
voucher(2, '20250211', 'Loopia', '4010'),
|
||||
voucher(3, '20250320', 'Kund AB', '3041', 900),
|
||||
],
|
||||
extra: ['#IB 0 1930 5000.00', '#IB 0 2081 -5000.00'],
|
||||
}),
|
||||
)
|
||||
// Mid-year export: the newest fiscal year has accounts but zero vouchers.
|
||||
const empty = parseSIEFile(sieYear({ year: 2026 }))
|
||||
|
||||
const merged = mergeParsedSIEFiles([rich, empty])
|
||||
|
||||
expect(merged.stats.totalVouchers).toBe(3)
|
||||
expect(merged.openingBalances).toHaveLength(2)
|
||||
|
||||
// The preview built from the merge reports the whole dataset, not the
|
||||
// empty newest file (the "Hittade 740 konton och 0 verifikationer" bug).
|
||||
const mappings: AccountMapping[] = []
|
||||
const preview = generateImportPreview(merged, mappings)
|
||||
expect(preview.voucherCount).toBe(3)
|
||||
// 1930, 2641, 4010, 3041 from #KONTO plus 2081 auto-registered from #IB.
|
||||
expect(preview.accountCount).toBe(5)
|
||||
|
||||
// The theater model gets the full history: both years, real accounts,
|
||||
// a counterparty seen twice.
|
||||
const model = buildTheaterModel(merged)
|
||||
expect(model.totalVouchers).toBe(3)
|
||||
expect(model.years.length).toBeGreaterThanOrEqual(2)
|
||||
expect(model.accounts.length).toBeGreaterThan(0)
|
||||
expect(model.counterparties.map((c) => c.name)).toContain('Loopia')
|
||||
})
|
||||
|
||||
it('keeps the most common non-empty company name and concatenates issues', () => {
|
||||
const a = parseSIEFile(sieYear({ year: 2024, companyName: 'Gamla Namnet AB' }))
|
||||
const b = parseSIEFile(sieYear({ year: 2025, companyName: 'One Punkt Com AB' }))
|
||||
const c = parseSIEFile(sieYear({ year: 2026, companyName: 'One Punkt Com AB' }))
|
||||
|
||||
const merged = mergeParsedSIEFiles([a, b, c])
|
||||
expect(merged.header.companyName).toBe('One Punkt Com AB')
|
||||
expect(merged.issues).toEqual([...a.issues, ...b.issues, ...c.issues])
|
||||
})
|
||||
|
||||
it('dedupes dimension registrations by number and (number, code)', () => {
|
||||
const dims = [
|
||||
'#DIM 6 "Projekt"',
|
||||
'#OBJEKT 6 "P1" "Bryggan"',
|
||||
]
|
||||
const a = parseSIEFile(sieYear({ year: 2024, extra: dims }))
|
||||
const b = parseSIEFile(
|
||||
sieYear({ year: 2025, extra: [...dims, '#OBJEKT 6 "P2" "Kajen"'] }),
|
||||
)
|
||||
|
||||
const merged = mergeParsedSIEFiles([a, b])
|
||||
expect(merged.dimensions).toHaveLength(1)
|
||||
expect(merged.dimensionValues.map((v) => v.code).sort()).toEqual(['P1', 'P2'])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Merge several parsed SIE files (one per fiscal year) into one
|
||||
* whole-dataset view.
|
||||
*
|
||||
* Mid-year provider exports have few or zero vouchers in the newest fiscal
|
||||
* year, so any preview or visualization built from a single file lies about
|
||||
* the dataset. The merged parse is for whole-dataset PRESENTATION (account
|
||||
* mapping, import preview counts, the migration theater model): the actual
|
||||
* import still runs per raw file.
|
||||
*
|
||||
* Semantics:
|
||||
* - accounts: union by number, first occurrence wins the name; missing
|
||||
* sruCode/accountType are filled in from later duplicates.
|
||||
* - vouchers: concatenated in file order (callers pass files oldest first).
|
||||
* - opening/closing/result balances: concatenated AS-IS. Each record keeps
|
||||
* the yearIndex relative to its SOURCE file's current year, so per-year
|
||||
* balance math on the merged output is not meaningful: two files can both
|
||||
* carry yearIndex 0 records for different calendar years.
|
||||
* - header.fiscalYears: union deduped by start+end, sorted oldest first,
|
||||
* re-indexed so the newest year is 0 (stats then reports the newest
|
||||
* year's bounds, matching the single-file "current year" convention).
|
||||
* - company name/orgnr: the most common non-empty value, first seen wins
|
||||
* ties.
|
||||
* - dimensions: deduped by dimension number; dimension values deduped by
|
||||
* (dimension number, code); first occurrence wins.
|
||||
* - issues: concatenated.
|
||||
*
|
||||
* Pure and browser-clean (no Node APIs): the client calls it too.
|
||||
*/
|
||||
|
||||
import type {
|
||||
FiscalYearInfo,
|
||||
ParsedSIEFile,
|
||||
SIEAccount,
|
||||
SIEDimension,
|
||||
SIEDimensionValue,
|
||||
} from './types'
|
||||
|
||||
/** Most common non-empty value; earlier values win ties. */
|
||||
function mostCommon(values: (string | null)[]): string | null {
|
||||
const counts = new Map<string, number>()
|
||||
let best: string | null = null
|
||||
let bestCount = 0
|
||||
for (const value of values) {
|
||||
if (!value) continue
|
||||
const count = (counts.get(value) ?? 0) + 1
|
||||
counts.set(value, count)
|
||||
if (count > bestCount) {
|
||||
best = value
|
||||
bestCount = count
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
function firstNonNull<T>(values: (T | null | undefined)[]): T | null {
|
||||
for (const value of values) {
|
||||
if (value !== null && value !== undefined) return value
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function mergeParsedSIEFiles(files: ParsedSIEFile[]): ParsedSIEFile {
|
||||
if (files.length === 0) {
|
||||
throw new Error('mergeParsedSIEFiles requires at least one parsed file')
|
||||
}
|
||||
if (files.length === 1) return files[0]
|
||||
|
||||
const headers = files.map((f) => f.header)
|
||||
|
||||
// Fiscal years: union by period, oldest first, newest re-indexed to 0.
|
||||
const yearsByPeriod = new Map<string, FiscalYearInfo>()
|
||||
for (const header of headers) {
|
||||
for (const fy of header.fiscalYears) {
|
||||
const key = `${fy.start}|${fy.end}`
|
||||
if (!yearsByPeriod.has(key)) yearsByPeriod.set(key, fy)
|
||||
}
|
||||
}
|
||||
const sortedYears = [...yearsByPeriod.values()].sort((a, b) =>
|
||||
a.start.localeCompare(b.start),
|
||||
)
|
||||
const fiscalYears = sortedYears.map((fy, i) => ({
|
||||
yearIndex: i - (sortedYears.length - 1),
|
||||
start: fy.start,
|
||||
end: fy.end,
|
||||
}))
|
||||
|
||||
// Accounts: union by number, first name wins, missing metadata filled in.
|
||||
const accountsByNumber = new Map<string, SIEAccount>()
|
||||
for (const file of files) {
|
||||
for (const account of file.accounts) {
|
||||
const existing = accountsByNumber.get(account.number)
|
||||
if (!existing) {
|
||||
accountsByNumber.set(account.number, { ...account })
|
||||
} else {
|
||||
if (!existing.sruCode && account.sruCode) existing.sruCode = account.sruCode
|
||||
if (!existing.accountType && account.accountType) {
|
||||
existing.accountType = account.accountType
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const accounts = [...accountsByNumber.values()]
|
||||
|
||||
// Dimension registry: dedupe by identity, first occurrence wins.
|
||||
const dimensionsByNo = new Map<number, SIEDimension>()
|
||||
const dimensionValuesByKey = new Map<string, SIEDimensionValue>()
|
||||
for (const file of files) {
|
||||
for (const dim of file.dimensions) {
|
||||
if (!dimensionsByNo.has(dim.sieDimNo)) dimensionsByNo.set(dim.sieDimNo, dim)
|
||||
}
|
||||
for (const value of file.dimensionValues) {
|
||||
const key = `${value.sieDimNo}|${value.code}`
|
||||
if (!dimensionValuesByKey.has(key)) dimensionValuesByKey.set(key, value)
|
||||
}
|
||||
}
|
||||
|
||||
const vouchers = files.flatMap((f) => f.vouchers)
|
||||
const currentFiscalYear = fiscalYears.find((fy) => fy.yearIndex === 0)
|
||||
|
||||
return {
|
||||
header: {
|
||||
sieType: files.reduce(
|
||||
(max, f) => (f.header.sieType > max ? f.header.sieType : max),
|
||||
files[0].header.sieType,
|
||||
),
|
||||
// Any file flagged as already imported taints the merged view.
|
||||
flagga: headers.some((h) => h.flagga === 1)
|
||||
? 1
|
||||
: firstNonNull(headers.map((h) => h.flagga)),
|
||||
program: firstNonNull(headers.map((h) => h.program)),
|
||||
programVersion: firstNonNull(headers.map((h) => h.programVersion)),
|
||||
generatedDate: firstNonNull(headers.map((h) => h.generatedDate)),
|
||||
format: firstNonNull(headers.map((h) => h.format)),
|
||||
companyName: mostCommon(headers.map((h) => h.companyName)),
|
||||
orgNumber: mostCommon(headers.map((h) => h.orgNumber)),
|
||||
address: firstNonNull(headers.map((h) => h.address)),
|
||||
fiscalYears,
|
||||
currency: firstNonNull(headers.map((h) => h.currency || null)) ?? 'SEK',
|
||||
kontoPlanType: firstNonNull(headers.map((h) => h.kontoPlanType)),
|
||||
},
|
||||
accounts,
|
||||
openingBalances: files.flatMap((f) => f.openingBalances),
|
||||
closingBalances: files.flatMap((f) => f.closingBalances),
|
||||
resultBalances: files.flatMap((f) => f.resultBalances),
|
||||
vouchers,
|
||||
dimensions: [...dimensionsByNo.values()],
|
||||
dimensionValues: [...dimensionValuesByKey.values()],
|
||||
issues: files.flatMap((f) => f.issues),
|
||||
stats: {
|
||||
totalAccounts: accounts.length,
|
||||
totalVouchers: vouchers.length,
|
||||
totalTransactionLines: vouchers.reduce((sum, v) => sum + v.lines.length, 0),
|
||||
fiscalYearStart: currentFiscalYear?.start ?? null,
|
||||
fiscalYearEnd: currentFiscalYear?.end ?? null,
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user