From 9c891ee72d323dd33d74aadb03b3edab37fa9ba2 Mon Sep 17 00:00:00 2001 From: Jakob Wennberg Date: Thu, 13 Aug 2026 15:22:19 +0200 Subject: [PATCH] fix(import): SEB CSV imports survive BOMs and bad format choices (#1565) * fix(import): handle BOMs at the byte level in decodeFileContent Inspect leading bytes before decoding: EF BB BF strips the UTF-8 BOM and decodes the remainder (falling back to Windows-1252 for the remainder only, so the fallback can no longer produce a literal mojibake prefix), and FF FE / FE FF decode as UTF-16LE/BE. stripBOM additionally strips a literal mojibake BOM prefix for string paths pre-decoded elsewhere. Co-Authored-By: Claude Fable 5 * fix(import): make an explicit SEB choice at least as good as auto-detect Three changes for the SEB bank CSV report: - parseBankFile: when an explicit format parses 0 transactions, fall back to auto-detection; a different format that parses rows is returned with a prepended info issue naming both formats. A working explicit parse is never overridden, and explicit generic_csv (the manual mapping escape hatch) is exempt. - SEB profile: sniff the header delimiter (';' vs ',') and split with the quote-aware parseCSVLine; accept a bare Datum date column as a lowest priority tier in parse only, never in detect. Its user-reachable issue strings are now Swedish. - Import page: when a parse yields 0 transactions, show the parser's real issues instead of only the generic no-transactions hint. The v1 agent route now decodes through the shared decodeFileContent and stamps external ids, import_source, and the stored file format from the format the parse result actually carries, so fallback imports dedup identically to auto-detected ones. Co-Authored-By: Claude Fable 5 * docs(api-v1): the bank import route also decodes UTF-16 CodeRabbit on #1565: decodeFileContent gained UTF-16LE/BE BOM support but the route overview and the registered endpoint description still listed only UTF-8 / Windows-1252. Skill regenerated (apiskill:generate). Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- app/(dashboard)/import/page.tsx | 16 ++- .../imports/bank/__tests__/route.test.ts | 19 +++ .../[companyId]/imports/bank/route.ts | 32 +++-- lib/import/bank-file/__tests__/parser.test.ts | 133 +++++++++++++++++- lib/import/bank-file/formats/seb.ts | 33 +++-- lib/import/bank-file/parser.ts | 34 ++++- lib/import/bank-file/types.ts | 2 +- lib/import/shared/__tests__/encoding.test.ts | 44 ++++++ lib/import/shared/encoding.ts | 41 +++++- skills/accounted-api/references/banking.md | 2 +- 10 files changed, 322 insertions(+), 34 deletions(-) diff --git a/app/(dashboard)/import/page.tsx b/app/(dashboard)/import/page.tsx index e553e6a4..3db7220e 100644 --- a/app/(dashboard)/import/page.tsx +++ b/app/(dashboard)/import/page.tsx @@ -278,8 +278,20 @@ function BankFileImportWizard() { description: `${txCount} transaktioner hittades`, }) } else { - // Format detected but no transactions parsed: parser couldn't extract rows - setBankError('Filen kunde läsas men inga transaktioner hittades. Kontrollera att filen innehåller transaktionsdata och inte bara rubriker.') + // Format detected but no transactions parsed: surface the parser's + // real issues when it produced any, so the user sees WHY (wrong + // columns, invalid dates, ...) instead of only a generic hint. + const parseIssues: BankFileParseResult['issues'] = data.data.parse_result.issues ?? [] + if (parseIssues.length > 0) { + setBankError( + parseIssues + .slice(0, 5) + .map((issue) => (issue.row > 0 ? `Rad ${issue.row}: ${issue.message}` : issue.message)) + .join(' ') + ) + } else { + setBankError('Filen kunde läsas men inga transaktioner hittades. Kontrollera att filen innehåller transaktionsdata och inte bara rubriker.') + } } } catch (err) { setBankError(err instanceof Error ? getErrorMessage(err) : 'Kunde inte läsa filen') diff --git a/app/api/v1/companies/[companyId]/imports/bank/__tests__/route.test.ts b/app/api/v1/companies/[companyId]/imports/bank/__tests__/route.test.ts index baf98bc4..6ca6b32e 100644 --- a/app/api/v1/companies/[companyId]/imports/bank/__tests__/route.test.ts +++ b/app/api/v1/companies/[companyId]/imports/bank/__tests__/route.test.ts @@ -278,6 +278,25 @@ describe('POST /api/v1/companies/:companyId/imports/bank', () => { } }) + it('stamps ids and provenance with the fallback format when an explicit override parses nothing', async () => { + // Swedbank file forced as `seb`: the parser falls back to the detected + // format, and external ids / import_source must follow the format the + // result carries so they equal what the auto-detect path would produce. + const swedbankCsv = [ + 'Kontouppgifter', + 'Clearingnummer,Kontonummer,Datum,Text,Belopp,Saldo', + '8123,12345678,2024-01-15,SPOTIFY AB,-99.00,12345.67', + ].join('\n') + + const res = await callRoute({ fileContent: swedbankCsv, search: '?format=seb' }) + expect(res.status).toBe(202) + + const rows = ingestedRows() + expect(rows).toHaveLength(1) + expect(rows[0].import_source).toBe('csv_swedbank') + expect(rows[0].external_id).toMatch(/^swedbank_/) + }) + it('never sends keys RawTransaction does not have (`source`, `counterparty`)', async () => { await callRoute() diff --git a/app/api/v1/companies/[companyId]/imports/bank/route.ts b/app/api/v1/companies/[companyId]/imports/bank/route.ts index a8d8af2d..c4a9c7f1 100644 --- a/app/api/v1/companies/[companyId]/imports/bank/route.ts +++ b/app/api/v1/companies/[companyId]/imports/bank/route.ts @@ -3,7 +3,8 @@ * * Bank-file import. Multipart upload: the file is the request body. The * route: - * 1. Decodes the file (UTF-8 / Windows-1252 auto-detected). + * 1. Decodes the file (UTF-8 / UTF-16 / Windows-1252 auto-detected; + * BOMs handled at the byte level). * 2. Detects the bank file format (SEB / Swedbank / Nordea / Handelsbanken * / Lansforsakringar / Lunar / ICA Banken / Skandia / CAMT053 / * Nordea Business / Wise / generic CSV), or honors the optional `format` @@ -36,6 +37,7 @@ import { generateExternalId, } from '@/lib/import/bank-file/parser' import { ingestTransactions, type RawTransaction } from '@/lib/transactions/ingest' +import { decodeFileContent } from '@/lib/import/shared/encoding' import type { BankFileFormatId } from '@/lib/import/bank-file/types' import { getErrorMessage as getUserErrorMessage } from '@/lib/errors/get-error-message' @@ -54,7 +56,7 @@ registerEndpoint({ path: '/api/v1/companies/:companyId/imports/bank', summary: 'Import a bank-file (CSV / XML / CAMT053).', description: - 'Accepts a bank statement file (UTF-8 / Windows-1252, up to 10 MB) as multipart/form-data. Auto-detects the bank format (SEB, Swedbank, Handelsbanken, Nordea, Nordea Business, Lansforsakringar, Lunar, ICA Banken, Skandia, Wise transaction history, Wise balance statement, CAMT053, generic CSV) or honors a `format` override. Parses transactions, ingests them into the `transactions` table (NOT into journal entries: see BFL note in pitfalls), and emits `transaction.synced` events. Returns operation_id for polling.', + 'Accepts a bank statement file (UTF-8 / UTF-16 / Windows-1252, up to 10 MB) as multipart/form-data. Auto-detects the bank format (SEB, Swedbank, Handelsbanken, Nordea, Nordea Business, Lansforsakringar, Lunar, ICA Banken, Skandia, Wise transaction history, Wise balance statement, CAMT053, generic CSV) or honors a `format` override. Parses transactions, ingests them into the `transactions` table (NOT into journal entries: see BFL note in pitfalls), and emits `transaction.synced` events. Returns operation_id for polling.', useWhen: 'Importing a bank statement export for a period. Common with PSD2 bank connections that don\'t auto-sync, or for legacy bank accounts.', doNotUseFor: @@ -158,13 +160,10 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( formatOverride = parsed.data } - // Decode the file. Bank files are typically Windows-1252 or UTF-8; we - // try UTF-8 first and fall back if invalid replacement chars appear. + // Decode the file with the shared importer decode (BOM-aware UTF-8 / + // UTF-16 / Windows-1252): one decode implementation for every path. const buffer = await file.arrayBuffer() - const utf8 = new TextDecoder('utf-8').decode(buffer) - const content = utf8.includes('�') - ? new TextDecoder('windows-1252').decode(buffer) - : utf8 + const content = decodeFileContent(buffer) const fileHash = await generateFileHash(content) @@ -178,6 +177,11 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( } const parseResult = parseBankFile(content, file.name, format) + // parseBankFile may fall back to a detected sibling format when an + // explicit override parses 0 transactions. Everything downstream (external + // ids, import_source, stored file_format) must use the format the result + // actually carries so ids equal what the auto-detect path would produce. + const effectiveFormat = parseResult.format const blockingIssues = parseResult.issues.filter((issue) => issue.severity === 'error') if (blockingIssues.length > 0) { // Cap the reported rows so a large malformed file cannot balloon the @@ -195,7 +199,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( if (parseResult.transactions.length === 0) { return v1ErrorResponseFromCode('BANK_FILE_NO_TRANSACTIONS', ctx.log, { requestId: ctx.requestId, - details: { format, filename: file.name }, + details: { format: effectiveFormat, filename: file.name }, }) } @@ -208,7 +212,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( params: { filename: file.name, file_size: file.size, - format, + format: effectiveFormat, file_hash: fileHash, transaction_count: parseResult.transactions.length, }, @@ -232,7 +236,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( company_id: ctx.companyId!, filename: file.name, file_hash: fileHash, - file_format: format, + file_format: effectiveFormat, transaction_count: parseResult.transactions.length, status: 'processing', date_from: parseResult.date_from, @@ -271,13 +275,13 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( // shared ingest contract, not in this route alone. const raw: RawTransaction[] = parseResult.transactions.map( (t, idx): RawTransaction => ({ - external_id: generateExternalId(t, format, idx), + external_id: generateExternalId(t, effectiveFormat, idx), date: t.date, amount: t.amount, currency: t.currency || 'SEK', description: t.description, reference: t.reference ?? null, - import_source: format === 'camt053' ? 'camt053' : `csv_${format}`, + import_source: effectiveFormat === 'camt053' ? 'camt053' : `csv_${effectiveFormat}`, }), ) @@ -329,7 +333,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( { id: op.id, result: { - format, + format: effectiveFormat, file_hash: fileHash, transactions_imported: ingestResult.imported, transactions_duplicates: ingestResult.duplicates, diff --git a/lib/import/bank-file/__tests__/parser.test.ts b/lib/import/bank-file/__tests__/parser.test.ts index 802b8bac..5c28821e 100644 --- a/lib/import/bank-file/__tests__/parser.test.ts +++ b/lib/import/bank-file/__tests__/parser.test.ts @@ -1389,7 +1389,9 @@ describe('parseBankFile: camt.053 XML format', () => { describe('parseBankFile: explicit format override', () => { it('uses the specified format instead of auto-detection', () => { - // Force parsing Nordea content as SEB (will produce issues but should use SEB format) + // Nordea content forced as SEB: the widened SEB parser handles it via + // delimiter sniffing + the bare-Datum tier, and a working explicit parse + // is never overridden by the auto-detect fallback. const result = parseBankFile( 'Datum,Transaktion,Kategori,Belopp,Saldo\n2024-01-15,Test,,"-100,00","5000,00"', 'nordea.csv', @@ -1397,6 +1399,8 @@ describe('parseBankFile: explicit format override', () => { ) expect(result.format).toBe('seb') + expect(result.transactions).toHaveLength(1) + expect(result.transactions[0].amount).toBe(-100) }) it('returns error for unknown formatId', () => { @@ -1407,7 +1411,7 @@ describe('parseBankFile: explicit format override', () => { expect(result.transactions).toHaveLength(0) expect(result.issues).toHaveLength(1) expect(result.issues[0].severity).toBe('error') - expect(result.issues[0].message).toContain('Unknown format') + expect(result.issues[0].message).toContain('Okänt format') }) it('returns format detection error when no format matches and no override given', () => { @@ -1430,6 +1434,131 @@ describe('parseBankFile: explicit format override', () => { }) }) +describe('parseBankFile: explicit format fallback to auto-detection', () => { + it('falls back to the detected format when the explicit choice parses 0 transactions', () => { + const result = parseBankFile(SWEDBANK_CSV, 'export.csv', 'seb') + + expect(result.format).toBe('swedbank') + expect(result.transactions).toHaveLength(3) + expect(result.issues[0].severity).toBe('info') + expect(result.issues[0].message).toContain('SEB') + expect(result.issues[0].message).toContain('Swedbank') + }) + + it('falls back to Handelsbanken when a Handelsbanken file is forced as SEB', () => { + const result = parseBankFile(HANDELSBANKEN_CSV, 'export.csv', 'seb') + + expect(result.format).toBe('handelsbanken') + expect(result.transactions).toHaveLength(3) + expect(result.issues[0].severity).toBe('info') + expect(result.issues[0].message).toContain('Handelsbanken') + }) + + it('never overrides a working explicit parse even when detection prefers another format', () => { + // NORDEA_CSV auto-detects as nordea, but forced-SEB parses it fine via + // delimiter sniffing + the bare-Datum tier: the user's choice stands. + expect(detectFileFormat(NORDEA_CSV, 'nordea.csv')!.id).toBe('nordea') + + const result = parseBankFile(NORDEA_CSV, 'nordea.csv', 'seb') + + expect(result.format).toBe('seb') + expect(result.transactions).toHaveLength(3) + expect(result.issues.filter((i) => i.severity === 'info')).toHaveLength(0) + }) + + it('keeps the explicit error result when no other format can parse the file', () => { + const result = parseBankFile(UNKNOWN_CSV, 'unknown.csv', 'seb') + + expect(result.format).toBe('seb') + expect(result.transactions).toHaveLength(0) + expect(result.issues[0].severity).toBe('error') + expect(result.issues[0].message).toContain('Kunde inte identifiera nödvändiga kolumner') + }) + + it('does not fall back for explicit generic_csv (the manual mapping escape hatch)', () => { + // The default generic mapping parses 0 rows of a Nordea file, but the + // user chose "Annan CSV" to map columns manually: never reroute them. + const result = parseBankFile(NORDEA_CSV, 'nordea.csv', 'generic_csv') + + expect(result.format).toBe('generic_csv') + }) +}) + +describe('parseBankFile: BOM handling', () => { + it('auto-detects and parses SEB CSV with a real UTF-8 BOM (U+FEFF)', () => { + const content = '\uFEFF' + SEB_CSV + + expect(detectFileFormat(content, 'seb.csv')!.id).toBe('seb') + + const result = parseBankFile(content, 'seb.csv') + expect(result.format).toBe('seb') + expect(result.transactions).toHaveLength(3) + }) + + it('auto-detects and parses SEB privat CSV with a mojibake BOM prefix', () => { + const content = '' + SEB_PRIVAT_CSV + + expect(detectFileFormat(content, 'kontoutdrag.csv')!.id).toBe('seb') + + const result = parseBankFile(content, 'kontoutdrag.csv') + expect(result.format).toBe('seb') + expect(result.transactions).toHaveLength(3) + }) + + it('detects an exact-match Datum header behind a mojibake BOM prefix', () => { + // Exact-match header checks (h === 'datum') are the genuinely BOM-fragile + // ones: a surviving mojibake prefix used to make this file undetectable. + const content = '' + [ + 'Datum;Text;Belopp;Saldo', + '2026-01-15;SPOTIFY AB;-99,00;1000,00', + ].join('\n') + + const format = detectFileFormat(content, 'export.csv') + expect(format).not.toBeNull() + expect(format!.id).toBe('nordea_business') + + const result = parseBankFile(content, 'export.csv') + expect(result.transactions).toHaveLength(1) + expect(result.transactions[0].amount).toBe(-99) + }) +}) + +describe('parseBankFile: SEB delimiter sniffing and bare-Datum tier', () => { + it('parses a comma-delimited SEB-labeled file under explicit seb', () => { + const commaSeb = [ + 'Bokföringsdag,Valutadag,Verifikationsnummer,Text,Belopp,Saldo', + '2024-01-15,2024-01-15,12345,SPOTIFY AB,"-99,00","12345,67"', + '2024-01-14,2024-01-14,12346,HEMKÖP,"-432,50","12444,67"', + ].join('\n') + + const result = parseBankFile(commaSeb, 'seb.csv', 'seb') + + expect(result.format).toBe('seb') + expect(result.transactions).toHaveLength(2) + expect(result.transactions[0].amount).toBe(-99) + expect(result.transactions[0].description).toBe('SPOTIFY AB') + expect(result.transactions[0].balance).toBe(12345.67) + }) + + it('parses a bare-Datum layout under explicit seb without claiming it in detect', () => { + const bareDatum = [ + 'Datum;Text;Belopp;Saldo', + '2026-02-01;SPOTIFY AB;-99,00;1000,00', + '2026-02-02;LÖN;25000,00;26000,00', + ].join('\n') + + // detect must NOT claim the bare-Datum layout: in auto-detection it + // belongs to other profiles (nordea_business format D family). + expect(getFormat('seb')!.detect(bareDatum, 'seb.csv')).toBe(false) + + const result = parseBankFile(bareDatum, 'seb.csv', 'seb') + + expect(result.format).toBe('seb') + expect(result.transactions).toHaveLength(2) + expect(result.transactions[1].amount).toBe(25000) + }) +}) + describe('generateExternalId', () => { const baseTx: ParsedBankTransaction = { date: '2024-01-15', diff --git a/lib/import/bank-file/formats/seb.ts b/lib/import/bank-file/formats/seb.ts index ccd4600f..8beed391 100644 --- a/lib/import/bank-file/formats/seb.ts +++ b/lib/import/bank-file/formats/seb.ts @@ -1,7 +1,8 @@ /** * SEB CSV format parser * - * Format: Semicolon-delimited, comma decimal separator + * Format: Semicolon-delimited, comma decimal separator (parse also accepts + * comma-delimited variants via delimiter sniffing; detect stays strict) * Columns vary but typically: Bokföringsdag, Valutadag, Verifikationsnummer, * Text/mottagare, Belopp, Saldo * Date format: YYYY-MM-DD @@ -11,6 +12,7 @@ import type { BankFileFormat, BankFileParseResult, ParsedBankTransaction, BankFileParseIssue } from '../types' import { prepareContent } from '../../shared/encoding' import { normalizeDate } from '../date-utils' +import { parseCSVLine } from './nordea' function parseCommaDecimal(value: string): number { const cleaned = value.replace(/\s/g, '').replace(',', '.') @@ -43,12 +45,25 @@ export const sebFormat: BankFileFormat = { const issues: BankFileParseIssue[] = [] let skippedRows = 0 - // Parse header to find column indices + // Parse header to find column indices. SEB normally exports + // semicolon-delimited files, but some export surfaces use commas: sniff + // the delimiter from the header line instead of assuming ';'. const headerLine = lines[0] || '' - const headers = headerLine.split(';').map((h) => h.trim().toLowerCase().replace(/"/g, '')) + const semicolons = (headerLine.match(/;/g) || []).length + const commas = (headerLine.match(/,/g) || []).length + const delimiter = commas > semicolons ? ',' : ';' + const headers = parseCSVLine(headerLine, delimiter).map((h) => + h.trim().toLowerCase().replace(/"/g, '') + ) // Find column indices dynamically - const dateIdx = headers.findIndex((h) => /bokf(ö|o)ringsda(g|tum)/.test(h)) + let dateIdx = headers.findIndex((h) => /bokf(ö|o)ringsda(g|tum)/.test(h)) + if (dateIdx === -1) { + // Lowest-priority tier: a bare "Datum" column. Only honored here in + // parse (an explicit user choice), never in detect, so this profile + // cannot steal files from other bank profiles during auto-detection. + dateIdx = headers.findIndex((h) => h === 'datum') + } const descIdx = headers.findIndex( (h) => h.includes('text') || h.includes('mottagare') || h.includes('beskrivning') ) @@ -58,7 +73,7 @@ export const sebFormat: BankFileFormat = { if (dateIdx === -1 || amountIdx === -1) { issues.push({ row: 1, - message: 'Could not identify required columns (date, amount)', + message: 'Kunde inte identifiera nödvändiga kolumner (datum, belopp)', severity: 'error', }) return { @@ -76,7 +91,7 @@ export const sebFormat: BankFileFormat = { const line = lines[i].trim() if (!line) continue - const fields = line.split(';').map((f) => f.trim().replace(/^"|"$/g, '')) + const fields = parseCSVLine(line, delimiter).map((f) => f.trim().replace(/^"|"$/g, '')) const date = fields[dateIdx] const description = fields[descIdx >= 0 ? descIdx : dateIdx + 1] || 'Unknown' @@ -84,21 +99,21 @@ export const sebFormat: BankFileFormat = { const balanceStr = balanceIdx >= 0 ? fields[balanceIdx] : undefined if (!date || !amountStr) { - issues.push({ row: i + 1, message: 'Missing required fields', severity: 'warning' }) + issues.push({ row: i + 1, message: 'Obligatoriska fält saknas', severity: 'warning' }) skippedRows++ continue } const amount = parseCommaDecimal(amountStr) if (isNaN(amount)) { - issues.push({ row: i + 1, message: `Invalid amount: ${amountStr}`, severity: 'warning' }) + issues.push({ row: i + 1, message: `Ogiltigt belopp: ${amountStr}`, severity: 'warning' }) skippedRows++ continue } const normalizedDate = normalizeDate(date) if (!normalizedDate) { - issues.push({ row: i + 1, message: `Invalid date: ${date}`, severity: 'warning' }) + issues.push({ row: i + 1, message: `Ogiltigt datum: ${date}`, severity: 'warning' }) skippedRows++ continue } diff --git a/lib/import/bank-file/parser.ts b/lib/import/bank-file/parser.ts index ad83ec7e..98ac8ef0 100644 --- a/lib/import/bank-file/parser.ts +++ b/lib/import/bank-file/parser.ts @@ -97,10 +97,42 @@ export function parseBankFile( transactions: [], date_from: null, date_to: null, - issues: [{ row: 0, message: `Unknown format: ${formatId}`, severity: 'error' }], + issues: [{ row: 0, message: `Okänt format: ${formatId}`, severity: 'error' }], stats: { total_rows: 0, parsed_rows: 0, skipped_rows: 0, total_income: 0, total_expenses: 0 }, } } + + const explicitResult = format.parse(content) + if (explicitResult.transactions.length > 0 || formatId === 'generic_csv') { + // A working explicit parse is never overridden. generic_csv is also + // exempt: it is the manual column-mapping escape hatch and its default + // mapping legitimately parses 0 rows before the user maps columns. + return explicitResult + } + + // The explicit choice parsed nothing: fall back to auto-detection so an + // explicitly selected bank is never WORSE than "Automatisk identifiering". + // detectFileFormat can never return generic_csv (its detect() is always + // false), so this cannot reroute the UI into the mapping flow. + const detected = detectFileFormat(content, filename) + if (detected && detected.id !== formatId) { + const detectedResult = detected.parse(content) + if (detectedResult.transactions.length > 0) { + return { + ...detectedResult, + issues: [ + { + row: 0, + message: `Filen matchade inte det valda formatet (${format.name}) och tolkades istället som ${detected.name}.`, + severity: 'info', + }, + ...detectedResult.issues, + ], + } + } + } + + return explicitResult } else { format = detectFileFormat(content, filename) || undefined if (!format) { diff --git a/lib/import/bank-file/types.ts b/lib/import/bank-file/types.ts index 002e309e..0fa43dfc 100644 --- a/lib/import/bank-file/types.ts +++ b/lib/import/bank-file/types.ts @@ -49,7 +49,7 @@ export interface BankFileDuplicateInfo { export interface BankFileParseIssue { row: number message: string - severity: 'warning' | 'error' + severity: 'info' | 'warning' | 'error' } /** Supported bank file format identifiers */ diff --git a/lib/import/shared/__tests__/encoding.test.ts b/lib/import/shared/__tests__/encoding.test.ts index 1c5babeb..e3df491e 100644 --- a/lib/import/shared/__tests__/encoding.test.ts +++ b/lib/import/shared/__tests__/encoding.test.ts @@ -5,6 +5,7 @@ import { hasEncodingIssues, recoverStringWithFFFD, recoverWordWithFFFD, + stripBOM, } from '../encoding' describe('decodeStringContent', () => { @@ -78,6 +79,49 @@ describe('decodeFileContent', () => { const cp1252 = buf([0x47, 0xd6, 0x54, 0x45, 0x42, 0x4f, 0x52, 0x47]) expect(decodeFileContent(cp1252)).toBe('GÖTEBORG') }) + + it('strips a UTF-8 BOM (EF BB BF) and decodes the remainder as UTF-8', () => { + const payload = Array.from(new TextEncoder().encode('Bokföringsdag;Belopp')) + const text = decodeFileContent(buf([0xef, 0xbb, 0xbf, ...payload])) + expect(text).toBe('Bokföringsdag;Belopp') + expect(text.charCodeAt(0)).not.toBe(0xfeff) + }) + + it('never produces a mojibake BOM prefix when a BOM-ed file needs the Windows-1252 fallback', () => { + // 'Datum;' + 0xD6 (Ö in Windows-1252, invalid as a lone UTF-8 byte), + // behind a UTF-8 BOM. The BOM bytes must never re-enter the fallback + // decode, so the result starts with 'Datum', not the literal mojibake. + const bytes = buf([0xef, 0xbb, 0xbf, 0x44, 0x61, 0x74, 0x75, 0x6d, 0x3b, 0xd6]) + const text = decodeFileContent(bytes) + expect(text.startsWith('')).toBe(false) + expect(text).toBe('Datum;Ö') + }) + + it('decodes UTF-16LE content behind a FF FE BOM', () => { + // 'Öre' in UTF-16LE: d6 00, 72 00, 65 00 + const bytes = buf([0xff, 0xfe, 0xd6, 0x00, 0x72, 0x00, 0x65, 0x00]) + expect(decodeFileContent(bytes)).toBe('Öre') + }) + + it('decodes UTF-16BE content behind a FE FF BOM', () => { + const bytes = buf([0xfe, 0xff, 0x00, 0xd6, 0x00, 0x72, 0x00, 0x65]) + expect(decodeFileContent(bytes)).toBe('Öre') + }) +}) + +describe('stripBOM', () => { + it('strips a leading U+FEFF', () => { + expect(stripBOM('\uFEFF' + 'Datum;Belopp')).toBe('Datum;Belopp') + }) + + it('strips a leading literal mojibake BOM ()', () => { + expect(stripBOM('Datum;Belopp')).toBe('Datum;Belopp') + }) + + it('is a no-op on clean content and never strips mid-string', () => { + expect(stripBOM('Datum;Belopp')).toBe('Datum;Belopp') + expect(stripBOM('Datum;Belopp')).toBe('Datum;Belopp') + }) }) // --- U+FFFD heuristic recovery --- diff --git a/lib/import/shared/encoding.ts b/lib/import/shared/encoding.ts index 94c96102..6b9c4be1 100644 --- a/lib/import/shared/encoding.ts +++ b/lib/import/shared/encoding.ts @@ -7,12 +7,38 @@ */ /** - * Decode file content, handling both UTF-8 and Windows-1252 encodings. + * Decode file content, handling BOMs plus UTF-8 and Windows-1252 encodings. * - * Strategy: Try UTF-8 first. If the result contains replacement characters - * (U+FFFD) or garbled Swedish chars, fall back to Windows-1252. + * Strategy: inspect the leading BYTES first. + * - FF FE / FE FF: UTF-16LE/BE (e.g. Excel "Unicode text" re-saves). Decode + * as such; TextDecoder consumes the BOM itself. + * - EF BB BF (UTF-8 BOM): strip the three bytes and decode the REMAINDER. + * If the remainder still contains U+FFFD (invalid UTF-8 somewhere in the + * file), decode the remainder as Windows-1252. The BOM bytes are never + * re-included, so the fallback can no longer produce a literal mojibake + * "" prefix that breaks exact-match header detection. + * - No BOM: try UTF-8; fall back to Windows-1252 on replacement characters + * or garbled Swedish chars (unchanged behavior). */ export function decodeFileContent(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer) + + if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) { + return new TextDecoder('utf-16le', { fatal: false }).decode(buffer) + } + if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) { + return new TextDecoder('utf-16be', { fatal: false }).decode(buffer) + } + + if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { + const rest = bytes.subarray(3) + const utf8Rest = new TextDecoder('utf-8', { fatal: false }).decode(rest) + if (!utf8Rest.includes('�')) { + return utf8Rest + } + return new TextDecoder('windows-1252', { fatal: false }).decode(rest) + } + const utf8Decoder = new TextDecoder('utf-8', { fatal: false }) const utf8Result = utf8Decoder.decode(buffer) @@ -72,12 +98,19 @@ export function normalizeLineEndings(content: string): string { } /** - * Strip BOM (Byte Order Mark) from start of content + * Strip BOM (Byte Order Mark) from start of content. + * + * Also strips the literal mojibake form "" (U+00EF U+00BB U+00BF): a + * UTF-8 BOM whose bytes were decoded as Windows-1252/Latin-1 upstream. + * Covers string-entry paths that were pre-decoded outside decodeFileContent. */ export function stripBOM(content: string): string { if (content.charCodeAt(0) === 0xfeff) { return content.slice(1) } + if (content.startsWith('')) { + return content.slice(3) + } return content } diff --git a/skills/accounted-api/references/banking.md b/skills/accounted-api/references/banking.md index 846fb01f..c3aa8c5f 100644 --- a/skills/accounted-api/references/banking.md +++ b/skills/accounted-api/references/banking.md @@ -12,7 +12,7 @@ are in SKILL.md and are not repeated per endpoint. **Import a bank-file (CSV / XML / CAMT053).** `scope:transactions:write · risk:medium · idempotent` -Accepts a bank statement file (UTF-8 / Windows-1252, up to 10 MB) as multipart/form-data. Auto-detects the bank format (SEB, Swedbank, Handelsbanken, Nordea, Nordea Business, Lansforsakringar, Lunar, ICA Banken, Skandia, Wise transaction history, Wise balance statement, CAMT053, generic CSV) or honors a `format` override. Parses transactions, ingests them into the `transactions` table (NOT into journal entries: see BFL note in pitfalls), and emits `transaction.synced` events. Returns operation_id for polling. +Accepts a bank statement file (UTF-8 / UTF-16 / Windows-1252, up to 10 MB) as multipart/form-data. Auto-detects the bank format (SEB, Swedbank, Handelsbanken, Nordea, Nordea Business, Lansforsakringar, Lunar, ICA Banken, Skandia, Wise transaction history, Wise balance statement, CAMT053, generic CSV) or honors a `format` override. Parses transactions, ingests them into the `transactions` table (NOT into journal entries: see BFL note in pitfalls), and emits `transaction.synced` events. Returns operation_id for polling. **Use when:** Importing a bank statement export for a period. Common with PSD2 bank connections that don't auto-sync, or for legacy bank accounts. **Do not use for:** SIE bookkeeping import (use /imports/sie). Auto-bank sync (use the enable-banking extension). Single-transaction creation (use POST /transactions/ingest with a 1-element array).