fix(import): SEB CSV imports survive BOMs and bad format choices (#1565)
* fix(import): handle BOMs at the byte level in decodeFileContent Inspect leading bytes before decoding: EF BB BF strips the UTF-8 BOM and decodes the remainder (falling back to Windows-1252 for the remainder only, so the fallback can no longer produce a literal mojibake prefix), and FF FE / FE FF decode as UTF-16LE/BE. stripBOM additionally strips a literal mojibake BOM prefix for string paths pre-decoded elsewhere. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(import): make an explicit SEB choice at least as good as auto-detect Three changes for the SEB bank CSV report: - parseBankFile: when an explicit format parses 0 transactions, fall back to auto-detection; a different format that parses rows is returned with a prepended info issue naming both formats. A working explicit parse is never overridden, and explicit generic_csv (the manual mapping escape hatch) is exempt. - SEB profile: sniff the header delimiter (';' vs ',') and split with the quote-aware parseCSVLine; accept a bare Datum date column as a lowest priority tier in parse only, never in detect. Its user-reachable issue strings are now Swedish. - Import page: when a parse yields 0 transactions, show the parser's real issues instead of only the generic no-transactions hint. The v1 agent route now decodes through the shared decodeFileContent and stamps external ids, import_source, and the stored file format from the format the parse result actually carries, so fallback imports dedup identically to auto-detected ones. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * docs(api-v1): the bank import route also decodes UTF-16 CodeRabbit on #1565: decodeFileContent gained UTF-16LE/BE BOM support but the route overview and the registered endpoint description still listed only UTF-8 / Windows-1252. Skill regenerated (apiskill:generate). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Claude Fable 5
Jakob Wennberg
Claude Fable 5
Jakob Wennberg
parent
0d3ba5268d
commit
9c891ee72d
@@ -278,6 +278,25 @@ describe('POST /api/v1/companies/:companyId/imports/bank', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('stamps ids and provenance with the fallback format when an explicit override parses nothing', async () => {
|
||||
// Swedbank file forced as `seb`: the parser falls back to the detected
|
||||
// format, and external ids / import_source must follow the format the
|
||||
// result carries so they equal what the auto-detect path would produce.
|
||||
const swedbankCsv = [
|
||||
'Kontouppgifter',
|
||||
'Clearingnummer,Kontonummer,Datum,Text,Belopp,Saldo',
|
||||
'8123,12345678,2024-01-15,SPOTIFY AB,-99.00,12345.67',
|
||||
].join('\n')
|
||||
|
||||
const res = await callRoute({ fileContent: swedbankCsv, search: '?format=seb' })
|
||||
expect(res.status).toBe(202)
|
||||
|
||||
const rows = ingestedRows()
|
||||
expect(rows).toHaveLength(1)
|
||||
expect(rows[0].import_source).toBe('csv_swedbank')
|
||||
expect(rows[0].external_id).toMatch(/^swedbank_/)
|
||||
})
|
||||
|
||||
it('never sends keys RawTransaction does not have (`source`, `counterparty`)', async () => {
|
||||
await callRoute()
|
||||
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
*
|
||||
* Bank-file import. Multipart upload: the file is the request body. The
|
||||
* route:
|
||||
* 1. Decodes the file (UTF-8 / Windows-1252 auto-detected).
|
||||
* 1. Decodes the file (UTF-8 / UTF-16 / Windows-1252 auto-detected;
|
||||
* BOMs handled at the byte level).
|
||||
* 2. Detects the bank file format (SEB / Swedbank / Nordea / Handelsbanken
|
||||
* / Lansforsakringar / Lunar / ICA Banken / Skandia / CAMT053 /
|
||||
* Nordea Business / Wise / generic CSV), or honors the optional `format`
|
||||
@@ -36,6 +37,7 @@ import {
|
||||
generateExternalId,
|
||||
} from '@/lib/import/bank-file/parser'
|
||||
import { ingestTransactions, type RawTransaction } from '@/lib/transactions/ingest'
|
||||
import { decodeFileContent } from '@/lib/import/shared/encoding'
|
||||
import type { BankFileFormatId } from '@/lib/import/bank-file/types'
|
||||
import { getErrorMessage as getUserErrorMessage } from '@/lib/errors/get-error-message'
|
||||
|
||||
@@ -54,7 +56,7 @@ registerEndpoint({
|
||||
path: '/api/v1/companies/:companyId/imports/bank',
|
||||
summary: 'Import a bank-file (CSV / XML / CAMT053).',
|
||||
description:
|
||||
'Accepts a bank statement file (UTF-8 / Windows-1252, up to 10 MB) as multipart/form-data. Auto-detects the bank format (SEB, Swedbank, Handelsbanken, Nordea, Nordea Business, Lansforsakringar, Lunar, ICA Banken, Skandia, Wise transaction history, Wise balance statement, CAMT053, generic CSV) or honors a `format` override. Parses transactions, ingests them into the `transactions` table (NOT into journal entries: see BFL note in pitfalls), and emits `transaction.synced` events. Returns operation_id for polling.',
|
||||
'Accepts a bank statement file (UTF-8 / UTF-16 / Windows-1252, up to 10 MB) as multipart/form-data. Auto-detects the bank format (SEB, Swedbank, Handelsbanken, Nordea, Nordea Business, Lansforsakringar, Lunar, ICA Banken, Skandia, Wise transaction history, Wise balance statement, CAMT053, generic CSV) or honors a `format` override. Parses transactions, ingests them into the `transactions` table (NOT into journal entries: see BFL note in pitfalls), and emits `transaction.synced` events. Returns operation_id for polling.',
|
||||
useWhen:
|
||||
'Importing a bank statement export for a period. Common with PSD2 bank connections that don\'t auto-sync, or for legacy bank accounts.',
|
||||
doNotUseFor:
|
||||
@@ -158,13 +160,10 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
|
||||
formatOverride = parsed.data
|
||||
}
|
||||
|
||||
// Decode the file. Bank files are typically Windows-1252 or UTF-8; we
|
||||
// try UTF-8 first and fall back if invalid replacement chars appear.
|
||||
// Decode the file with the shared importer decode (BOM-aware UTF-8 /
|
||||
// UTF-16 / Windows-1252): one decode implementation for every path.
|
||||
const buffer = await file.arrayBuffer()
|
||||
const utf8 = new TextDecoder('utf-8').decode(buffer)
|
||||
const content = utf8.includes('�')
|
||||
? new TextDecoder('windows-1252').decode(buffer)
|
||||
: utf8
|
||||
const content = decodeFileContent(buffer)
|
||||
|
||||
const fileHash = await generateFileHash(content)
|
||||
|
||||
@@ -178,6 +177,11 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
|
||||
}
|
||||
|
||||
const parseResult = parseBankFile(content, file.name, format)
|
||||
// parseBankFile may fall back to a detected sibling format when an
|
||||
// explicit override parses 0 transactions. Everything downstream (external
|
||||
// ids, import_source, stored file_format) must use the format the result
|
||||
// actually carries so ids equal what the auto-detect path would produce.
|
||||
const effectiveFormat = parseResult.format
|
||||
const blockingIssues = parseResult.issues.filter((issue) => issue.severity === 'error')
|
||||
if (blockingIssues.length > 0) {
|
||||
// Cap the reported rows so a large malformed file cannot balloon the
|
||||
@@ -195,7 +199,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
|
||||
if (parseResult.transactions.length === 0) {
|
||||
return v1ErrorResponseFromCode('BANK_FILE_NO_TRANSACTIONS', ctx.log, {
|
||||
requestId: ctx.requestId,
|
||||
details: { format, filename: file.name },
|
||||
details: { format: effectiveFormat, filename: file.name },
|
||||
})
|
||||
}
|
||||
|
||||
@@ -208,7 +212,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
|
||||
params: {
|
||||
filename: file.name,
|
||||
file_size: file.size,
|
||||
format,
|
||||
format: effectiveFormat,
|
||||
file_hash: fileHash,
|
||||
transaction_count: parseResult.transactions.length,
|
||||
},
|
||||
@@ -232,7 +236,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
|
||||
company_id: ctx.companyId!,
|
||||
filename: file.name,
|
||||
file_hash: fileHash,
|
||||
file_format: format,
|
||||
file_format: effectiveFormat,
|
||||
transaction_count: parseResult.transactions.length,
|
||||
status: 'processing',
|
||||
date_from: parseResult.date_from,
|
||||
@@ -271,13 +275,13 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
|
||||
// shared ingest contract, not in this route alone.
|
||||
const raw: RawTransaction[] = parseResult.transactions.map(
|
||||
(t, idx): RawTransaction => ({
|
||||
external_id: generateExternalId(t, format, idx),
|
||||
external_id: generateExternalId(t, effectiveFormat, idx),
|
||||
date: t.date,
|
||||
amount: t.amount,
|
||||
currency: t.currency || 'SEK',
|
||||
description: t.description,
|
||||
reference: t.reference ?? null,
|
||||
import_source: format === 'camt053' ? 'camt053' : `csv_${format}`,
|
||||
import_source: effectiveFormat === 'camt053' ? 'camt053' : `csv_${effectiveFormat}`,
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -329,7 +333,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
|
||||
{
|
||||
id: op.id,
|
||||
result: {
|
||||
format,
|
||||
format: effectiveFormat,
|
||||
file_hash: fileHash,
|
||||
transactions_imported: ingestResult.imported,
|
||||
transactions_duplicates: ingestResult.duplicates,
|
||||
|
||||
Reference in New Issue
Block a user