fix(import): SEB CSV imports survive BOMs and bad format choices (#1565)

* fix(import): handle BOMs at the byte level in decodeFileContent Inspect leading bytes before decoding: EF BB BF strips the UTF-8 BOM and decodes the remainder (falling back to Windows-1252 for the remainder only, so the fallback can no longer produce a literal mojibake prefix), and FF FE / FE FF decode as UTF-16LE/BE. stripBOM additionally strips a literal mojibake BOM prefix for string paths pre-decoded elsewhere. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(import): make an explicit SEB choice at least as good as auto-detect Three changes for the SEB bank CSV report: - parseBankFile: when an explicit format parses 0 transactions, fall back to auto-detection; a different format that parses rows is returned with a prepended info issue naming both formats. A working explicit parse is never overridden, and explicit generic_csv (the manual mapping escape hatch) is exempt. - SEB profile: sniff the header delimiter (';' vs ',') and split with the quote-aware parseCSVLine; accept a bare Datum date column as a lowest priority tier in parse only, never in detect. Its user-reachable issue strings are now Swedish. - Import page: when a parse yields 0 transactions, show the parser's real issues instead of only the generic no-transactions hint. The v1 agent route now decodes through the shared decodeFileContent and stamps external ids, import_source, and the stored file format from the format the parse result actually carries, so fallback imports dedup identically to auto-detected ones. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * docs(api-v1): the bank import route also decodes UTF-16 CodeRabbit on #1565: decodeFileContent gained UTF-16LE/BE BOM support but the route overview and the registered endpoint description still listed only UTF-8 / Windows-1252. Skill regenerated (apiskill:generate). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-13 15:22:19 +02:00
parent 0d3ba5268d
commit 9c891ee72d
10 changed files with 322 additions and 34 deletions
@@ -1389,7 +1389,9 @@ describe('parseBankFile: camt.053 XML format', () => {

 describe('parseBankFile: explicit format override', () => {
  it('uses the specified format instead of auto-detection', () => {
-    // Force parsing Nordea content as SEB (will produce issues but should use SEB format)
+    // Nordea content forced as SEB: the widened SEB parser handles it via
+    // delimiter sniffing + the bare-Datum tier, and a working explicit parse
+    // is never overridden by the auto-detect fallback.
    const result = parseBankFile(
      'Datum,Transaktion,Kategori,Belopp,Saldo\n2024-01-15,Test,,"-100,00","5000,00"',
      'nordea.csv',
@@ -1397,6 +1399,8 @@ describe('parseBankFile: explicit format override', () => {
    )

    expect(result.format).toBe('seb')
+    expect(result.transactions).toHaveLength(1)
+    expect(result.transactions[0].amount).toBe(-100)
  })

  it('returns error for unknown formatId', () => {
@@ -1407,7 +1411,7 @@ describe('parseBankFile: explicit format override', () => {
    expect(result.transactions).toHaveLength(0)
    expect(result.issues).toHaveLength(1)
    expect(result.issues[0].severity).toBe('error')
-    expect(result.issues[0].message).toContain('Unknown format')
+    expect(result.issues[0].message).toContain('Okänt format')
  })

  it('returns format detection error when no format matches and no override given', () => {
@@ -1430,6 +1434,131 @@ describe('parseBankFile: explicit format override', () => {
  })
 })

+describe('parseBankFile: explicit format fallback to auto-detection', () => {
+  it('falls back to the detected format when the explicit choice parses 0 transactions', () => {
+    const result = parseBankFile(SWEDBANK_CSV, 'export.csv', 'seb')
+
+    expect(result.format).toBe('swedbank')
+    expect(result.transactions).toHaveLength(3)
+    expect(result.issues[0].severity).toBe('info')
+    expect(result.issues[0].message).toContain('SEB')
+    expect(result.issues[0].message).toContain('Swedbank')
+  })
+
+  it('falls back to Handelsbanken when a Handelsbanken file is forced as SEB', () => {
+    const result = parseBankFile(HANDELSBANKEN_CSV, 'export.csv', 'seb')
+
+    expect(result.format).toBe('handelsbanken')
+    expect(result.transactions).toHaveLength(3)
+    expect(result.issues[0].severity).toBe('info')
+    expect(result.issues[0].message).toContain('Handelsbanken')
+  })
+
+  it('never overrides a working explicit parse even when detection prefers another format', () => {
+    // NORDEA_CSV auto-detects as nordea, but forced-SEB parses it fine via
+    // delimiter sniffing + the bare-Datum tier: the user's choice stands.
+    expect(detectFileFormat(NORDEA_CSV, 'nordea.csv')!.id).toBe('nordea')
+
+    const result = parseBankFile(NORDEA_CSV, 'nordea.csv', 'seb')
+
+    expect(result.format).toBe('seb')
+    expect(result.transactions).toHaveLength(3)
+    expect(result.issues.filter((i) => i.severity === 'info')).toHaveLength(0)
+  })
+
+  it('keeps the explicit error result when no other format can parse the file', () => {
+    const result = parseBankFile(UNKNOWN_CSV, 'unknown.csv', 'seb')
+
+    expect(result.format).toBe('seb')
+    expect(result.transactions).toHaveLength(0)
+    expect(result.issues[0].severity).toBe('error')
+    expect(result.issues[0].message).toContain('Kunde inte identifiera nödvändiga kolumner')
+  })
+
+  it('does not fall back for explicit generic_csv (the manual mapping escape hatch)', () => {
+    // The default generic mapping parses 0 rows of a Nordea file, but the
+    // user chose "Annan CSV" to map columns manually: never reroute them.
+    const result = parseBankFile(NORDEA_CSV, 'nordea.csv', 'generic_csv')
+
+    expect(result.format).toBe('generic_csv')
+  })
+})
+
+describe('parseBankFile: BOM handling', () => {
+  it('auto-detects and parses SEB CSV with a real UTF-8 BOM (U+FEFF)', () => {
+    const content = '\uFEFF' + SEB_CSV
+
+    expect(detectFileFormat(content, 'seb.csv')!.id).toBe('seb')
+
+    const result = parseBankFile(content, 'seb.csv')
+    expect(result.format).toBe('seb')
+    expect(result.transactions).toHaveLength(3)
+  })
+
+  it('auto-detects and parses SEB privat CSV with a mojibake BOM prefix', () => {
+    const content = 'ï»¿' + SEB_PRIVAT_CSV
+
+    expect(detectFileFormat(content, 'kontoutdrag.csv')!.id).toBe('seb')
+
+    const result = parseBankFile(content, 'kontoutdrag.csv')
+    expect(result.format).toBe('seb')
+    expect(result.transactions).toHaveLength(3)
+  })
+
+  it('detects an exact-match Datum header behind a mojibake BOM prefix', () => {
+    // Exact-match header checks (h === 'datum') are the genuinely BOM-fragile
+    // ones: a surviving mojibake prefix used to make this file undetectable.
+    const content = 'ï»¿' + [
+      'Datum;Text;Belopp;Saldo',
+      '2026-01-15;SPOTIFY AB;-99,00;1000,00',
+    ].join('\n')
+
+    const format = detectFileFormat(content, 'export.csv')
+    expect(format).not.toBeNull()
+    expect(format!.id).toBe('nordea_business')
+
+    const result = parseBankFile(content, 'export.csv')
+    expect(result.transactions).toHaveLength(1)
+    expect(result.transactions[0].amount).toBe(-99)
+  })
+})
+
+describe('parseBankFile: SEB delimiter sniffing and bare-Datum tier', () => {
+  it('parses a comma-delimited SEB-labeled file under explicit seb', () => {
+    const commaSeb = [
+      'Bokföringsdag,Valutadag,Verifikationsnummer,Text,Belopp,Saldo',
+      '2024-01-15,2024-01-15,12345,SPOTIFY AB,"-99,00","12345,67"',
+      '2024-01-14,2024-01-14,12346,HEMKÖP,"-432,50","12444,67"',
+    ].join('\n')
+
+    const result = parseBankFile(commaSeb, 'seb.csv', 'seb')
+
+    expect(result.format).toBe('seb')
+    expect(result.transactions).toHaveLength(2)
+    expect(result.transactions[0].amount).toBe(-99)
+    expect(result.transactions[0].description).toBe('SPOTIFY AB')
+    expect(result.transactions[0].balance).toBe(12345.67)
+  })
+
+  it('parses a bare-Datum layout under explicit seb without claiming it in detect', () => {
+    const bareDatum = [
+      'Datum;Text;Belopp;Saldo',
+      '2026-02-01;SPOTIFY AB;-99,00;1000,00',
+      '2026-02-02;LÖN;25000,00;26000,00',
+    ].join('\n')
+
+    // detect must NOT claim the bare-Datum layout: in auto-detection it
+    // belongs to other profiles (nordea_business format D family).
+    expect(getFormat('seb')!.detect(bareDatum, 'seb.csv')).toBe(false)
+
+    const result = parseBankFile(bareDatum, 'seb.csv', 'seb')
+
+    expect(result.format).toBe('seb')
+    expect(result.transactions).toHaveLength(2)
+    expect(result.transactions[1].amount).toBe(25000)
+  })
+})
+
 describe('generateExternalId', () => {
  const baseTx: ParsedBankTransaction = {
    date: '2024-01-15',
@@ -1,7 +1,8 @@
 /**
 * SEB CSV format parser
 *
- * Format: Semicolon-delimited, comma decimal separator
+ * Format: Semicolon-delimited, comma decimal separator (parse also accepts
+ *   comma-delimited variants via delimiter sniffing; detect stays strict)
 * Columns vary but typically: Bokföringsdag, Valutadag, Verifikationsnummer,
 *   Text/mottagare, Belopp, Saldo
 * Date format: YYYY-MM-DD
@@ -11,6 +12,7 @@
 import type { BankFileFormat, BankFileParseResult, ParsedBankTransaction, BankFileParseIssue } from '../types'
 import { prepareContent } from '../../shared/encoding'
 import { normalizeDate } from '../date-utils'
+import { parseCSVLine } from './nordea'

 function parseCommaDecimal(value: string): number {
  const cleaned = value.replace(/\s/g, '').replace(',', '.')
@@ -43,12 +45,25 @@ export const sebFormat: BankFileFormat = {
    const issues: BankFileParseIssue[] = []
    let skippedRows = 0

-    // Parse header to find column indices
+    // Parse header to find column indices. SEB normally exports
+    // semicolon-delimited files, but some export surfaces use commas: sniff
+    // the delimiter from the header line instead of assuming ';'.
    const headerLine = lines[0] || ''
-    const headers = headerLine.split(';').map((h) => h.trim().toLowerCase().replace(/"/g, ''))
+    const semicolons = (headerLine.match(/;/g) || []).length
+    const commas = (headerLine.match(/,/g) || []).length
+    const delimiter = commas > semicolons ? ',' : ';'
+    const headers = parseCSVLine(headerLine, delimiter).map((h) =>
+      h.trim().toLowerCase().replace(/"/g, '')
+    )

    // Find column indices dynamically
-    const dateIdx = headers.findIndex((h) => /bokf(ö|o)ringsda(g|tum)/.test(h))
+    let dateIdx = headers.findIndex((h) => /bokf(ö|o)ringsda(g|tum)/.test(h))
+    if (dateIdx === -1) {
+      // Lowest-priority tier: a bare "Datum" column. Only honored here in
+      // parse (an explicit user choice), never in detect, so this profile
+      // cannot steal files from other bank profiles during auto-detection.
+      dateIdx = headers.findIndex((h) => h === 'datum')
+    }
    const descIdx = headers.findIndex(
      (h) => h.includes('text') || h.includes('mottagare') || h.includes('beskrivning')
    )
@@ -58,7 +73,7 @@ export const sebFormat: BankFileFormat = {
    if (dateIdx === -1 || amountIdx === -1) {
      issues.push({
        row: 1,
-        message: 'Could not identify required columns (date, amount)',
+        message: 'Kunde inte identifiera nödvändiga kolumner (datum, belopp)',
        severity: 'error',
      })
      return {
@@ -76,7 +91,7 @@ export const sebFormat: BankFileFormat = {
      const line = lines[i].trim()
      if (!line) continue

-      const fields = line.split(';').map((f) => f.trim().replace(/^"|"$/g, ''))
+      const fields = parseCSVLine(line, delimiter).map((f) => f.trim().replace(/^"|"$/g, ''))

      const date = fields[dateIdx]
      const description = fields[descIdx >= 0 ? descIdx : dateIdx + 1] || 'Unknown'
@@ -84,21 +99,21 @@ export const sebFormat: BankFileFormat = {
      const balanceStr = balanceIdx >= 0 ? fields[balanceIdx] : undefined

      if (!date || !amountStr) {
-        issues.push({ row: i + 1, message: 'Missing required fields', severity: 'warning' })
+        issues.push({ row: i + 1, message: 'Obligatoriska fält saknas', severity: 'warning' })
        skippedRows++
        continue
      }

      const amount = parseCommaDecimal(amountStr)
      if (isNaN(amount)) {
-        issues.push({ row: i + 1, message: `Invalid amount: ${amountStr}`, severity: 'warning' })
+        issues.push({ row: i + 1, message: `Ogiltigt belopp: ${amountStr}`, severity: 'warning' })
        skippedRows++
        continue
      }

      const normalizedDate = normalizeDate(date)
      if (!normalizedDate) {
-        issues.push({ row: i + 1, message: `Invalid date: ${date}`, severity: 'warning' })
+        issues.push({ row: i + 1, message: `Ogiltigt datum: ${date}`, severity: 'warning' })
        skippedRows++
        continue
      }
@@ -97,10 +97,42 @@ export function parseBankFile(
        transactions: [],
        date_from: null,
        date_to: null,
-        issues: [{ row: 0, message: `Unknown format: ${formatId}`, severity: 'error' }],
+        issues: [{ row: 0, message: `Okänt format: ${formatId}`, severity: 'error' }],
        stats: { total_rows: 0, parsed_rows: 0, skipped_rows: 0, total_income: 0, total_expenses: 0 },
      }
    }
+
+    const explicitResult = format.parse(content)
+    if (explicitResult.transactions.length > 0 || formatId === 'generic_csv') {
+      // A working explicit parse is never overridden. generic_csv is also
+      // exempt: it is the manual column-mapping escape hatch and its default
+      // mapping legitimately parses 0 rows before the user maps columns.
+      return explicitResult
+    }
+
+    // The explicit choice parsed nothing: fall back to auto-detection so an
+    // explicitly selected bank is never WORSE than "Automatisk identifiering".
+    // detectFileFormat can never return generic_csv (its detect() is always
+    // false), so this cannot reroute the UI into the mapping flow.
+    const detected = detectFileFormat(content, filename)
+    if (detected && detected.id !== formatId) {
+      const detectedResult = detected.parse(content)
+      if (detectedResult.transactions.length > 0) {
+        return {
+          ...detectedResult,
+          issues: [
+            {
+              row: 0,
+              message: `Filen matchade inte det valda formatet (${format.name}) och tolkades istället som ${detected.name}.`,
+              severity: 'info',
+            },
+            ...detectedResult.issues,
+          ],
+        }
+      }
+    }
+
+    return explicitResult
  } else {
    format = detectFileFormat(content, filename) || undefined
    if (!format) {
@@ -49,7 +49,7 @@ export interface BankFileDuplicateInfo {
 export interface BankFileParseIssue {
  row: number
  message: string
-  severity: 'warning' | 'error'
+  severity: 'info' | 'warning' | 'error'
 }

 /** Supported bank file format identifiers */
@@ -5,6 +5,7 @@ import {
  hasEncodingIssues,
  recoverStringWithFFFD,
  recoverWordWithFFFD,
+  stripBOM,
 } from '../encoding'

 describe('decodeStringContent', () => {
@@ -78,6 +79,49 @@ describe('decodeFileContent', () => {
    const cp1252 = buf([0x47, 0xd6, 0x54, 0x45, 0x42, 0x4f, 0x52, 0x47])
    expect(decodeFileContent(cp1252)).toBe('GÖTEBORG')
  })
+
+  it('strips a UTF-8 BOM (EF BB BF) and decodes the remainder as UTF-8', () => {
+    const payload = Array.from(new TextEncoder().encode('Bokföringsdag;Belopp'))
+    const text = decodeFileContent(buf([0xef, 0xbb, 0xbf, ...payload]))
+    expect(text).toBe('Bokföringsdag;Belopp')
+    expect(text.charCodeAt(0)).not.toBe(0xfeff)
+  })
+
+  it('never produces a mojibake BOM prefix when a BOM-ed file needs the Windows-1252 fallback', () => {
+    // 'Datum;' + 0xD6 (Ö in Windows-1252, invalid as a lone UTF-8 byte),
+    // behind a UTF-8 BOM. The BOM bytes must never re-enter the fallback
+    // decode, so the result starts with 'Datum', not the literal mojibake.
+    const bytes = buf([0xef, 0xbb, 0xbf, 0x44, 0x61, 0x74, 0x75, 0x6d, 0x3b, 0xd6])
+    const text = decodeFileContent(bytes)
+    expect(text.startsWith('ï»¿')).toBe(false)
+    expect(text).toBe('Datum;Ö')
+  })
+
+  it('decodes UTF-16LE content behind a FF FE BOM', () => {
+    // 'Öre' in UTF-16LE: d6 00, 72 00, 65 00
+    const bytes = buf([0xff, 0xfe, 0xd6, 0x00, 0x72, 0x00, 0x65, 0x00])
+    expect(decodeFileContent(bytes)).toBe('Öre')
+  })
+
+  it('decodes UTF-16BE content behind a FE FF BOM', () => {
+    const bytes = buf([0xfe, 0xff, 0x00, 0xd6, 0x00, 0x72, 0x00, 0x65])
+    expect(decodeFileContent(bytes)).toBe('Öre')
+  })
+})
+
+describe('stripBOM', () => {
+  it('strips a leading U+FEFF', () => {
+    expect(stripBOM('\uFEFF' + 'Datum;Belopp')).toBe('Datum;Belopp')
+  })
+
+  it('strips a leading literal mojibake BOM (ï»¿)', () => {
+    expect(stripBOM('ï»¿Datum;Belopp')).toBe('Datum;Belopp')
+  })
+
+  it('is a no-op on clean content and never strips mid-string', () => {
+    expect(stripBOM('Datum;Belopp')).toBe('Datum;Belopp')
+    expect(stripBOM('Datum;ï»¿Belopp')).toBe('Datum;ï»¿Belopp')
+  })
 })

 // --- U+FFFD heuristic recovery ---
@@ -7,12 +7,38 @@
 */

 /**
- * Decode file content, handling both UTF-8 and Windows-1252 encodings.
+ * Decode file content, handling BOMs plus UTF-8 and Windows-1252 encodings.
 *
- * Strategy: Try UTF-8 first. If the result contains replacement characters
- * (U+FFFD) or garbled Swedish chars, fall back to Windows-1252.
+ * Strategy: inspect the leading BYTES first.
+ * - FF FE / FE FF: UTF-16LE/BE (e.g. Excel "Unicode text" re-saves). Decode
+ *   as such; TextDecoder consumes the BOM itself.
+ * - EF BB BF (UTF-8 BOM): strip the three bytes and decode the REMAINDER.
+ *   If the remainder still contains U+FFFD (invalid UTF-8 somewhere in the
+ *   file), decode the remainder as Windows-1252. The BOM bytes are never
+ *   re-included, so the fallback can no longer produce a literal mojibake
+ *   "ï»¿" prefix that breaks exact-match header detection.
+ * - No BOM: try UTF-8; fall back to Windows-1252 on replacement characters
+ *   or garbled Swedish chars (unchanged behavior).
 */
 export function decodeFileContent(buffer: ArrayBuffer): string {
+  const bytes = new Uint8Array(buffer)
+
+  if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
+    return new TextDecoder('utf-16le', { fatal: false }).decode(buffer)
+  }
+  if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
+    return new TextDecoder('utf-16be', { fatal: false }).decode(buffer)
+  }
+
+  if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
+    const rest = bytes.subarray(3)
+    const utf8Rest = new TextDecoder('utf-8', { fatal: false }).decode(rest)
+    if (!utf8Rest.includes('�')) {
+      return utf8Rest
+    }
+    return new TextDecoder('windows-1252', { fatal: false }).decode(rest)
+  }
+
  const utf8Decoder = new TextDecoder('utf-8', { fatal: false })
  const utf8Result = utf8Decoder.decode(buffer)

@@ -72,12 +98,19 @@ export function normalizeLineEndings(content: string): string {
 }

 /**
- * Strip BOM (Byte Order Mark) from start of content
+ * Strip BOM (Byte Order Mark) from start of content.
+ *
+ * Also strips the literal mojibake form "ï»¿" (U+00EF U+00BB U+00BF): a
+ * UTF-8 BOM whose bytes were decoded as Windows-1252/Latin-1 upstream.
+ * Covers string-entry paths that were pre-decoded outside decodeFileContent.
 */
 export function stripBOM(content: string): string {
  if (content.charCodeAt(0) === 0xfeff) {
    return content.slice(1)
  }
+  if (content.startsWith('ï»¿')) {
+    return content.slice(3)
+  }
  return content
 }