From 3ea03c0fe1472e4b656659d997ebdaad56d35b00 Mon Sep 17 00:00:00 2001 From: Jakob Wennberg Date: Tue, 18 Aug 2026 16:22:14 +0200 Subject: [PATCH] fix(import): stop the generic CSV mapper picking a time column as description (#1689) A Lunar 2026 export (Date, Time, Title, Amount, Balance, Transaction ID) that reached the manual "Annan CSV" mapping was seeded with Time as the description: no description keyword matched Title, and the positional fallback took the first non-numeric, non-date column, which is the clock time sitting between Date and Title. - suggestColumnMapping: add title / titel to the description keywords; exclude clock-time columns from every description pass, by header label (Time, Tid, Tidpunkt, Klockslag, Transaktionstid, ...) and by HH:MM / HH:MM:SS values, so header-less files are covered too. Last resort still seeds something the user can correct. - Lunar detector: sniff the delimiter (comma, semicolon, tab) instead of refusing any file containing a semicolon, so a re-saved or localized copy of the same English header set is parsed by the dedicated parser and never reaches the mapping flow. Header cells are matched exactly (date, title|text, amount, balance), the same resolution parse() uses, which also stops substring hits like Update/Context from claiming a file. - Mapping UI header-row detection: add title / balance to the keyword list for English exports. Regression tests: Lunar-style header through the generic path maps Title, a header-less Time column is skipped by value, Datum;Tid;Titel maps Titel, semicolon- and tab-delimited 2026 Lunar files detect and parse, Swedish and non-Lunar English headers are not claimed. All 7 fail without the fix. Closes #1671 Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- DECISIONS.md | 1 + .../import/BankFileColumnMappingStep.tsx | 2 + lib/import/bank-file/__tests__/parser.test.ts | 50 ++++++++++++ .../__tests__/suggest-column-mapping.test.ts | 79 +++++++++++++++++++ lib/import/bank-file/formats/generic-csv.ts | 59 +++++++++++--- lib/import/bank-file/formats/lunar.ts | 70 ++++++++++++---- 6 files changed, 234 insertions(+), 27 deletions(-) diff --git a/DECISIONS.md b/DECISIONS.md index aac5fee4..91255ba3 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -1050,3 +1050,4 @@ One line per decision: `[YYYY-MM-DD] : `. Appended by agents and [2026-08-18] Shopify webshop_orders port: vat_breakdown is reconstructed from the ORDER-LEVEL taxLines (net = tax / rate, remainder as a 0%-bucket, refuse on missing rates or overshoot) instead of summing line items like the WooCommerce sync: Shopify's discountedTotalSet excludes cart-level discount allocations and lineItems is a paginated connection, so part-summing can silently produce a wrong per-rate net, while tax-per-rate and the charged total are authoritative order-level facts. Refund VAT is always prorated from the parent's mix (Shopify's Refund object exposes no per-rate tax without paging refundLineItems per refund). [2026-08-18] Shopify order feed keeps its paid-only qualification (PAID/PARTIALLY_REFUNDED/REFUNDED) after the webshop_orders port, unlike WooCommerce which also imports unpaid orders for the invoice flow: widening qualification is a product decision, out of scope for the port; unpaid orders re-surface via updatedAt when payment captures. The line-item snapshot is stored only when the parts reconstruct the charged total to the ore (else [] and the invoice conversion falls back to one aggregate line), and the bookkeeping-lock row filter was dropped: an Orders-page row behind the lock is an overview row, not permanent inbox noise, and booking is still blocked by the lock triggers (parity with WooCommerce). [2026-08-18] Skattekontoutdrag sum mismatch (opening + events != closing) demoted from a hard 400 to a preview confirm gate showing ingående/händelser/utgående/differens, mirroring the orgnr-mismatch gate: Sebastian's real export was refused on it (2026-08-18) with no way forward and no figures to diagnose; nothing is booked at import and dedup makes a later complete re-import safe, so refusing the file only blocked the rows that WERE readable. Parser also takes the earliest opening / latest closing across several marker pairs, reads a marker saldo from a trailing running-saldo column, and accepts U+2212 / plus-sign amounts; the route logs the figures (amounts and counts, never row text) so the next report is diagnosable from Vercel logs. Kept the hard reject only for zero readable rows. +[2026-08-18] Generic CSV mapping (#1671): description guess now excludes clock-time columns (Time/Tid/Klockslag by label, HH:MM by values) and knows Lunar's Title/Titel label; Lunar detect() sniffs comma/semicolon/tab and matches header CELLS exactly (date, title|text, amount, balance) instead of substrings, aligned with what parse() resolves on. NOT changed: the 2026-08-13 generic_csv exemption from the parsed-0-rows auto-detect fallback stays; lifting it would route an explicit "Annan CSV" pick into a dedicated parser and remove the manual escape hatch. Not verified against the customer's actual file (Gmail thread not readable in-session): the semicolon/tab widening is the plausible detection miss, a Swedish-localized Lunar header is not confirmed to exist and was not special-cased in the Lunar detector (the generic path now maps it correctly anyway). diff --git a/components/import/BankFileColumnMappingStep.tsx b/components/import/BankFileColumnMappingStep.tsx index 12eca6fe..178c24b0 100644 --- a/components/import/BankFileColumnMappingStep.tsx +++ b/components/import/BankFileColumnMappingStep.tsx @@ -41,6 +41,8 @@ const HEADER_KEYWORDS = [ 'amount', 'description', 'date', + 'title', + 'balance', ] interface BankFileColumnMappingStepProps { diff --git a/lib/import/bank-file/__tests__/parser.test.ts b/lib/import/bank-file/__tests__/parser.test.ts index 12d646d4..b91e02ed 100644 --- a/lib/import/bank-file/__tests__/parser.test.ts +++ b/lib/import/bank-file/__tests__/parser.test.ts @@ -1325,6 +1325,56 @@ describe('parseBankFile: Lunar format', () => { expect(result.date_to).toBe('2026-06-30') }) + // Issue #1671: the same 2026 header set delimited by semicolon or tab (a + // spreadsheet re-save, a localized copy) used to fall through to the manual + // mapping flow, where Time was picked as the description. The detector now + // sniffs the delimiter, so the dedicated parser handles these files. + const LUNAR_CSV_2026_SEMICOLON = [ + 'Date;Time;Title;Amount;Balance;Transaction ID', + '2026-06-30;12:11;Incoming payment;12 345,00;98 764,94;7f0a4c9e-1111-2222-3333-444455556666', + '2026-06-12;05:47;Fee;-1,49;86 419,94;7f0a4c9e-1111-2222-3333-444455557777', + '2026-05-12;05:47;Card purchase;"-2 500,00";"86 421,43";7f0a4c9e-1111-2222-3333-444455558888', + ].join('\n') + + const LUNAR_CSV_2026_TAB = '\uFEFF' + [ + 'Date\tTime\tTitle\tAmount\tBalance\tTransaction ID', + '2026-06-30\t12:11\tIncoming payment\t12 345,00\t98 764,94\t7f0a4c9e-1111-2222-3333-444455556666', + '2026-06-12\t05:47\tFee\t-1,49\t86 419,94\t7f0a4c9e-1111-2222-3333-444455557777', + ].join('\n') + + it('REGRESSION (#1671): detects and parses a semicolon-delimited 2026 Lunar export', () => { + expect(detectFileFormat(LUNAR_CSV_2026_SEMICOLON, 'lunar.csv')!.id).toBe('lunar') + + const result = parseBankFile(LUNAR_CSV_2026_SEMICOLON, 'lunar.csv') + expect(result.format).toBe('lunar') + expect(result.transactions).toHaveLength(3) + expect(result.issues).toHaveLength(0) + expect(result.transactions.map((t) => t.description)).toEqual(['Incoming payment', 'Fee', 'Card purchase']) + expect(result.transactions.map((t) => t.amount)).toEqual([12345, -1.49, -2500]) + expect(result.transactions[2].balance).toBe(86421.43) + expect(result.transactions[0].date).toBe('2026-06-30') + }) + + it('REGRESSION (#1671): detects and parses a tab-delimited 2026 Lunar export', () => { + expect(detectFileFormat(LUNAR_CSV_2026_TAB, 'lunar.csv')!.id).toBe('lunar') + + const result = parseBankFile(LUNAR_CSV_2026_TAB, 'lunar.csv') + expect(result.format).toBe('lunar') + expect(result.transactions).toHaveLength(2) + expect(result.transactions.map((t) => t.description)).toEqual(['Incoming payment', 'Fee']) + expect(result.transactions.map((t) => t.amount)).toEqual([12345, -1.49]) + }) + + it('does not claim a Swedish-header semicolon file or an English file without the Lunar column set', () => { + const lunar = getFormat('lunar')! + // Swedish labels: not Lunar, whatever the delimiter + expect(lunar.detect('Datum;Text;Belopp;Saldo\n2024-01-15;SPOTIFY;-99,00;100,00', 'x.csv')).toBe(false) + // "Balance" only as part of another label, no title/text cell: not Lunar + expect(lunar.detect('Date,Description,Amount,Running Balance\n2024-01-15,SPOTIFY,-99.00,100.00', 'x.csv')).toBe(false) + // Substring hits inside other words are not the Lunar header set + expect(lunar.detect('Update,Context,Amounts,Balances\n1,2,3,4', 'x.csv')).toBe(false) + }) + it('still parses the legacy Lunar period thousands separator ("1.234,56")', () => { const legacy = [ 'Date,Text,Amount,Balance', diff --git a/lib/import/bank-file/__tests__/suggest-column-mapping.test.ts b/lib/import/bank-file/__tests__/suggest-column-mapping.test.ts index 11165032..e986fee4 100644 --- a/lib/import/bank-file/__tests__/suggest-column-mapping.test.ts +++ b/lib/import/bank-file/__tests__/suggest-column-mapping.test.ts @@ -177,6 +177,85 @@ describe('suggestColumnMapping', () => { expect(result.balance).toBe(3) }) + // Regression tests for issue #1671: a Lunar 2026 export routed through the + // manual mapping (detection missed: semicolon/tab copy) seeded `Time` as the + // description because no label keyword matched `Title` and the positional + // fallback took the first non-numeric, non-date column. + it('REGRESSION (#1671): Lunar-style Date,Time,Title header maps Title, not Time, as description', () => { + const headers = ['Date', 'Time', 'Title', 'Amount', 'Balance', 'Transaction ID'] + const dataRows = [ + ['2026-06-30', '12:11', 'Incoming payment', '12 345,00', '98 764,94', '7f0a4c9e-1111-2222-3333-444455556666'], + ['2026-06-12', '05:47', 'Fee', '-1,49', '86 419,94', '7f0a4c9e-1111-2222-3333-444455557777'], + ['2026-05-12', '05:47', 'Card purchase', '-2 500,00', '86 421,43', '7f0a4c9e-1111-2222-3333-444455558888'], + ] + + const result = suggestColumnMapping(headers, dataRows) + + expect(result.date).toBe(0) + expect(result.description).toBe(2) // Title, NOT 1 (Time) + expect(result.amount).toBe(3) + expect(result.balance).toBe(4) + }) + + it('REGRESSION (#1671): skips a Time column by its HH:MM values when there is no header row', () => { + const dataRows = [ + ['2026-06-30', '12:11', 'Incoming payment', '12 345,00', '98 764,94'], + ['2026-06-12', '05:47', 'Fee', '-1,49', '86 419,94'], + ['2026-05-12', '05:47:03', 'Card purchase', '-2 500,00', '86 421,43'], + ] + + const result = suggestColumnMapping(null, dataRows) + + expect(result.date).toBe(0) + expect(result.description).toBe(2) // the text column, NOT 1 (clock time) + expect(result.amount).toBe(3) + expect(result.balance).toBe(4) + }) + + it('maps a Swedish Datum;Tid;Titel layout to Titel and never to Tid', () => { + const headers = ['Datum', 'Tid', 'Titel', 'Belopp', 'Saldo'] + const dataRows = [ + ['2026-06-30', '12:11', 'Inbetalning', '12 345,00', '98 764,94'], + ['2026-06-12', '05:47', 'Avgift', '-1,49', '86 419,94'], + ] + + const result = suggestColumnMapping(headers, dataRows) + + expect(result.date).toBe(0) + expect(result.description).toBe(2) // Titel + expect(result.amount).toBe(3) + expect(result.balance).toBe(4) + }) + + it('skips a time-labelled column in the positional fallback even when no description keyword matches', () => { + // 'Notering' is not a description keyword, so the label pass misses and + // the fallback must step over Transaktionstid (time by label AND values). + const headers = ['Datum', 'Transaktionstid', 'Notering', 'Belopp'] + const dataRows = [ + ['2026-06-30', '12:11', 'Hyra juni', '-9 500,00'], + ['2026-06-12', '05:47', 'Swish', '250,00'], + ] + + const result = suggestColumnMapping(headers, dataRows) + + expect(result.description).toBe(2) // Notering, NOT 1 (Transaktionstid) + }) + + it('still seeds a description as a last resort when only a time column remains', () => { + // Nothing better exists: the UI must still get a value the user can change. + const headers = ['Datum', 'Tid', 'Belopp'] + const dataRows = [ + ['2026-06-30', '12:11', '-9 500,00'], + ['2026-06-12', '05:47', '250,00'], + ] + + const result = suggestColumnMapping(headers, dataRows) + + expect(result.date).toBe(0) + expect(result.amount).toBe(2) + expect(result.description).toBe(1) + }) + it('returns all -1 for empty input', () => { expect(suggestColumnMapping(null, [])).toEqual({ date: -1, description: -1, amount: -1, balance: -1 }) }) diff --git a/lib/import/bank-file/formats/generic-csv.ts b/lib/import/bank-file/formats/generic-csv.ts index 591f203e..643920f9 100644 --- a/lib/import/bank-file/formats/generic-csv.ts +++ b/lib/import/bank-file/formats/generic-csv.ts @@ -257,11 +257,31 @@ function pickDateHeader(headers: string[]): number { return headers.findIndex((h) => (h.includes('datum') || h.includes('date')) && !h.includes('valuta')) } -/** Pick the best description column by header label. */ +/** + * Header labels that name a clock time, not a description: Lunar's 2026 export + * carries `Time` next to `Date`, and Swedish exports spell it Tid / Tidpunkt / + * Klockslag. Anchored at the end so compounds like Transaktionstid and + * "Transaction time" match too. A time-of-day column is never a description, + * whatever else the heuristics fail to resolve. + */ +const TIME_HEADER_RE = /(^|[^a-zåäö])(tidpunkt|klockslag|klocka|hour|hours|timestamp)$|tid$|time$/ + +/** A cell that looks like a clock time (HH:MM or HH:MM:SS). */ +const TIME_VALUE_RE = /^\d{1,2}:\d{2}(:\d{2})?$/ + +/** + * Pick the best description column by header label. + * + * `title` / `titel` are the Lunar 2026 export's description column; without + * them the label pass missed and the positional fallback grabbed the Time + * column sitting between Date and Title (issue #1671). Time-ish labels are + * excluded outright so a keyword like `text` can never land on + * "Transaktionstid" either. + */ function pickDescriptionHeader(headers: string[]): number { - const keywords = ['text', 'beskrivning', 'description', 'rubrik', 'meddelande', 'referens', 'mottagare', 'namn'] + const keywords = ['text', 'beskrivning', 'description', 'title', 'titel', 'rubrik', 'meddelande', 'referens', 'mottagare', 'namn'] for (const kw of keywords) { - const idx = headers.findIndex((h) => h === kw || h.includes(kw)) + const idx = headers.findIndex((h) => (h === kw || h.includes(kw)) && !TIME_HEADER_RE.test(h)) if (idx >= 0) return idx } return -1 @@ -269,13 +289,14 @@ function pickDescriptionHeader(headers: string[]): number { /** Per-column value statistics across the sampled data rows. */ function analyzeColumns(dataRows: string[][], colCount: number) { - const acc = Array.from({ length: colCount }, () => ({ numeric: 0, date: 0, negative: 0, nonEmpty: 0 })) + const acc = Array.from({ length: colCount }, () => ({ numeric: 0, date: 0, time: 0, negative: 0, nonEmpty: 0 })) for (const row of dataRows.slice(0, 20)) { for (let i = 0; i < colCount; i++) { const raw = (row[i] ?? '').trim() if (!raw) continue acc[i].nonEmpty++ if (SUGGEST_DATE_PATTERNS.some((re) => re.test(raw))) acc[i].date++ + if (TIME_VALUE_RE.test(raw)) acc[i].time++ const cleaned = normalizeMinusSign(raw).replace(/\s/g, '') if (/^-?\d+([.,]\d+)?$/.test(cleaned)) { acc[i].numeric++ @@ -285,6 +306,7 @@ function analyzeColumns(dataRows: string[][], colCount: number) { } return acc.map((s) => ({ isDate: s.nonEmpty > 0 && s.date / s.nonEmpty >= 0.5, + isTime: s.nonEmpty > 0 && s.time / s.nonEmpty >= 0.5, isNumeric: s.nonEmpty > 0 && s.numeric / s.nonEmpty >= 0.5, hasNegative: s.negative > 0, })) @@ -357,15 +379,26 @@ export function suggestColumnMapping( } if (result.description === -1) { - result.description = stats.findIndex( - (s, i) => i !== result.date && i !== result.amount && i !== result.balance && !s.isNumeric && !s.isDate - ) - if (result.description === -1) { - for (let i = 0; i < colCount; i++) { - if (i !== result.date && i !== result.amount && i !== result.balance) { - result.description = i - break - } + // A clock-time column (Lunar's `Time`, a Swedish `Tid`) is text-shaped to + // the numeric/date tests, so it must be excluded explicitly: by header + // label when there is one, and by HH:MM values either way. Otherwise the + // positional fallback picks it as the description whenever it sits before + // the real text column (issue #1671). + const hdr = headers?.map((h) => h.trim().toLowerCase().replace(/"/g, '')) ?? [] + const isTimeColumn = (i: number) => stats[i].isTime || (hdr[i] !== undefined && TIME_HEADER_RE.test(hdr[i])) + const unassigned = (i: number) => i !== result.date && i !== result.amount && i !== result.balance + const passes: Array<(i: number) => boolean> = [ + (i) => unassigned(i) && !stats[i].isNumeric && !stats[i].isDate && !isTimeColumn(i), + (i) => unassigned(i) && !isTimeColumn(i), + // Last resort: anything not already assigned, so the UI still seeds a + // value the user can correct. + unassigned, + ] + for (const pass of passes) { + const idx = stats.findIndex((_s, i) => pass(i)) + if (idx >= 0) { + result.description = idx + break } } } diff --git a/lib/import/bank-file/formats/lunar.ts b/lib/import/bank-file/formats/lunar.ts index 6dec67cd..adf4ec13 100644 --- a/lib/import/bank-file/formats/lunar.ts +++ b/lib/import/bank-file/formats/lunar.ts @@ -11,6 +11,12 @@ * - English headers distinguish Lunar from Nordea (Swedish headers) * - Amounts use comma as decimal separator but are quoted since the file * delimiter is also comma + * - The delimiter is sniffed from the header line: the documented export is + * comma-delimited, but a semicolon- or tab-delimited copy of the same + * header set (a spreadsheet re-save, a localized export) carries the same + * distinctive English columns and must not fall through to the manual + * mapping flow, where the Time column used to be picked as the description + * (issue #1671) * - Thousand separator is a space in the 2026 export (e.g. "12 345,00"); * legacy exports used a period (e.g. "1.234,56"). Both are handled. */ @@ -20,6 +26,46 @@ import { prepareContent } from '../../shared/encoding' import { normalizeDate } from '../date-utils' import { parseCSVLine } from './nordea' +const LUNAR_DELIMITERS = [',', ';', '\t'] + +/** + * Sniff the field delimiter from the header line. Comma is the documented + * Lunar export and wins ties; semicolon and tab are accepted when they split + * the header into more cells. + */ +function sniffLunarDelimiter(headerLine: string): string { + let best = ',' + let bestCount = parseCSVLine(headerLine, ',').length + for (const delimiter of LUNAR_DELIMITERS.slice(1)) { + const count = parseCSVLine(headerLine, delimiter).length + if (count > bestCount) { + best = delimiter + bestCount = count + } + } + return best +} + +function parseLunarHeader(headerLine: string, delimiter: string): string[] { + return parseCSVLine(headerLine, delimiter).map((h) => h.trim().toLowerCase().replace(/"/g, '')) +} + +/** + * The Lunar header set: date, a description column ("title" in the 2026 + * export, "text" in the legacy one), amount and balance, matched as whole + * cells so a Swedish "Datum" export or an English file with e.g. "Running + * balance" is never claimed. Exact cells are also what parse() resolves on, + * so detect() can never accept a header parse() then rejects. + */ +function isLunarHeader(cells: string[]): boolean { + return ( + cells.includes('date') && + (cells.includes('title') || cells.includes('text')) && + cells.includes('amount') && + cells.includes('balance') + ) +} + function parseLunarAmount(value: string): number { // Lunar format: "12 345,00" (2026, space thousands) or "1.234,56" (legacy, // period thousands). Strip all whitespace (including NBSP U+00A0 and narrow @@ -39,17 +85,12 @@ export const lunarFormat: BankFileFormat = { detect(content: string, _filename: string): boolean { const prepared = prepareContent(content) - const firstLine = prepared.split('\n')[0]?.toLowerCase() || '' - // Lunar: comma-delimited with English headers - // Must NOT contain semicolons, must have "date", "amount", "balance" and a - // description column: "title" (2026 export) or "text" (legacy export) - return ( - !firstLine.includes(';') && - firstLine.includes('date') && - (firstLine.includes('title') || firstLine.includes('text')) && - firstLine.includes('amount') && - firstLine.includes('balance') - ) + const firstLine = prepared.split('\n')[0] || '' + // Lunar: English headers "date", "amount", "balance" and a description + // column: "title" (2026 export) or "text" (legacy export). Delimiter is + // sniffed (comma, semicolon or tab); the Swedish-header banks are all + // checked before this format, so the English set is what distinguishes it. + return isLunarHeader(parseLunarHeader(firstLine, sniffLunarDelimiter(firstLine))) }, parse(content: string): BankFileParseResult { @@ -60,9 +101,10 @@ export const lunarFormat: BankFileFormat = { const issues: BankFileParseIssue[] = [] let skippedRows = 0 - // Parse header + // Parse header; the delimiter is sniffed from it and reused for every row const headerLine = lines[0] || '' - const headers = parseCSVLine(headerLine, ',').map((h) => h.trim().toLowerCase().replace(/"/g, '')) + const delimiter = sniffLunarDelimiter(headerLine) + const headers = parseLunarHeader(headerLine, delimiter) const dateIdx = headers.findIndex((h) => h === 'date') // "title" is the 2026 export's description column; "text" is the legacy one @@ -92,7 +134,7 @@ export const lunarFormat: BankFileFormat = { const line = lines[i].trim() if (!line) continue - const fields = parseCSVLine(line, ',').map((f) => f.trim().replace(/^"|"$/g, '')) + const fields = parseCSVLine(line, delimiter).map((f) => f.trim().replace(/^"|"$/g, '')) const date = fields[dateIdx] const description = descIdx >= 0 ? fields[descIdx] : 'Unknown'