From 566ed72984c7ded96f0d37b41b3dfeb8432fe717 Mon Sep 17 00:00:00 2001 From: Mattsson <111893710+mattssonn@users.noreply.github.com> Date: Wed, 20 May 2026 10:08:32 +0200 Subject: [PATCH] Bug/mcp connection issue (#541) * feat(api): implement caching and logging in health check endpoint - Added in-memory caching for health check responses to reduce load on Postgres. - Introduced logging for error handling in health check. - Updated response structure to exclude error details from public responses. feat(api): enhance OAuth consent UI and scope handling - Improved consent UI to reflect exact requested scopes and added better user guidance. - Updated scope handling logic to ensure least-privilege access. - Enhanced styling for better user experience and accessibility. chore(docker): improve security and resource management in Docker setup - Updated Docker Compose configuration to enforce read-only file systems and resource limits. - Added health checks and logging options for better observability. - Introduced optional Caddy reverse proxy for TLS termination. fix(migrations): resolve ambiguity in create_company_with_owner function - Dropped orphaned 3-arg overload of create_company_with_owner function. - Recreated canonical 4-arg version with cash account seeding logic. - Ensured proper permissions for function execution in Postgres. * feat: enhance security checks for team membership in company creation * test: add CSP tests for OAuth authorization endpoint * feat: enhance error handling and reporting in bank file import process --- app/api/import/bank-file/execute/route.ts | 19 +- .../authorize/__tests__/route.test.ts | 185 ++++++++++++++++++ app/api/mcp-oauth/authorize/route.ts | 44 +++-- .../[companyId]/imports/bank/route.ts | 3 +- .../import/BankFileColumnMappingStep.tsx | 92 +++++++-- components/import/BankFileResultStep.tsx | 12 ++ components/import/BankFileUploadStep.tsx | 8 + lib/import/bank-file/__tests__/parser.test.ts | 158 ++++++++++++++- lib/import/bank-file/formats/generic-csv.ts | 23 ++- lib/import/bank-file/formats/northmill.ts | 165 ++++++++++++++++ lib/import/bank-file/parser.ts | 2 + lib/import/bank-file/types.ts | 1 + lib/transactions/ingest.ts | 8 + types/index.ts | 2 + 14 files changed, 674 insertions(+), 48 deletions(-) create mode 100644 app/api/mcp-oauth/authorize/__tests__/route.test.ts create mode 100644 lib/import/bank-file/formats/northmill.ts diff --git a/app/api/import/bank-file/execute/route.ts b/app/api/import/bank-file/execute/route.ts index 1914b598..c6365a53 100644 --- a/app/api/import/bank-file/execute/route.ts +++ b/app/api/import/bank-file/execute/route.ts @@ -99,6 +99,21 @@ export const POST = withRouteContext( if (role === 'viewer') ingestOptions.rawInsertOnly = true const ingestResult = await ingestTransactions(supabase, companyId, user.id, rawTransactions, ingestOptions) + if (ingestResult.errors > 0 && ingestResult.first_error) { + opLog.error('bank file ingest reported insert errors', new Error(ingestResult.first_error.message), { + errorCount: ingestResult.errors, + code: ingestResult.first_error.code, + details: ingestResult.first_error.details, + hint: ingestResult.first_error.hint, + }) + } + + const errorMessage = ingestResult.errors > 0 + ? ingestResult.first_error + ? `${ingestResult.errors} fel: ${ingestResult.first_error.message}${ingestResult.first_error.details ? ` (${ingestResult.first_error.details})` : ''}` + : `${ingestResult.errors} transactions failed to import` + : null + await supabase .from('bank_file_imports') .update({ @@ -106,9 +121,7 @@ export const POST = withRouteContext( duplicate_count: ingestResult.duplicates, matched_count: ingestResult.auto_matched_invoices, status: ingestResult.errors > 0 && ingestResult.imported === 0 ? 'failed' : 'completed', - error_message: ingestResult.errors > 0 - ? `${ingestResult.errors} transactions failed to import` - : null, + error_message: errorMessage, }) .eq('id', importRecord.id) diff --git a/app/api/mcp-oauth/authorize/__tests__/route.test.ts b/app/api/mcp-oauth/authorize/__tests__/route.test.ts new file mode 100644 index 00000000..c008c414 --- /dev/null +++ b/app/api/mcp-oauth/authorize/__tests__/route.test.ts @@ -0,0 +1,185 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +const mocks = vi.hoisted(() => ({ + createClient: vi.fn(), + isAllowedRedirectUri: vi.fn(), + requireCompanyId: vi.fn(), + getBranding: vi.fn(), +})) + +vi.mock('@/lib/supabase/server', () => ({ + createClient: () => mocks.createClient(), +})) + +vi.mock('@/lib/auth/oauth-allowlist', () => ({ + isAllowedRedirectUri: (...args: unknown[]) => mocks.isAllowedRedirectUri(...args), +})) + +vi.mock('@/lib/company/context', () => ({ + requireCompanyId: (...args: unknown[]) => mocks.requireCompanyId(...args), +})) + +vi.mock('@/lib/branding/service', () => ({ + getBranding: () => mocks.getBranding(), +})) + +import { GET } from '../route' + +function buildAuthorizeUrl(params: Record): string { + const url = new URL('http://localhost/api/mcp-oauth/authorize') + Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v)) + return url.toString() +} + +function buildSupabase(user: { id: string } | null, companyName = 'Test AB') { + return { + auth: { getUser: vi.fn().mockResolvedValue({ data: { user }, error: null }) }, + from: vi.fn().mockReturnValue({ + select: vi.fn().mockReturnValue({ + eq: vi.fn().mockReturnValue({ + single: vi.fn().mockResolvedValue({ + data: { company_name: companyName }, + error: null, + }), + }), + }), + }), + } +} + +describe('GET /api/mcp-oauth/authorize — CSP', () => { + beforeEach(() => { + vi.clearAllMocks() + process.env.SUPABASE_SERVICE_ROLE_KEY = 'test-service-key' + mocks.createClient.mockResolvedValue(buildSupabase({ id: 'user-1' })) + mocks.isAllowedRedirectUri.mockResolvedValue(true) + mocks.requireCompanyId.mockResolvedValue('company-1') + mocks.getBranding.mockReturnValue({ appName: 'gnubok' }) + }) + + it("form-action includes the redirect_uri origin so the post-consent redirect isn't blocked", async () => { + // Regression: the consent form POSTs same-origin, but the server's 303 + // response redirects to the client callback. CSP form-action re-checks + // every hop in the chain, so 'self' alone blocks the post-consent step. + const request = new Request( + buildAuthorizeUrl({ + response_type: 'code', + redirect_uri: 'https://claude.ai/api/mcp/auth_callback', + code_challenge: 'abc', + code_challenge_method: 'S256', + scope: 'mcp', + state: 'xyz', + }) + ) + const response = await GET(request) + expect(response.status).toBe(200) + + const csp = response.headers.get('Content-Security-Policy') + expect(csp).toBeTruthy() + expect(csp).toMatch(/form-action 'self' https:\/\/claude\.ai(;|$)/) + // 'self' is preserved so the same-origin POST still works. + expect(csp).toContain("form-action 'self'") + }) + + it('form-action uses the redirect origin only (no path/query leakage)', async () => { + const request = new Request( + buildAuthorizeUrl({ + response_type: 'code', + redirect_uri: 'https://claude.com/api/oauth/callback?env=prod', + code_challenge: 'abc', + code_challenge_method: 'S256', + scope: 'mcp', + }) + ) + const response = await GET(request) + expect(response.status).toBe(200) + + const csp = response.headers.get('Content-Security-Policy') ?? '' + expect(csp).toContain('https://claude.com') + // Origin only — no path, no query string in the source expression. + expect(csp).not.toContain('/api/oauth/callback') + expect(csp).not.toContain('env=prod') + }) + + it('renders both read and write rows when client passes only the legacy `mcp` scope marker', async () => { + // Claude's connector sends scope=mcp today. The consent UI must render + // every scope group so the user can opt into :write grants — only the + // :read rows are pre-checked (Art. 25(2) data-protection-by-default). + // Regression: commit 04c097c2 hid all write rows by clamping the + // ceiling to DEFAULT_OAUTH_SCOPES. + const request = new Request( + buildAuthorizeUrl({ + response_type: 'code', + redirect_uri: 'https://claude.ai/api/mcp/auth_callback', + code_challenge: 'abc', + code_challenge_method: 'S256', + scope: 'mcp', + }) + ) + const response = await GET(request) + expect(response.status).toBe(200) + const html = await response.text() + + // Write scopes must render as checkboxes (un-pre-checked). + expect(html).toMatch(/value="transactions:write"/) + expect(html).toMatch(/value="bookkeeping:write"/) + expect(html).toMatch(/value="invoices:write"/) + expect(html).toMatch(/value="pending_operations:approve"/) + + // The :write checkbox must NOT be pre-checked when the client passed + // no explicit scope request — the user has to opt in deliberately. + const writeRow = html.match( + /]*value="transactions:write"[^>]*>/ + )?.[0] + expect(writeRow).toBeDefined() + expect(writeRow!).not.toContain('checked') + + // The :read counterpart must still be pre-checked (safe default). + const readRow = html.match( + /]*value="transactions:read"[^>]*>/ + )?.[0] + expect(readRow).toBeDefined() + expect(readRow!).toContain('checked') + }) + + it('renders only the requested scopes when the client passes them explicitly', async () => { + // RFC 6749 §3.3 strict least-privilege: an explicit `scope=` shrinks the + // ceiling, so a client that asked for read-only cannot have a write box + // surface at consent time. + const request = new Request( + buildAuthorizeUrl({ + response_type: 'code', + redirect_uri: 'https://claude.ai/api/mcp/auth_callback', + code_challenge: 'abc', + code_challenge_method: 'S256', + scope: 'transactions:read invoices:read', + }) + ) + const response = await GET(request) + expect(response.status).toBe(200) + const html = await response.text() + + expect(html).toContain('value="transactions:read"') + expect(html).toContain('value="invoices:read"') + expect(html).not.toContain('value="transactions:write"') + expect(html).not.toContain('value="bookkeeping:write"') + }) + + it('rejects disallowed redirect_uri before any CSP would be emitted', async () => { + mocks.isAllowedRedirectUri.mockResolvedValue(false) + const request = new Request( + buildAuthorizeUrl({ + response_type: 'code', + redirect_uri: 'https://evil.example/cb', + code_challenge: 'abc', + code_challenge_method: 'S256', + scope: 'mcp', + }) + ) + const response = await GET(request) + expect(response.status).toBe(400) + // Important: the form-action whitelist must never be populated from an + // untrusted origin. A 400 here keeps the allowlist as the single source + // of truth for which origins can land at this endpoint. + }) +}) diff --git a/app/api/mcp-oauth/authorize/route.ts b/app/api/mcp-oauth/authorize/route.ts index b7c6fe98..21a8a0f1 100644 --- a/app/api/mcp-oauth/authorize/route.ts +++ b/app/api/mcp-oauth/authorize/route.ts @@ -200,19 +200,19 @@ export async function GET(request: Request) { const scopeBindingValue = scopeParam ?? '' const scopeBindingSignature = signScopeBinding(scopeBindingValue) - // The consent UI is bounded to a server-enforced ceiling, regardless of - // what the user ticks: + // Two-level model for the consent UI: // - // - Client requested specific scopes → ceiling = that set (RFC 6749 §3.3 - // strict least-privilege). - // - Client passed no scope (or only the legacy `mcp` marker — the case - // Claude's connector hits today) → ceiling = DEFAULT_OAUTH_SCOPES - // (read-only). This preserves GDPR Art. 25(2) data-protection-by-default - // and keeps a server-enforced read-only guarantee for clients that - // never declared any intent. Widening the ceiling beyond - // DEFAULT_OAUTH_SCOPES requires the client to ask for it via the - // `scope` parameter. - const grantCeiling = new Set(parsed.scopes ?? DEFAULT_OAUTH_SCOPES) + // - Client requested specific scopes → ceiling = that set, pre-checked = + // that set (RFC 6749 §3.3 strict least-privilege). + // - Client passed no scope (or only the legacy `mcp` marker — Claude's + // connector today) → ceiling = ALL_SCOPES so every read/write row + // renders; pre-checked = DEFAULT_OAUTH_SCOPES so only the read rows + // start ticked. The user has to actively tick :write to widen the + // grant. This preserves GDPR Art. 25(2) (defaults are minimal / + // read-only) while still letting the resource owner authorise write + // scopes per RFC 6749 §3.3 ("based on … the resource owner's + // instructions") — which is the whole point of the consent step. + const grantCeiling = new Set(parsed.scopes ?? ALL_SCOPES) const preChecked = new Set(parsed.scopes ?? DEFAULT_OAUTH_SCOPES) const scopeCheckboxesHtml = renderScopeCheckboxes(preChecked, grantCeiling) @@ -574,11 +574,18 @@ export async function GET(request: Request) { // script-src bound to the per-request nonce ensures the consent page's // inline JS can only be the block we actually emitted. Anything injected // by a forged response or persisted XSS would be blocked. + // + // form-action must include the redirect_uri origin: the POST handler + // returns a 303 to the OAuth client's callback (e.g. claude.ai), and CSP + // form-action re-checks every hop in the redirect chain. With only 'self' + // the browser would block the post-consent redirect. The origin is safe + // to whitelist here because isAllowedRedirectUri() already gated it above. + const redirectOrigin = new URL(redirectUri).origin const csp = [ "default-src 'none'", `script-src 'nonce-${cspNonce}'`, "style-src 'unsafe-inline'", - "form-action 'self'", + `form-action 'self' ${redirectOrigin}`, "base-uri 'none'", "frame-ancestors 'none'", ].join('; ') @@ -677,13 +684,14 @@ export async function POST(request: Request) { // can never end up with write grants, even if the user tampered // with the form (least-privilege, SOC 2 CC6.3, NIST AC-6). // • If the client passed no scope (or only the `mcp` marker), the - // ceiling = DEFAULT_OAUTH_SCOPES (read-only). A client that never - // declared write intent cannot receive write grants, even if the - // user tampered with the form — preserving GDPR Art. 25(2) - // data-protection-by-default. + // ceiling = ALL_SCOPES. The resource owner has full discretion at + // consent time, which RFC 6749 §3.3 permits ("based on … the + // resource owner's instructions"). The silent fallback when the + // user selects nothing remains DEFAULT_OAUTH_SCOPES (read-only), + // preserving GDPR Art. 25(2) data-protection-by-default. const submittedScopes = formData.getAll('scopes').filter((s): s is string => typeof s === 'string') const validated = validateScopes(submittedScopes) - const clientCeiling: ApiKeyScope[] = parsed.scopes ?? [...DEFAULT_OAUTH_SCOPES] + const clientCeiling: ApiKeyScope[] = parsed.scopes ?? [...ALL_SCOPES] const ceilingSet = new Set(clientCeiling) const boundedToClient = (validated ?? []).filter(s => ceilingSet.has(s)) const grantedScopes: ApiKeyScope[] = boundedToClient.length > 0 diff --git a/app/api/v1/companies/[companyId]/imports/bank/route.ts b/app/api/v1/companies/[companyId]/imports/bank/route.ts index 628d37c3..55a88576 100644 --- a/app/api/v1/companies/[companyId]/imports/bank/route.ts +++ b/app/api/v1/companies/[companyId]/imports/bank/route.ts @@ -60,7 +60,7 @@ registerEndpoint({ 'SIE bookkeeping import (use /imports/sie). Auto-bank sync (use the enable-banking extension). Single-transaction creation (use POST /transactions/ingest with a 1-element array).', pitfalls: [ 'File size cap: 10 MB. Larger files require splitting client-side.', - '`format` query parameter is optional; auto-detection works for all supported banks. Pass `format` only to force a specific format. Accepted values: seb, swedbank, handelsbanken, nordea, nordea_business, lansforsakringar, ica_banken, skandia, lunar, generic_csv, camt053.', + '`format` query parameter is optional; auto-detection works for all supported banks. Pass `format` only to force a specific format. Accepted values: seb, swedbank, handelsbanken, nordea, nordea_business, lansforsakringar, ica_banken, skandia, lunar, northmill, generic_csv, camt053.', 'Duplicate detection is by external_id (composed from date + amount + counterparty); a re-import of the same file with the same flag set typically deduplicates rather than creating doubles.', 'BFL 5 kap 6-7 §§ note: this endpoint creates `transactions` rows (the underlag for a verifikation), NOT verifikationer themselves. The verifikation content requirements are in BFL 5 kap 6-7 §§; until each transaction is matched to an invoice/supplier-invoice (POST /transactions/{id}/match-*) or categorised (POST /transactions/{id}/categorize), the bookkeeping obligation isn\'t discharged. A successful import here means the data is ingested — not booked.', 'A successful import returns operation_id; poll /operations/{id} for the final ingested/duplicates/errors counts.', @@ -134,6 +134,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>( 'ica_banken', 'skandia', 'lunar', + 'northmill', 'generic_csv', 'camt053', ]) diff --git a/components/import/BankFileColumnMappingStep.tsx b/components/import/BankFileColumnMappingStep.tsx index 3de47639..4fd545d4 100644 --- a/components/import/BankFileColumnMappingStep.tsx +++ b/components/import/BankFileColumnMappingStep.tsx @@ -22,9 +22,27 @@ import { } from '@/components/ui/table' import { ArrowLeft, ArrowRight, Columns3 } from 'lucide-react' import { formatCurrency } from '@/lib/utils' -import { getCSVPreview } from '@/lib/import/bank-file/formats/generic-csv' +import { getCSVPreview, normalizeMinusSign } from '@/lib/import/bank-file/formats/generic-csv' import type { GenericCSVColumnMapping } from '@/lib/import/bank-file/types' +const HEADER_KEYWORDS = [ + 'datum', + 'bokföringsdag', + 'bokforingsdag', + 'transaktionsdatum', + 'reskontradatum', + 'beskrivning', + 'belopp', + 'transaktion', + 'text', + 'mottagare', + 'saldo', + 'valuta', + 'amount', + 'description', + 'date', +] + interface BankFileColumnMappingStepProps { rawFileContent: string onConfirm: (mapping: GenericCSVColumnMapping) => void @@ -59,39 +77,70 @@ export default function BankFileColumnMappingStep({ const [decimalSep, setDecimalSep] = useState<',' | '.'>(',') const [dateFormat, setDateFormat] = useState('YYYY-MM-DD') - // Re-parse headers and preview whenever delimiter or file content changes + // Re-parse headers and preview whenever delimiter or file content changes. + // Pull a generous slice (30 rows) so we can scan past metadata preambles like + // Northmill's 5-line Kontonummer/Saldo/Kontohavare/Org.Nr/Period header. const parsedRows = useMemo( - () => getCSVPreview(rawFileContent, delimiter, 10), + () => getCSVPreview(rawFileContent, delimiter, 30), [rawFileContent, delimiter] ) - // Auto-detect whether the first row is a header: if any cell on row 0 looks - // like a date (YYYY-MM-DD, DD.MM.YYYY, DD/MM/YYYY, YYYYMMDD), it's data, not a header. - // Users can still override via the switch. + // Auto-detect the header row index by scanning for a row whose cells + // contain known column-name keywords (bokföringsdag, beskrivning, belopp, …). + // A row needs ≥ 2 keyword hits to qualify, which excludes metadata rows where + // only the label cell happens to match (e.g. "Saldo,251495,41,SEK"). + // Falls back to row 0 if nothing qualifies — preserves prior behavior for + // simple files where the header truly is the first row. const DATE_PATTERNS = [/^\d{4}-\d{2}-\d{2}$/, /^\d{2}[./]\d{2}[./]\d{4}$/, /^\d{8}$/] + const detectedHeaderRow = useMemo(() => { + let best: { idx: number; score: number; cells: number } | null = null + for (let i = 0; i < Math.min(parsedRows.length, 20); i++) { + const row = parsedRows[i] + if (!row || row.length < 2) continue + const hits = row.filter((cell) => { + const c = cell.trim().toLowerCase() + return c.length > 0 && HEADER_KEYWORDS.some((kw) => c === kw || c.includes(kw)) + }).length + if (hits < 2) continue + if (!best || hits > best.score || (hits === best.score && row.length > best.cells)) { + best = { idx: i, score: hits, cells: row.length } + } + } + return best?.idx ?? 0 + }, [parsedRows]) + + // Detect whether the file actually has a header row at all: if the auto-detect + // landed on row 0 but row 0 already looks like data (any cell is a date), assume + // no header. Otherwise trust the detection. const detectedHasHeader = useMemo(() => { - const firstRow = parsedRows[0] - if (!firstRow) return true - const hasDateCell = firstRow.some((cell) => + const headerRow = parsedRows[detectedHeaderRow] + if (!headerRow) return true + const looksLikeData = headerRow.some((cell) => DATE_PATTERNS.some((re) => re.test(cell.trim())) ) - return !hasDateCell - }, [parsedRows]) + return !looksLikeData + }, [parsedRows, detectedHeaderRow]) const [hasHeaderOverride, setHasHeaderOverride] = useState(null) const hasHeader = hasHeaderOverride ?? detectedHasHeader + // skip_rows = number of rows to skip before transaction data starts. + // When hasHeader: skip past the header row (detectedHeaderRow + 1). + // When no header: skip nothing — data starts at row 0. + const skipRows = hasHeader ? detectedHeaderRow + 1 : 0 + const columnHeaders = useMemo(() => { - if (hasHeader && parsedRows[0]) return parsedRows[0] + if (hasHeader && parsedRows[detectedHeaderRow]) return parsedRows[detectedHeaderRow] const count = parsedRows[0]?.length ?? 0 return Array.from({ length: count }, (_, i) => `Kolumn ${i + 1}`) - }, [parsedRows, hasHeader]) + }, [parsedRows, hasHeader, detectedHeaderRow]) - const dataRows = hasHeader ? parsedRows.slice(1) : parsedRows + const dataRows = hasHeader ? parsedRows.slice(detectedHeaderRow + 1) : parsedRows // Auto-guess date/description/amount columns from the first data row. // Only used as initial defaults — user can override any pick. - const AMOUNT_RE = /^-?\d+([.,]\d+)?$/ + // Match ASCII and Unicode minus; some banks (e.g. Northmill) use U+2212. + const AMOUNT_RE = /^[-\u2212\u2013\u2014\u2010]?\d+([.,]\d+)?$/ useEffect(() => { if (dateCol !== -1 || descCol !== -1 || amountCol !== -1) return const sample = dataRows[0] @@ -124,7 +173,7 @@ export default function BankFileColumnMappingStep({ ...(balanceCol >= 0 && { balance: balanceCol }), delimiter, decimal_separator: decimalSep, - skip_rows: hasHeader ? 1 : 0, + skip_rows: skipRows, date_format: dateFormat, } onConfirm(mapping) @@ -150,7 +199,9 @@ export default function BankFileColumnMappingStep({

- Slå av om filen saknar rubrikrad och första raden redan innehåller transaktionsdata. + {hasHeader && detectedHeaderRow > 0 + ? `Hoppar över ${detectedHeaderRow} metadatarader. Rubrikraden upptäcktes på rad ${detectedHeaderRow + 1}.` + : 'Slå av om filen saknar rubrikrad och första raden redan innehåller transaktionsdata.'}

@@ -350,11 +401,12 @@ export default function BankFileColumnMappingStep({ - {dataRows.slice(0, 5).map((row, i) => { + {dataRows.slice(0, 5).map((row, i) => { const amountStr = row[amountCol] || '0' + const normalizedAmountStr = normalizeMinusSign(amountStr) const amount = decimalSep === ',' - ? parseFloat(amountStr.replace(/\s/g, '').replace(',', '.')) - : parseFloat(amountStr.replace(/\s/g, '')) + ? parseFloat(normalizedAmountStr.replace(/\s/g, '').replace(',', '.')) + : parseFloat(normalizedAmountStr.replace(/\s/g, '')) return ( diff --git a/components/import/BankFileResultStep.tsx b/components/import/BankFileResultStep.tsx index 96aeb405..f641b5c5 100644 --- a/components/import/BankFileResultStep.tsx +++ b/components/import/BankFileResultStep.tsx @@ -46,6 +46,18 @@ export default function BankFileResultStep({ : `${result.errors} fel uppstod under importen.`} + {!isSuccess && result.first_error && ( + +
+

Databasfel

+

+ {result.first_error.message} + {result.first_error.details ? ` — ${result.first_error.details}` : ''} + {result.first_error.code ? ` (${result.first_error.code})` : ''} +

+
+ + )} {/* Next steps */} diff --git a/components/import/BankFileUploadStep.tsx b/components/import/BankFileUploadStep.tsx index 8a4459b7..ed1167c6 100644 --- a/components/import/BankFileUploadStep.tsx +++ b/components/import/BankFileUploadStep.tsx @@ -31,6 +31,7 @@ const FORMAT_NAMES: Record = { ica_banken: 'ICA Banken', skandia: 'Skandia', lunar: 'Lunar', + northmill: 'Northmill', generic_csv: 'CSV (manuell mappning)', camt053: 'ISO 20022 camt.053', } @@ -131,6 +132,7 @@ export default function BankFileUploadStep({ ICA Banken Skandia Lunar + Northmill ISO 20022 camt.053 (XML) Annan CSV (manuell mappning) @@ -263,6 +265,12 @@ export default function BankFileUploadStep({ Logga in → Konto → Transaktioner → Exportera (CSV)

+
+

Northmill

+

+ Logga in → Konto → Kontoutdrag → Ladda ner (CSV) +

+
diff --git a/lib/import/bank-file/__tests__/parser.test.ts b/lib/import/bank-file/__tests__/parser.test.ts index 8438e430..b042da11 100644 --- a/lib/import/bank-file/__tests__/parser.test.ts +++ b/lib/import/bank-file/__tests__/parser.test.ts @@ -8,7 +8,7 @@ import { detectFileFormat, parseBankFile, generateExternalId, generateFileHash, getFormat, getAllFormats } from '../parser' import type { ParsedBankTransaction, BankFileFormatId } from '../types' -import { parseGenericCSV } from '../formats/generic-csv' +import { parseGenericCSV, normalizeMinusSign } from '../formats/generic-csv' import { parseCSVLine } from '../formats/nordea' // --------------------------------------------------------------------------- @@ -171,6 +171,25 @@ const LUNAR_CSV = [ '2024-01-13,LÖNEUTBETALNING,"25.000,00","12.877,17"', ].join('\n') +// Northmill exports include a 5-line metadata preamble (Kontonummer, Saldo, +// Kontohavare, Org. Nr, Period) plus blank lines before the actual transaction +// header. Negative amounts use Unicode minus (U+2212), not ASCII hyphen. +const NORTHMILL_CSV = [ + 'Kontonummer,9750-8770139', + 'Saldo,"251495,41",SEK', + 'Kontohavare,Arcim Technology AB', + 'Org. Nr,559538-6219', + 'Period,2025-10-01,2026-04-07', + '', + '', + '', + 'Bokföringsdag,Beskrivning,Belopp,Saldo,Valuta', + '2026-04-01,Månadsavgift företagspaket april,"\u2212139,00","251495,41",SEK', + '2026-01-22,200176580348155,"\u22121000,00","251912,41",SEK', + '2025-10-16,092221155575,"400000,00","422005,00",SEK', + '2025-10-01,Inbetalning av aktiekapital,"25000,00","25000,00",SEK', +].join('\n') + const UNKNOWN_CSV = [ 'id,name,value,timestamp', '1,Widget A,100,2024-01-15T10:00:00', @@ -357,6 +376,22 @@ describe('detectFileFormat', () => { expect(format!.id).toBe('lunar') }) + it('detects Northmill CSV from Kontonummer preamble + transaction header', () => { + const format = detectFileFormat(NORTHMILL_CSV, 'Northmill-Account-Statement.csv') + expect(format).not.toBeNull() + expect(format!.id).toBe('northmill') + }) + + it('does not detect Northmill on a file that just happens to mention Kontonummer in transactions', () => { + const fake = [ + 'Datum,Transaktion,Kategori,Belopp,Saldo', + '2024-01-15,Överföring kontonummer 1234,Inkomst,"100,00","1000,00"', + ].join('\n') + const format = detectFileFormat(fake, 'test.csv') + // Should detect as Nordea, not Northmill — Northmill needs Kontonummer at start of first line + expect(format!.id).toBe('nordea') + }) + it('returns null for unrecognized CSV content', () => { const format = detectFileFormat(UNKNOWN_CSV, 'data.csv') expect(format).toBeNull() @@ -1054,6 +1089,53 @@ describe('parseBankFile — Lunar format', () => { }) }) +describe('parseBankFile — Northmill format', () => { + it('skips the 5-line metadata preamble and blank lines, parses transaction rows', () => { + const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv') + + expect(result.format).toBe('northmill') + expect(result.format_name).toBe('Northmill') + expect(result.transactions).toHaveLength(4) + expect(result.issues).toHaveLength(0) + }) + + it('correctly parses negative amounts that use Unicode minus (U+2212)', () => { + const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv') + + // First transaction is "−139,00" with U+2212 — must become -139, not NaN + expect(result.transactions[0].amount).toBe(-139) + expect(result.transactions[0].description).toBe('Månadsavgift företagspaket april') + expect(result.transactions[0].date).toBe('2026-04-01') + + expect(result.transactions[1].amount).toBe(-1000) + expect(result.transactions[2].amount).toBe(400000) + expect(result.transactions[3].amount).toBe(25000) + }) + + it('extracts the saldo (running balance) column', () => { + const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv') + + expect(result.transactions[0].balance).toBe(251495.41) + expect(result.transactions[3].balance).toBe(25000) + }) + + it('calculates income vs expenses correctly with Unicode minus amounts', () => { + const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv') + + expect(result.stats.total_income).toBe(425000) + expect(result.stats.total_expenses).toBe(-1139) + expect(result.stats.parsed_rows).toBe(4) + expect(result.stats.skipped_rows).toBe(0) + }) + + it('extracts the correct date range from the transactions, not the Period metadata row', () => { + const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv') + + expect(result.date_from).toBe('2025-10-01') + expect(result.date_to).toBe('2026-04-01') + }) +}) + describe('parseBankFile — camt.053 XML format', () => { it('parses XML with credit and debit entries', () => { const result = parseBankFile(CAMT053_XML, 'statement.xml') @@ -1716,6 +1798,80 @@ describe('parseGenericCSV — column bounds checking', () => { expect(result.stats.skipped_rows).toBe(0) expect(result.issues).toHaveLength(0) }) + + it('parses amounts that use Unicode minus (U+2212) — was NaN before normalization', () => { + const content = [ + 'Date,Description,Amount', + '2024-01-15,SPOTIFY,"\u221299,00"', + '2024-01-16,REFUND,"\u201350,00"', + '2024-01-17,SALARY,"25000,00"', + ].join('\n') + + const result = parseGenericCSV(content, { + date: 0, + description: 1, + amount: 2, + delimiter: ',', + decimal_separator: ',', + skip_rows: 1, + date_format: 'YYYY-MM-DD', + }) + + expect(result.transactions).toHaveLength(3) + expect(result.transactions[0].amount).toBe(-99) + expect(result.transactions[1].amount).toBe(-50) + expect(result.transactions[2].amount).toBe(25000) + }) + + it('skips metadata rows when skip_rows is set higher than 1 (multi-row preamble)', () => { + // Mimics a file with 5 metadata rows + header + 2 transactions. + const content = [ + 'Account,12345', + 'Owner,Acme AB', + 'Period,2024-01,2024-12', + 'Currency,SEK', + 'Type,Statement', + 'Date,Description,Amount', + '2024-01-15,SPOTIFY,-99.00', + '2024-01-16,SALARY,25000.00', + ].join('\n') + + const result = parseGenericCSV(content, { + date: 0, + description: 1, + amount: 2, + delimiter: ',', + decimal_separator: '.', + skip_rows: 6, + date_format: 'YYYY-MM-DD', + }) + + expect(result.transactions).toHaveLength(2) + expect(result.transactions[0].amount).toBe(-99) + expect(result.transactions[1].amount).toBe(25000) + expect(result.issues).toHaveLength(0) + }) +}) + +describe('normalizeMinusSign', () => { + it('replaces U+2212 (minus sign) with ASCII hyphen', () => { + expect(normalizeMinusSign('\u2212139,00')).toBe('-139,00') + }) + + it('replaces U+2013 (en dash) and U+2014 (em dash) with ASCII hyphen', () => { + expect(normalizeMinusSign('\u2013100')).toBe('-100') + expect(normalizeMinusSign('\u2014250')).toBe('-250') + }) + + it('leaves ASCII hyphen and digits untouched', () => { + expect(normalizeMinusSign('-139.00')).toBe('-139.00') + expect(normalizeMinusSign('139.00')).toBe('139.00') + }) + + it('makes parseFloat work on Unicode-minus strings (regression for Northmill)', () => { + expect(parseFloat('\u2212139.00')).toBeNaN() + expect(parseFloat(normalizeMinusSign('\u2212139.00'))).toBe(-139) + }) }) // --- Fix 8: parseCSVLine unclosed quote handling --- diff --git a/lib/import/bank-file/formats/generic-csv.ts b/lib/import/bank-file/formats/generic-csv.ts index dcaed87d..6c54861f 100644 --- a/lib/import/bank-file/formats/generic-csv.ts +++ b/lib/import/bank-file/formats/generic-csv.ts @@ -10,6 +10,15 @@ import { prepareContent } from '../../shared/encoding' import { parseCSVLine } from './nordea' import { normalizeDate } from '../date-utils' +/** + * Normalize Unicode minus variants (U+2212 "−", U+2013 "–", U+2014 "—", U+2010 "‐") + * to ASCII hyphen-minus so parseFloat can read them. Northmill and some other + * banks export negatives with U+2212; parseFloat returns NaN for those. + */ +export function normalizeMinusSign(value: string): string { + return value.replace(/[\u2212\u2013\u2014\u2010]/g, '-') +} + /** * Parse a generic CSV with user-provided column mapping */ @@ -89,12 +98,15 @@ export function parseGenericCSV( continue } - // Parse amount based on configured decimal separator + // Parse amount based on configured decimal separator. + // Normalize Unicode minus first — some banks (e.g. Northmill) use U+2212 + // instead of ASCII hyphen, which parseFloat treats as NaN. + const normalizedAmount = normalizeMinusSign(amountStr) let amount: number if (mapping.decimal_separator === ',') { - amount = parseFloat(amountStr.replace(/\s/g, '').replace(',', '.')) + amount = parseFloat(normalizedAmount.replace(/\s/g, '').replace(',', '.')) } else { - amount = parseFloat(amountStr.replace(/\s/g, '')) + amount = parseFloat(normalizedAmount.replace(/\s/g, '')) } if (isNaN(amount)) { @@ -113,10 +125,11 @@ export function parseGenericCSV( let balance: number | null = null if (balanceStr) { + const normalizedBalance = normalizeMinusSign(balanceStr) if (mapping.decimal_separator === ',') { - balance = parseFloat(balanceStr.replace(/\s/g, '').replace(',', '.')) + balance = parseFloat(normalizedBalance.replace(/\s/g, '').replace(',', '.')) } else { - balance = parseFloat(balanceStr.replace(/\s/g, '')) + balance = parseFloat(normalizedBalance.replace(/\s/g, '')) } if (isNaN(balance)) balance = null } diff --git a/lib/import/bank-file/formats/northmill.ts b/lib/import/bank-file/formats/northmill.ts new file mode 100644 index 00000000..85e2eed8 --- /dev/null +++ b/lib/import/bank-file/formats/northmill.ts @@ -0,0 +1,165 @@ +/** + * Northmill CSV format parser + * + * Format: Comma-delimited, comma decimal separator (amounts are quoted) + * Preamble: 5 metadata rows (Kontonummer, Saldo, Kontohavare, Org. Nr, Period) + * followed by blank lines before the transaction header. + * Header: Bokföringsdag,Beskrivning,Belopp,Saldo,Valuta + * Date format: YYYY-MM-DD + * Encoding: UTF-8 + * + * Notes: + * - Negative amounts use Unicode minus sign (U+2212 "−"), not ASCII hyphen. + * The parser normalizes the minus before calling parseFloat. + */ + +import type { BankFileFormat, BankFileParseResult, ParsedBankTransaction, BankFileParseIssue } from '../types' +import { prepareContent } from '../../shared/encoding' +import { normalizeDate } from '../date-utils' +import { parseCSVLine } from './nordea' +import { normalizeMinusSign } from './generic-csv' + +const HEADER_KEYWORDS = ['bokföringsdag', 'beskrivning', 'belopp'] + +function isNorthmillHeader(line: string): boolean { + const lower = line.toLowerCase() + if (lower.includes(';')) return false + return HEADER_KEYWORDS.every((kw) => lower.includes(kw)) +} + +function parseAmount(value: string): number { + // Northmill: "−139,00" (Unicode minus) or "400000,00" (no thousand separator on positives) + const cleaned = normalizeMinusSign(value).replace(/\s/g, '').replace(',', '.') + return parseFloat(cleaned) +} + +export const northmillFormat: BankFileFormat = { + id: 'northmill', + name: 'Northmill', + description: 'Northmill kontoutdrag (CSV med metadata-rader och Unicode-minus)', + fileExtensions: ['.csv', '.txt'], + + detect(content: string, _filename: string): boolean { + const prepared = prepareContent(content) + const lines = prepared.split('\n') + // Northmill preamble's first non-empty line is "Kontonummer," + const firstNonEmpty = lines.find((l) => l.trim() !== '') || '' + if (!/^kontonummer\s*,/i.test(firstNonEmpty.trim())) return false + // Also look for the transaction header within the first ~20 lines to be sure + return lines.slice(0, 20).some(isNorthmillHeader) + }, + + parse(content: string): BankFileParseResult { + const prepared = prepareContent(content) + const rawLines = prepared.split('\n') + + const transactions: ParsedBankTransaction[] = [] + const issues: BankFileParseIssue[] = [] + let skippedRows = 0 + + const headerIdx = rawLines.findIndex(isNorthmillHeader) + if (headerIdx === -1) { + return { + format: 'northmill', + format_name: 'Northmill', + transactions: [], + date_from: null, + date_to: null, + issues: [{ row: 0, message: 'Kunde inte hitta rubrikraden (Bokföringsdag, Beskrivning, Belopp).', severity: 'error' }], + stats: { total_rows: 0, parsed_rows: 0, skipped_rows: 0, total_income: 0, total_expenses: 0 }, + } + } + + const headers = parseCSVLine(rawLines[headerIdx], ',').map((h) => h.trim().toLowerCase().replace(/"/g, '')) + const dateIdx = headers.findIndex((h) => h.includes('bokföringsdag') || h.includes('bokforingsdag')) + const descIdx = headers.findIndex((h) => h.includes('beskrivning')) + const amountIdx = headers.findIndex((h) => h.includes('belopp')) + const balanceIdx = headers.findIndex((h) => h.includes('saldo')) + + if (dateIdx === -1 || descIdx === -1 || amountIdx === -1) { + issues.push({ + row: headerIdx + 1, + message: 'Rubrikraden saknar nödvändiga kolumner (Bokföringsdag, Beskrivning, Belopp).', + severity: 'error', + }) + return { + format: 'northmill', + format_name: 'Northmill', + transactions: [], + date_from: null, + date_to: null, + issues, + stats: { total_rows: 0, parsed_rows: 0, skipped_rows: 0, total_income: 0, total_expenses: 0 }, + } + } + + for (let i = headerIdx + 1; i < rawLines.length; i++) { + const line = rawLines[i].trim() + if (!line) continue + + const fields = parseCSVLine(line, ',').map((f) => f.trim().replace(/^"|"$/g, '')) + const dateStr = fields[dateIdx] + const description = fields[descIdx] || 'Unknown' + const amountStr = fields[amountIdx] + const balanceStr = balanceIdx >= 0 ? fields[balanceIdx] : undefined + + if (!dateStr || !amountStr) { + const missing: string[] = [] + if (!dateStr) missing.push('datum') + if (!amountStr) missing.push('belopp') + issues.push({ row: i + 1, message: `Saknar ${missing.join(' och ')}`, severity: 'warning' }) + skippedRows++ + continue + } + + const date = normalizeDate(dateStr) + if (!date) { + issues.push({ row: i + 1, message: `Ogiltigt datumformat: ${dateStr}`, severity: 'warning' }) + skippedRows++ + continue + } + + const amount = parseAmount(amountStr) + if (isNaN(amount)) { + issues.push({ row: i + 1, message: `Ogiltigt belopp: ${amountStr}`, severity: 'warning' }) + skippedRows++ + continue + } + + let balance: number | null = null + if (balanceStr) { + const b = parseAmount(balanceStr) + balance = isNaN(b) ? null : b + } + + transactions.push({ + date, + description: description.trim(), + amount, + currency: 'SEK', + balance, + reference: null, + counterparty: null, + raw_line: line, + }) + } + + const dates = transactions.map((t) => t.date).sort() + + return { + format: 'northmill', + format_name: 'Northmill', + transactions, + date_from: dates[0] || null, + date_to: dates[dates.length - 1] || null, + issues, + stats: { + total_rows: rawLines.length - headerIdx - 1, + parsed_rows: transactions.length, + skipped_rows: skippedRows, + total_income: Math.round(transactions.filter((t) => t.amount > 0).reduce((s, t) => s + t.amount, 0) * 100) / 100, + total_expenses: Math.round(transactions.filter((t) => t.amount < 0).reduce((s, t) => s + t.amount, 0) * 100) / 100, + }, + } + }, +} diff --git a/lib/import/bank-file/parser.ts b/lib/import/bank-file/parser.ts index fe04673d..cec9e63e 100644 --- a/lib/import/bank-file/parser.ts +++ b/lib/import/bank-file/parser.ts @@ -16,6 +16,7 @@ import { lansforsakringarFormat } from './formats/lansforsakringar' import { icaBankenFormat } from './formats/ica-banken' import { skandiaFormat } from './formats/skandia' import { lunarFormat } from './formats/lunar' +import { northmillFormat } from './formats/northmill' import { camt053Format } from './formats/camt053' import { genericCSVFormat } from './formats/generic-csv' @@ -36,6 +37,7 @@ const FORMATS: BankFileFormat[] = [ icaBankenFormat, skandiaFormat, lunarFormat, + northmillFormat, genericCSVFormat, ] diff --git a/lib/import/bank-file/types.ts b/lib/import/bank-file/types.ts index cac4b883..b913de42 100644 --- a/lib/import/bank-file/types.ts +++ b/lib/import/bank-file/types.ts @@ -52,6 +52,7 @@ export type BankFileFormatId = | 'ica_banken' | 'skandia' | 'lunar' + | 'northmill' | 'generic_csv' | 'camt053' diff --git a/lib/transactions/ingest.ts b/lib/transactions/ingest.ts index 073ae6d0..22a89f0e 100644 --- a/lib/transactions/ingest.ts +++ b/lib/transactions/ingest.ts @@ -263,6 +263,14 @@ export async function ingestTransactions( if (insertError || !newTransaction) { result.errors++ + if (!result.first_error && insertError) { + result.first_error = { + message: insertError.message, + code: insertError.code ?? null, + details: insertError.details ?? null, + hint: insertError.hint ?? null, + } + } continue } diff --git a/types/index.ts b/types/index.ts index 21bc25c3..9e33b440 100644 --- a/types/index.ts +++ b/types/index.ts @@ -2578,6 +2578,8 @@ export interface IngestResult { auto_matched_invoices: number errors: number transaction_ids: string[] + /** First insert error encountered, surfaced for debugging. Optional. */ + first_error?: { message: string; code?: string | null; details?: string | null; hint?: string | null } } // ── Invoice extraction (used by invoice-inbox extension and core utils) ──