Bug/mcp connection issue (#541)

* feat(api): implement caching and logging in health check endpoint

- Added in-memory caching for health check responses to reduce load on Postgres.
- Introduced logging for error handling in health check.
- Updated response structure to exclude error details from public responses.

feat(api): enhance OAuth consent UI and scope handling

- Improved consent UI to reflect exact requested scopes and added better user guidance.
- Updated scope handling logic to ensure least-privilege access.
- Enhanced styling for better user experience and accessibility.

chore(docker): improve security and resource management in Docker setup

- Updated Docker Compose configuration to enforce read-only file systems and resource limits.
- Added health checks and logging options for better observability.
- Introduced optional Caddy reverse proxy for TLS termination.

fix(migrations): resolve ambiguity in create_company_with_owner function

- Dropped orphaned 3-arg overload of create_company_with_owner function.
- Recreated canonical 4-arg version with cash account seeding logic.
- Ensured proper permissions for function execution in Postgres.

* feat: enhance security checks for team membership in company creation

* test: add CSP tests for OAuth authorization endpoint

* feat: enhance error handling and reporting in bank file import process
This commit is contained in:
Mattsson
2026-05-20 10:08:32 +02:00
committed by GitHub
parent 9aced4790c
commit 566ed72984
14 changed files with 674 additions and 48 deletions
+16 -3
View File
@@ -99,6 +99,21 @@ export const POST = withRouteContext(
if (role === 'viewer') ingestOptions.rawInsertOnly = true
const ingestResult = await ingestTransactions(supabase, companyId, user.id, rawTransactions, ingestOptions)
if (ingestResult.errors > 0 && ingestResult.first_error) {
opLog.error('bank file ingest reported insert errors', new Error(ingestResult.first_error.message), {
errorCount: ingestResult.errors,
code: ingestResult.first_error.code,
details: ingestResult.first_error.details,
hint: ingestResult.first_error.hint,
})
}
const errorMessage = ingestResult.errors > 0
? ingestResult.first_error
? `${ingestResult.errors} fel: ${ingestResult.first_error.message}${ingestResult.first_error.details ? ` (${ingestResult.first_error.details})` : ''}`
: `${ingestResult.errors} transactions failed to import`
: null
await supabase
.from('bank_file_imports')
.update({
@@ -106,9 +121,7 @@ export const POST = withRouteContext(
duplicate_count: ingestResult.duplicates,
matched_count: ingestResult.auto_matched_invoices,
status: ingestResult.errors > 0 && ingestResult.imported === 0 ? 'failed' : 'completed',
error_message: ingestResult.errors > 0
? `${ingestResult.errors} transactions failed to import`
: null,
error_message: errorMessage,
})
.eq('id', importRecord.id)
@@ -0,0 +1,185 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
const mocks = vi.hoisted(() => ({
createClient: vi.fn(),
isAllowedRedirectUri: vi.fn(),
requireCompanyId: vi.fn(),
getBranding: vi.fn(),
}))
vi.mock('@/lib/supabase/server', () => ({
createClient: () => mocks.createClient(),
}))
vi.mock('@/lib/auth/oauth-allowlist', () => ({
isAllowedRedirectUri: (...args: unknown[]) => mocks.isAllowedRedirectUri(...args),
}))
vi.mock('@/lib/company/context', () => ({
requireCompanyId: (...args: unknown[]) => mocks.requireCompanyId(...args),
}))
vi.mock('@/lib/branding/service', () => ({
getBranding: () => mocks.getBranding(),
}))
import { GET } from '../route'
function buildAuthorizeUrl(params: Record<string, string>): string {
const url = new URL('http://localhost/api/mcp-oauth/authorize')
Object.entries(params).forEach(([k, v]) => url.searchParams.set(k, v))
return url.toString()
}
function buildSupabase(user: { id: string } | null, companyName = 'Test AB') {
return {
auth: { getUser: vi.fn().mockResolvedValue({ data: { user }, error: null }) },
from: vi.fn().mockReturnValue({
select: vi.fn().mockReturnValue({
eq: vi.fn().mockReturnValue({
single: vi.fn().mockResolvedValue({
data: { company_name: companyName },
error: null,
}),
}),
}),
}),
}
}
describe('GET /api/mcp-oauth/authorize — CSP', () => {
beforeEach(() => {
vi.clearAllMocks()
process.env.SUPABASE_SERVICE_ROLE_KEY = 'test-service-key'
mocks.createClient.mockResolvedValue(buildSupabase({ id: 'user-1' }))
mocks.isAllowedRedirectUri.mockResolvedValue(true)
mocks.requireCompanyId.mockResolvedValue('company-1')
mocks.getBranding.mockReturnValue({ appName: 'gnubok' })
})
it("form-action includes the redirect_uri origin so the post-consent redirect isn't blocked", async () => {
// Regression: the consent form POSTs same-origin, but the server's 303
// response redirects to the client callback. CSP form-action re-checks
// every hop in the chain, so 'self' alone blocks the post-consent step.
const request = new Request(
buildAuthorizeUrl({
response_type: 'code',
redirect_uri: 'https://claude.ai/api/mcp/auth_callback',
code_challenge: 'abc',
code_challenge_method: 'S256',
scope: 'mcp',
state: 'xyz',
})
)
const response = await GET(request)
expect(response.status).toBe(200)
const csp = response.headers.get('Content-Security-Policy')
expect(csp).toBeTruthy()
expect(csp).toMatch(/form-action 'self' https:\/\/claude\.ai(;|$)/)
// 'self' is preserved so the same-origin POST still works.
expect(csp).toContain("form-action 'self'")
})
it('form-action uses the redirect origin only (no path/query leakage)', async () => {
const request = new Request(
buildAuthorizeUrl({
response_type: 'code',
redirect_uri: 'https://claude.com/api/oauth/callback?env=prod',
code_challenge: 'abc',
code_challenge_method: 'S256',
scope: 'mcp',
})
)
const response = await GET(request)
expect(response.status).toBe(200)
const csp = response.headers.get('Content-Security-Policy') ?? ''
expect(csp).toContain('https://claude.com')
// Origin only — no path, no query string in the source expression.
expect(csp).not.toContain('/api/oauth/callback')
expect(csp).not.toContain('env=prod')
})
it('renders both read and write rows when client passes only the legacy `mcp` scope marker', async () => {
// Claude's connector sends scope=mcp today. The consent UI must render
// every scope group so the user can opt into :write grants — only the
// :read rows are pre-checked (Art. 25(2) data-protection-by-default).
// Regression: commit 04c097c2 hid all write rows by clamping the
// ceiling to DEFAULT_OAUTH_SCOPES.
const request = new Request(
buildAuthorizeUrl({
response_type: 'code',
redirect_uri: 'https://claude.ai/api/mcp/auth_callback',
code_challenge: 'abc',
code_challenge_method: 'S256',
scope: 'mcp',
})
)
const response = await GET(request)
expect(response.status).toBe(200)
const html = await response.text()
// Write scopes must render as checkboxes (un-pre-checked).
expect(html).toMatch(/value="transactions:write"/)
expect(html).toMatch(/value="bookkeeping:write"/)
expect(html).toMatch(/value="invoices:write"/)
expect(html).toMatch(/value="pending_operations:approve"/)
// The :write checkbox must NOT be pre-checked when the client passed
// no explicit scope request — the user has to opt in deliberately.
const writeRow = html.match(
/<input[^>]*value="transactions:write"[^>]*>/
)?.[0]
expect(writeRow).toBeDefined()
expect(writeRow!).not.toContain('checked')
// The :read counterpart must still be pre-checked (safe default).
const readRow = html.match(
/<input[^>]*value="transactions:read"[^>]*>/
)?.[0]
expect(readRow).toBeDefined()
expect(readRow!).toContain('checked')
})
it('renders only the requested scopes when the client passes them explicitly', async () => {
// RFC 6749 §3.3 strict least-privilege: an explicit `scope=` shrinks the
// ceiling, so a client that asked for read-only cannot have a write box
// surface at consent time.
const request = new Request(
buildAuthorizeUrl({
response_type: 'code',
redirect_uri: 'https://claude.ai/api/mcp/auth_callback',
code_challenge: 'abc',
code_challenge_method: 'S256',
scope: 'transactions:read invoices:read',
})
)
const response = await GET(request)
expect(response.status).toBe(200)
const html = await response.text()
expect(html).toContain('value="transactions:read"')
expect(html).toContain('value="invoices:read"')
expect(html).not.toContain('value="transactions:write"')
expect(html).not.toContain('value="bookkeeping:write"')
})
it('rejects disallowed redirect_uri before any CSP would be emitted', async () => {
mocks.isAllowedRedirectUri.mockResolvedValue(false)
const request = new Request(
buildAuthorizeUrl({
response_type: 'code',
redirect_uri: 'https://evil.example/cb',
code_challenge: 'abc',
code_challenge_method: 'S256',
scope: 'mcp',
})
)
const response = await GET(request)
expect(response.status).toBe(400)
// Important: the form-action whitelist must never be populated from an
// untrusted origin. A 400 here keeps the allowlist as the single source
// of truth for which origins can land at this endpoint.
})
})
+26 -18
View File
@@ -200,19 +200,19 @@ export async function GET(request: Request) {
const scopeBindingValue = scopeParam ?? ''
const scopeBindingSignature = signScopeBinding(scopeBindingValue)
// The consent UI is bounded to a server-enforced ceiling, regardless of
// what the user ticks:
// Two-level model for the consent UI:
//
// - Client requested specific scopes → ceiling = that set (RFC 6749 §3.3
// strict least-privilege).
// - Client passed no scope (or only the legacy `mcp` marker — the case
// Claude's connector hits today) → ceiling = DEFAULT_OAUTH_SCOPES
// (read-only). This preserves GDPR Art. 25(2) data-protection-by-default
// and keeps a server-enforced read-only guarantee for clients that
// never declared any intent. Widening the ceiling beyond
// DEFAULT_OAUTH_SCOPES requires the client to ask for it via the
// `scope` parameter.
const grantCeiling = new Set<ApiKeyScope>(parsed.scopes ?? DEFAULT_OAUTH_SCOPES)
// - Client requested specific scopes → ceiling = that set, pre-checked =
// that set (RFC 6749 §3.3 strict least-privilege).
// - Client passed no scope (or only the legacy `mcp` marker — Claude's
// connector today) → ceiling = ALL_SCOPES so every read/write row
// renders; pre-checked = DEFAULT_OAUTH_SCOPES so only the read rows
// start ticked. The user has to actively tick :write to widen the
// grant. This preserves GDPR Art. 25(2) (defaults are minimal /
// read-only) while still letting the resource owner authorise write
// scopes per RFC 6749 §3.3 ("based on … the resource owner's
// instructions") — which is the whole point of the consent step.
const grantCeiling = new Set<ApiKeyScope>(parsed.scopes ?? ALL_SCOPES)
const preChecked = new Set<ApiKeyScope>(parsed.scopes ?? DEFAULT_OAUTH_SCOPES)
const scopeCheckboxesHtml = renderScopeCheckboxes(preChecked, grantCeiling)
@@ -574,11 +574,18 @@ export async function GET(request: Request) {
// script-src bound to the per-request nonce ensures the consent page's
// inline JS can only be the block we actually emitted. Anything injected
// by a forged response or persisted XSS would be blocked.
//
// form-action must include the redirect_uri origin: the POST handler
// returns a 303 to the OAuth client's callback (e.g. claude.ai), and CSP
// form-action re-checks every hop in the redirect chain. With only 'self'
// the browser would block the post-consent redirect. The origin is safe
// to whitelist here because isAllowedRedirectUri() already gated it above.
const redirectOrigin = new URL(redirectUri).origin
const csp = [
"default-src 'none'",
`script-src 'nonce-${cspNonce}'`,
"style-src 'unsafe-inline'",
"form-action 'self'",
`form-action 'self' ${redirectOrigin}`,
"base-uri 'none'",
"frame-ancestors 'none'",
].join('; ')
@@ -677,13 +684,14 @@ export async function POST(request: Request) {
// can never end up with write grants, even if the user tampered
// with the form (least-privilege, SOC 2 CC6.3, NIST AC-6).
// • If the client passed no scope (or only the `mcp` marker), the
// ceiling = DEFAULT_OAUTH_SCOPES (read-only). A client that never
// declared write intent cannot receive write grants, even if the
// user tampered with the form — preserving GDPR Art. 25(2)
// data-protection-by-default.
// ceiling = ALL_SCOPES. The resource owner has full discretion at
// consent time, which RFC 6749 §3.3 permits ("based on … the
// resource owner's instructions"). The silent fallback when the
// user selects nothing remains DEFAULT_OAUTH_SCOPES (read-only),
// preserving GDPR Art. 25(2) data-protection-by-default.
const submittedScopes = formData.getAll('scopes').filter((s): s is string => typeof s === 'string')
const validated = validateScopes(submittedScopes)
const clientCeiling: ApiKeyScope[] = parsed.scopes ?? [...DEFAULT_OAUTH_SCOPES]
const clientCeiling: ApiKeyScope[] = parsed.scopes ?? [...ALL_SCOPES]
const ceilingSet = new Set<ApiKeyScope>(clientCeiling)
const boundedToClient = (validated ?? []).filter(s => ceilingSet.has(s))
const grantedScopes: ApiKeyScope[] = boundedToClient.length > 0
@@ -60,7 +60,7 @@ registerEndpoint({
'SIE bookkeeping import (use /imports/sie). Auto-bank sync (use the enable-banking extension). Single-transaction creation (use POST /transactions/ingest with a 1-element array).',
pitfalls: [
'File size cap: 10 MB. Larger files require splitting client-side.',
'`format` query parameter is optional; auto-detection works for all supported banks. Pass `format` only to force a specific format. Accepted values: seb, swedbank, handelsbanken, nordea, nordea_business, lansforsakringar, ica_banken, skandia, lunar, generic_csv, camt053.',
'`format` query parameter is optional; auto-detection works for all supported banks. Pass `format` only to force a specific format. Accepted values: seb, swedbank, handelsbanken, nordea, nordea_business, lansforsakringar, ica_banken, skandia, lunar, northmill, generic_csv, camt053.',
'Duplicate detection is by external_id (composed from date + amount + counterparty); a re-import of the same file with the same flag set typically deduplicates rather than creating doubles.',
'BFL 5 kap 6-7 §§ note: this endpoint creates `transactions` rows (the underlag for a verifikation), NOT verifikationer themselves. The verifikation content requirements are in BFL 5 kap 6-7 §§; until each transaction is matched to an invoice/supplier-invoice (POST /transactions/{id}/match-*) or categorised (POST /transactions/{id}/categorize), the bookkeeping obligation isn\'t discharged. A successful import here means the data is ingested — not booked.',
'A successful import returns operation_id; poll /operations/{id} for the final ingested/duplicates/errors counts.',
@@ -134,6 +134,7 @@ export const POST = withApiV1<{ params: Promise<{ companyId: string }> }>(
'ica_banken',
'skandia',
'lunar',
'northmill',
'generic_csv',
'camt053',
])
+72 -20
View File
@@ -22,9 +22,27 @@ import {
} from '@/components/ui/table'
import { ArrowLeft, ArrowRight, Columns3 } from 'lucide-react'
import { formatCurrency } from '@/lib/utils'
import { getCSVPreview } from '@/lib/import/bank-file/formats/generic-csv'
import { getCSVPreview, normalizeMinusSign } from '@/lib/import/bank-file/formats/generic-csv'
import type { GenericCSVColumnMapping } from '@/lib/import/bank-file/types'
const HEADER_KEYWORDS = [
'datum',
'bokföringsdag',
'bokforingsdag',
'transaktionsdatum',
'reskontradatum',
'beskrivning',
'belopp',
'transaktion',
'text',
'mottagare',
'saldo',
'valuta',
'amount',
'description',
'date',
]
interface BankFileColumnMappingStepProps {
rawFileContent: string
onConfirm: (mapping: GenericCSVColumnMapping) => void
@@ -59,39 +77,70 @@ export default function BankFileColumnMappingStep({
const [decimalSep, setDecimalSep] = useState<',' | '.'>(',')
const [dateFormat, setDateFormat] = useState<string>('YYYY-MM-DD')
// Re-parse headers and preview whenever delimiter or file content changes
// Re-parse headers and preview whenever delimiter or file content changes.
// Pull a generous slice (30 rows) so we can scan past metadata preambles like
// Northmill's 5-line Kontonummer/Saldo/Kontohavare/Org.Nr/Period header.
const parsedRows = useMemo(
() => getCSVPreview(rawFileContent, delimiter, 10),
() => getCSVPreview(rawFileContent, delimiter, 30),
[rawFileContent, delimiter]
)
// Auto-detect whether the first row is a header: if any cell on row 0 looks
// like a date (YYYY-MM-DD, DD.MM.YYYY, DD/MM/YYYY, YYYYMMDD), it's data, not a header.
// Users can still override via the switch.
// Auto-detect the header row index by scanning for a row whose cells
// contain known column-name keywords (bokföringsdag, beskrivning, belopp, …).
// A row needs ≥ 2 keyword hits to qualify, which excludes metadata rows where
// only the label cell happens to match (e.g. "Saldo,251495,41,SEK").
// Falls back to row 0 if nothing qualifies — preserves prior behavior for
// simple files where the header truly is the first row.
const DATE_PATTERNS = [/^\d{4}-\d{2}-\d{2}$/, /^\d{2}[./]\d{2}[./]\d{4}$/, /^\d{8}$/]
const detectedHeaderRow = useMemo(() => {
let best: { idx: number; score: number; cells: number } | null = null
for (let i = 0; i < Math.min(parsedRows.length, 20); i++) {
const row = parsedRows[i]
if (!row || row.length < 2) continue
const hits = row.filter((cell) => {
const c = cell.trim().toLowerCase()
return c.length > 0 && HEADER_KEYWORDS.some((kw) => c === kw || c.includes(kw))
}).length
if (hits < 2) continue
if (!best || hits > best.score || (hits === best.score && row.length > best.cells)) {
best = { idx: i, score: hits, cells: row.length }
}
}
return best?.idx ?? 0
}, [parsedRows])
// Detect whether the file actually has a header row at all: if the auto-detect
// landed on row 0 but row 0 already looks like data (any cell is a date), assume
// no header. Otherwise trust the detection.
const detectedHasHeader = useMemo(() => {
const firstRow = parsedRows[0]
if (!firstRow) return true
const hasDateCell = firstRow.some((cell) =>
const headerRow = parsedRows[detectedHeaderRow]
if (!headerRow) return true
const looksLikeData = headerRow.some((cell) =>
DATE_PATTERNS.some((re) => re.test(cell.trim()))
)
return !hasDateCell
}, [parsedRows])
return !looksLikeData
}, [parsedRows, detectedHeaderRow])
const [hasHeaderOverride, setHasHeaderOverride] = useState<boolean | null>(null)
const hasHeader = hasHeaderOverride ?? detectedHasHeader
// skip_rows = number of rows to skip before transaction data starts.
// When hasHeader: skip past the header row (detectedHeaderRow + 1).
// When no header: skip nothing — data starts at row 0.
const skipRows = hasHeader ? detectedHeaderRow + 1 : 0
const columnHeaders = useMemo(() => {
if (hasHeader && parsedRows[0]) return parsedRows[0]
if (hasHeader && parsedRows[detectedHeaderRow]) return parsedRows[detectedHeaderRow]
const count = parsedRows[0]?.length ?? 0
return Array.from({ length: count }, (_, i) => `Kolumn ${i + 1}`)
}, [parsedRows, hasHeader])
}, [parsedRows, hasHeader, detectedHeaderRow])
const dataRows = hasHeader ? parsedRows.slice(1) : parsedRows
const dataRows = hasHeader ? parsedRows.slice(detectedHeaderRow + 1) : parsedRows
// Auto-guess date/description/amount columns from the first data row.
// Only used as initial defaults — user can override any pick.
const AMOUNT_RE = /^-?\d+([.,]\d+)?$/
// Match ASCII and Unicode minus; some banks (e.g. Northmill) use U+2212.
const AMOUNT_RE = /^[-\u2212\u2013\u2014\u2010]?\d+([.,]\d+)?$/
useEffect(() => {
if (dateCol !== -1 || descCol !== -1 || amountCol !== -1) return
const sample = dataRows[0]
@@ -124,7 +173,7 @@ export default function BankFileColumnMappingStep({
...(balanceCol >= 0 && { balance: balanceCol }),
delimiter,
decimal_separator: decimalSep,
skip_rows: hasHeader ? 1 : 0,
skip_rows: skipRows,
date_format: dateFormat,
}
onConfirm(mapping)
@@ -150,7 +199,9 @@ export default function BankFileColumnMappingStep({
<div className="space-y-0.5">
<Label htmlFor="has-header">Har filen rubrikrad?</Label>
<p className="text-xs text-muted-foreground">
Slå av om filen saknar rubrikrad och första raden redan innehåller transaktionsdata.
{hasHeader && detectedHeaderRow > 0
? `Hoppar över ${detectedHeaderRow} metadatarader. Rubrikraden upptäcktes på rad ${detectedHeaderRow + 1}.`
: 'Slå av om filen saknar rubrikrad och första raden redan innehåller transaktionsdata.'}
</p>
</div>
<Switch id="has-header" checked={hasHeader} onCheckedChange={setHasHeaderOverride} />
@@ -350,11 +401,12 @@ export default function BankFileColumnMappingStep({
</TableRow>
</TableHeader>
<TableBody>
{dataRows.slice(0, 5).map((row, i) => {
{dataRows.slice(0, 5).map((row, i) => {
const amountStr = row[amountCol] || '0'
const normalizedAmountStr = normalizeMinusSign(amountStr)
const amount = decimalSep === ','
? parseFloat(amountStr.replace(/\s/g, '').replace(',', '.'))
: parseFloat(amountStr.replace(/\s/g, ''))
? parseFloat(normalizedAmountStr.replace(/\s/g, '').replace(',', '.'))
: parseFloat(normalizedAmountStr.replace(/\s/g, ''))
return (
<TableRow key={i}>
+12
View File
@@ -46,6 +46,18 @@ export default function BankFileResultStep({
: `${result.errors} fel uppstod under importen.`}
</CardDescription>
</CardHeader>
{!isSuccess && result.first_error && (
<CardContent>
<div className="rounded-md border border-destructive/30 bg-destructive/5 p-3 text-sm">
<p className="font-medium text-destructive">Databasfel</p>
<p className="mt-1 font-mono text-xs text-muted-foreground break-all">
{result.first_error.message}
{result.first_error.details ? ` — ${result.first_error.details}` : ''}
{result.first_error.code ? ` (${result.first_error.code})` : ''}
</p>
</div>
</CardContent>
)}
</Card>
{/* Next steps */}
+8
View File
@@ -31,6 +31,7 @@ const FORMAT_NAMES: Record<string, string> = {
ica_banken: 'ICA Banken',
skandia: 'Skandia',
lunar: 'Lunar',
northmill: 'Northmill',
generic_csv: 'CSV (manuell mappning)',
camt053: 'ISO 20022 camt.053',
}
@@ -131,6 +132,7 @@ export default function BankFileUploadStep({
<SelectItem value="ica_banken">ICA Banken</SelectItem>
<SelectItem value="skandia">Skandia</SelectItem>
<SelectItem value="lunar">Lunar</SelectItem>
<SelectItem value="northmill">Northmill</SelectItem>
<SelectItem value="camt053">ISO 20022 camt.053 (XML)</SelectItem>
<SelectItem value="generic_csv">Annan CSV (manuell mappning)</SelectItem>
</SelectContent>
@@ -263,6 +265,12 @@ export default function BankFileUploadStep({
Logga in → Konto → Transaktioner → Exportera (CSV)
</p>
</div>
<div>
<p className="font-medium">Northmill</p>
<p className="text-muted-foreground">
Logga in → Konto → Kontoutdrag → Ladda ner (CSV)
</p>
</div>
</CardContent>
</Card>
</div>
+157 -1
View File
@@ -8,7 +8,7 @@
import { detectFileFormat, parseBankFile, generateExternalId, generateFileHash, getFormat, getAllFormats } from '../parser'
import type { ParsedBankTransaction, BankFileFormatId } from '../types'
import { parseGenericCSV } from '../formats/generic-csv'
import { parseGenericCSV, normalizeMinusSign } from '../formats/generic-csv'
import { parseCSVLine } from '../formats/nordea'
// ---------------------------------------------------------------------------
@@ -171,6 +171,25 @@ const LUNAR_CSV = [
'2024-01-13,LÖNEUTBETALNING,"25.000,00","12.877,17"',
].join('\n')
// Northmill exports include a 5-line metadata preamble (Kontonummer, Saldo,
// Kontohavare, Org. Nr, Period) plus blank lines before the actual transaction
// header. Negative amounts use Unicode minus (U+2212), not ASCII hyphen.
const NORTHMILL_CSV = [
'Kontonummer,9750-8770139',
'Saldo,"251495,41",SEK',
'Kontohavare,Arcim Technology AB',
'Org. Nr,559538-6219',
'Period,2025-10-01,2026-04-07',
'',
'',
'',
'Bokföringsdag,Beskrivning,Belopp,Saldo,Valuta',
'2026-04-01,Månadsavgift företagspaket april,"\u2212139,00","251495,41",SEK',
'2026-01-22,200176580348155,"\u22121000,00","251912,41",SEK',
'2025-10-16,092221155575,"400000,00","422005,00",SEK',
'2025-10-01,Inbetalning av aktiekapital,"25000,00","25000,00",SEK',
].join('\n')
const UNKNOWN_CSV = [
'id,name,value,timestamp',
'1,Widget A,100,2024-01-15T10:00:00',
@@ -357,6 +376,22 @@ describe('detectFileFormat', () => {
expect(format!.id).toBe('lunar')
})
it('detects Northmill CSV from Kontonummer preamble + transaction header', () => {
const format = detectFileFormat(NORTHMILL_CSV, 'Northmill-Account-Statement.csv')
expect(format).not.toBeNull()
expect(format!.id).toBe('northmill')
})
it('does not detect Northmill on a file that just happens to mention Kontonummer in transactions', () => {
const fake = [
'Datum,Transaktion,Kategori,Belopp,Saldo',
'2024-01-15,Överföring kontonummer 1234,Inkomst,"100,00","1000,00"',
].join('\n')
const format = detectFileFormat(fake, 'test.csv')
// Should detect as Nordea, not Northmill — Northmill needs Kontonummer at start of first line
expect(format!.id).toBe('nordea')
})
it('returns null for unrecognized CSV content', () => {
const format = detectFileFormat(UNKNOWN_CSV, 'data.csv')
expect(format).toBeNull()
@@ -1054,6 +1089,53 @@ describe('parseBankFile — Lunar format', () => {
})
})
describe('parseBankFile — Northmill format', () => {
it('skips the 5-line metadata preamble and blank lines, parses transaction rows', () => {
const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv')
expect(result.format).toBe('northmill')
expect(result.format_name).toBe('Northmill')
expect(result.transactions).toHaveLength(4)
expect(result.issues).toHaveLength(0)
})
it('correctly parses negative amounts that use Unicode minus (U+2212)', () => {
const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv')
// First transaction is "−139,00" with U+2212 — must become -139, not NaN
expect(result.transactions[0].amount).toBe(-139)
expect(result.transactions[0].description).toBe('Månadsavgift företagspaket april')
expect(result.transactions[0].date).toBe('2026-04-01')
expect(result.transactions[1].amount).toBe(-1000)
expect(result.transactions[2].amount).toBe(400000)
expect(result.transactions[3].amount).toBe(25000)
})
it('extracts the saldo (running balance) column', () => {
const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv')
expect(result.transactions[0].balance).toBe(251495.41)
expect(result.transactions[3].balance).toBe(25000)
})
it('calculates income vs expenses correctly with Unicode minus amounts', () => {
const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv')
expect(result.stats.total_income).toBe(425000)
expect(result.stats.total_expenses).toBe(-1139)
expect(result.stats.parsed_rows).toBe(4)
expect(result.stats.skipped_rows).toBe(0)
})
it('extracts the correct date range from the transactions, not the Period metadata row', () => {
const result = parseBankFile(NORTHMILL_CSV, 'Northmill.csv')
expect(result.date_from).toBe('2025-10-01')
expect(result.date_to).toBe('2026-04-01')
})
})
describe('parseBankFile — camt.053 XML format', () => {
it('parses XML with credit and debit entries', () => {
const result = parseBankFile(CAMT053_XML, 'statement.xml')
@@ -1716,6 +1798,80 @@ describe('parseGenericCSV — column bounds checking', () => {
expect(result.stats.skipped_rows).toBe(0)
expect(result.issues).toHaveLength(0)
})
it('parses amounts that use Unicode minus (U+2212) — was NaN before normalization', () => {
const content = [
'Date,Description,Amount',
'2024-01-15,SPOTIFY,"\u221299,00"',
'2024-01-16,REFUND,"\u201350,00"',
'2024-01-17,SALARY,"25000,00"',
].join('\n')
const result = parseGenericCSV(content, {
date: 0,
description: 1,
amount: 2,
delimiter: ',',
decimal_separator: ',',
skip_rows: 1,
date_format: 'YYYY-MM-DD',
})
expect(result.transactions).toHaveLength(3)
expect(result.transactions[0].amount).toBe(-99)
expect(result.transactions[1].amount).toBe(-50)
expect(result.transactions[2].amount).toBe(25000)
})
it('skips metadata rows when skip_rows is set higher than 1 (multi-row preamble)', () => {
// Mimics a file with 5 metadata rows + header + 2 transactions.
const content = [
'Account,12345',
'Owner,Acme AB',
'Period,2024-01,2024-12',
'Currency,SEK',
'Type,Statement',
'Date,Description,Amount',
'2024-01-15,SPOTIFY,-99.00',
'2024-01-16,SALARY,25000.00',
].join('\n')
const result = parseGenericCSV(content, {
date: 0,
description: 1,
amount: 2,
delimiter: ',',
decimal_separator: '.',
skip_rows: 6,
date_format: 'YYYY-MM-DD',
})
expect(result.transactions).toHaveLength(2)
expect(result.transactions[0].amount).toBe(-99)
expect(result.transactions[1].amount).toBe(25000)
expect(result.issues).toHaveLength(0)
})
})
describe('normalizeMinusSign', () => {
it('replaces U+2212 (minus sign) with ASCII hyphen', () => {
expect(normalizeMinusSign('\u2212139,00')).toBe('-139,00')
})
it('replaces U+2013 (en dash) and U+2014 (em dash) with ASCII hyphen', () => {
expect(normalizeMinusSign('\u2013100')).toBe('-100')
expect(normalizeMinusSign('\u2014250')).toBe('-250')
})
it('leaves ASCII hyphen and digits untouched', () => {
expect(normalizeMinusSign('-139.00')).toBe('-139.00')
expect(normalizeMinusSign('139.00')).toBe('139.00')
})
it('makes parseFloat work on Unicode-minus strings (regression for Northmill)', () => {
expect(parseFloat('\u2212139.00')).toBeNaN()
expect(parseFloat(normalizeMinusSign('\u2212139.00'))).toBe(-139)
})
})
// --- Fix 8: parseCSVLine unclosed quote handling ---
+18 -5
View File
@@ -10,6 +10,15 @@ import { prepareContent } from '../../shared/encoding'
import { parseCSVLine } from './nordea'
import { normalizeDate } from '../date-utils'
/**
* Normalize Unicode minus variants (U+2212 "−", U+2013 "–", U+2014 "—", U+2010 "‐")
* to ASCII hyphen-minus so parseFloat can read them. Northmill and some other
* banks export negatives with U+2212; parseFloat returns NaN for those.
*/
export function normalizeMinusSign(value: string): string {
return value.replace(/[\u2212\u2013\u2014\u2010]/g, '-')
}
/**
* Parse a generic CSV with user-provided column mapping
*/
@@ -89,12 +98,15 @@ export function parseGenericCSV(
continue
}
// Parse amount based on configured decimal separator
// Parse amount based on configured decimal separator.
// Normalize Unicode minus first — some banks (e.g. Northmill) use U+2212
// instead of ASCII hyphen, which parseFloat treats as NaN.
const normalizedAmount = normalizeMinusSign(amountStr)
let amount: number
if (mapping.decimal_separator === ',') {
amount = parseFloat(amountStr.replace(/\s/g, '').replace(',', '.'))
amount = parseFloat(normalizedAmount.replace(/\s/g, '').replace(',', '.'))
} else {
amount = parseFloat(amountStr.replace(/\s/g, ''))
amount = parseFloat(normalizedAmount.replace(/\s/g, ''))
}
if (isNaN(amount)) {
@@ -113,10 +125,11 @@ export function parseGenericCSV(
let balance: number | null = null
if (balanceStr) {
const normalizedBalance = normalizeMinusSign(balanceStr)
if (mapping.decimal_separator === ',') {
balance = parseFloat(balanceStr.replace(/\s/g, '').replace(',', '.'))
balance = parseFloat(normalizedBalance.replace(/\s/g, '').replace(',', '.'))
} else {
balance = parseFloat(balanceStr.replace(/\s/g, ''))
balance = parseFloat(normalizedBalance.replace(/\s/g, ''))
}
if (isNaN(balance)) balance = null
}
+165
View File
@@ -0,0 +1,165 @@
/**
* Northmill CSV format parser
*
* Format: Comma-delimited, comma decimal separator (amounts are quoted)
* Preamble: 5 metadata rows (Kontonummer, Saldo, Kontohavare, Org. Nr, Period)
* followed by blank lines before the transaction header.
* Header: Bokföringsdag,Beskrivning,Belopp,Saldo,Valuta
* Date format: YYYY-MM-DD
* Encoding: UTF-8
*
* Notes:
* - Negative amounts use Unicode minus sign (U+2212 "−"), not ASCII hyphen.
* The parser normalizes the minus before calling parseFloat.
*/
import type { BankFileFormat, BankFileParseResult, ParsedBankTransaction, BankFileParseIssue } from '../types'
import { prepareContent } from '../../shared/encoding'
import { normalizeDate } from '../date-utils'
import { parseCSVLine } from './nordea'
import { normalizeMinusSign } from './generic-csv'
const HEADER_KEYWORDS = ['bokföringsdag', 'beskrivning', 'belopp']
function isNorthmillHeader(line: string): boolean {
const lower = line.toLowerCase()
if (lower.includes(';')) return false
return HEADER_KEYWORDS.every((kw) => lower.includes(kw))
}
function parseAmount(value: string): number {
// Northmill: "−139,00" (Unicode minus) or "400000,00" (no thousand separator on positives)
const cleaned = normalizeMinusSign(value).replace(/\s/g, '').replace(',', '.')
return parseFloat(cleaned)
}
export const northmillFormat: BankFileFormat = {
id: 'northmill',
name: 'Northmill',
description: 'Northmill kontoutdrag (CSV med metadata-rader och Unicode-minus)',
fileExtensions: ['.csv', '.txt'],
detect(content: string, _filename: string): boolean {
const prepared = prepareContent(content)
const lines = prepared.split('\n')
// Northmill preamble's first non-empty line is "Kontonummer,<account>"
const firstNonEmpty = lines.find((l) => l.trim() !== '') || ''
if (!/^kontonummer\s*,/i.test(firstNonEmpty.trim())) return false
// Also look for the transaction header within the first ~20 lines to be sure
return lines.slice(0, 20).some(isNorthmillHeader)
},
parse(content: string): BankFileParseResult {
const prepared = prepareContent(content)
const rawLines = prepared.split('\n')
const transactions: ParsedBankTransaction[] = []
const issues: BankFileParseIssue[] = []
let skippedRows = 0
const headerIdx = rawLines.findIndex(isNorthmillHeader)
if (headerIdx === -1) {
return {
format: 'northmill',
format_name: 'Northmill',
transactions: [],
date_from: null,
date_to: null,
issues: [{ row: 0, message: 'Kunde inte hitta rubrikraden (Bokföringsdag, Beskrivning, Belopp).', severity: 'error' }],
stats: { total_rows: 0, parsed_rows: 0, skipped_rows: 0, total_income: 0, total_expenses: 0 },
}
}
const headers = parseCSVLine(rawLines[headerIdx], ',').map((h) => h.trim().toLowerCase().replace(/"/g, ''))
const dateIdx = headers.findIndex((h) => h.includes('bokföringsdag') || h.includes('bokforingsdag'))
const descIdx = headers.findIndex((h) => h.includes('beskrivning'))
const amountIdx = headers.findIndex((h) => h.includes('belopp'))
const balanceIdx = headers.findIndex((h) => h.includes('saldo'))
if (dateIdx === -1 || descIdx === -1 || amountIdx === -1) {
issues.push({
row: headerIdx + 1,
message: 'Rubrikraden saknar nödvändiga kolumner (Bokföringsdag, Beskrivning, Belopp).',
severity: 'error',
})
return {
format: 'northmill',
format_name: 'Northmill',
transactions: [],
date_from: null,
date_to: null,
issues,
stats: { total_rows: 0, parsed_rows: 0, skipped_rows: 0, total_income: 0, total_expenses: 0 },
}
}
for (let i = headerIdx + 1; i < rawLines.length; i++) {
const line = rawLines[i].trim()
if (!line) continue
const fields = parseCSVLine(line, ',').map((f) => f.trim().replace(/^"|"$/g, ''))
const dateStr = fields[dateIdx]
const description = fields[descIdx] || 'Unknown'
const amountStr = fields[amountIdx]
const balanceStr = balanceIdx >= 0 ? fields[balanceIdx] : undefined
if (!dateStr || !amountStr) {
const missing: string[] = []
if (!dateStr) missing.push('datum')
if (!amountStr) missing.push('belopp')
issues.push({ row: i + 1, message: `Saknar ${missing.join(' och ')}`, severity: 'warning' })
skippedRows++
continue
}
const date = normalizeDate(dateStr)
if (!date) {
issues.push({ row: i + 1, message: `Ogiltigt datumformat: ${dateStr}`, severity: 'warning' })
skippedRows++
continue
}
const amount = parseAmount(amountStr)
if (isNaN(amount)) {
issues.push({ row: i + 1, message: `Ogiltigt belopp: ${amountStr}`, severity: 'warning' })
skippedRows++
continue
}
let balance: number | null = null
if (balanceStr) {
const b = parseAmount(balanceStr)
balance = isNaN(b) ? null : b
}
transactions.push({
date,
description: description.trim(),
amount,
currency: 'SEK',
balance,
reference: null,
counterparty: null,
raw_line: line,
})
}
const dates = transactions.map((t) => t.date).sort()
return {
format: 'northmill',
format_name: 'Northmill',
transactions,
date_from: dates[0] || null,
date_to: dates[dates.length - 1] || null,
issues,
stats: {
total_rows: rawLines.length - headerIdx - 1,
parsed_rows: transactions.length,
skipped_rows: skippedRows,
total_income: Math.round(transactions.filter((t) => t.amount > 0).reduce((s, t) => s + t.amount, 0) * 100) / 100,
total_expenses: Math.round(transactions.filter((t) => t.amount < 0).reduce((s, t) => s + t.amount, 0) * 100) / 100,
},
}
},
}
+2
View File
@@ -16,6 +16,7 @@ import { lansforsakringarFormat } from './formats/lansforsakringar'
import { icaBankenFormat } from './formats/ica-banken'
import { skandiaFormat } from './formats/skandia'
import { lunarFormat } from './formats/lunar'
import { northmillFormat } from './formats/northmill'
import { camt053Format } from './formats/camt053'
import { genericCSVFormat } from './formats/generic-csv'
@@ -36,6 +37,7 @@ const FORMATS: BankFileFormat[] = [
icaBankenFormat,
skandiaFormat,
lunarFormat,
northmillFormat,
genericCSVFormat,
]
+1
View File
@@ -52,6 +52,7 @@ export type BankFileFormatId =
| 'ica_banken'
| 'skandia'
| 'lunar'
| 'northmill'
| 'generic_csv'
| 'camt053'
+8
View File
@@ -263,6 +263,14 @@ export async function ingestTransactions(
if (insertError || !newTransaction) {
result.errors++
if (!result.first_error && insertError) {
result.first_error = {
message: insertError.message,
code: insertError.code ?? null,
details: insertError.details ?? null,
hint: insertError.hint ?? null,
}
}
continue
}
+2
View File
@@ -2578,6 +2578,8 @@ export interface IngestResult {
auto_matched_invoices: number
errors: number
transaction_ids: string[]
/** First insert error encountered, surfaced for debugging. Optional. */
first_error?: { message: string; code?: string | null; details?: string | null; hint?: string | null }
}
// ── Invoice extraction (used by invoice-inbox extension and core utils) ──