From 78a581bca16f023993d254fad8747b6062c0a52d Mon Sep 17 00:00:00 2001 From: Jakob Wennberg Date: Thu, 13 Aug 2026 15:14:23 +0200 Subject: [PATCH] fix(import): guard against CP437-as-CP1252 mojibake entering via pre-decoded SIE text (#1569) * refactor(arcim-migration): remove the dead gateway SIE export path fetchSIEExport and SIEExportFile have had zero callers since the direct provider clients replaced the Arcim Sync gateway (#181, #718). The path returned SIE as a pre-decoded string, and the gateway's decode of CP437 bytes as windows-1252 is what wrote the 2026-03-17 mojibake into posted entries. Deleting it makes the string-typed SIE fetch impossible to re-wire; a comment marks the grave. The consent lifecycle and entity accessors stay untouched. Co-Authored-By: Claude Fable 5 * fix(import): warn when SIE text carries CP437-as-CP1252 mojibake The 2026-03-17 migration wrote mojibake ("L"neutbetalning"-style C1 specials) into posted entries because the retired gateway handed the /import-sie handler an already-decoded string: byte-level encoding detection never saw it, and nothing downstream checked. The live bug is gone; this is the tripwire so the signature can never land silently again. - lib/import/sie-artifact-scan.ts: pure scanner over parsed SIE account names and voucher/line descriptions, reusing hasCp1252Artifact from charset-repair; flags at >= 2 hits so a lone legitimate curly quote or apostrophe cannot false-positive a whole file. - arcim-migration /import-sie: warn-never-block; the Swedish warning rides on result.warnings, which the workspace UI already renders, plus a server-side log.warn. - wizard parse route: same scan, surfaced through the existing parse-issue warnings card in the preview, pointing at the first affected line. Co-Authored-By: Claude Fable 5 * test(bookkeeping): pin the reported gateway mojibake strings Adds the four strings reported from the affected company's journal as reverse_cp437 cases (all reverse losslessly) plus a false-positive guard: space-padded typography must never route into the CP437 reversal. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- .../import/sie/parse/__tests__/route.test.ts | 148 ++++++++++++++++ app/api/import/sie/parse/route.ts | 24 +++ .../import-sie-mojibake-tripwire.test.ts | 146 ++++++++++++++++ extensions/general/arcim-migration/index.ts | 21 +++ .../arcim-migration/lib/arcim-client.ts | 26 +-- .../__tests__/charset-repair.test.ts | 21 +++ .../__tests__/sie-artifact-scan.test.ts | 161 ++++++++++++++++++ lib/import/sie-artifact-scan.ts | 92 ++++++++++ 8 files changed, 618 insertions(+), 21 deletions(-) create mode 100644 app/api/import/sie/parse/__tests__/route.test.ts create mode 100644 extensions/general/arcim-migration/__tests__/import-sie-mojibake-tripwire.test.ts create mode 100644 lib/import/__tests__/sie-artifact-scan.test.ts create mode 100644 lib/import/sie-artifact-scan.ts diff --git a/app/api/import/sie/parse/__tests__/route.test.ts b/app/api/import/sie/parse/__tests__/route.test.ts new file mode 100644 index 00000000..51069548 --- /dev/null +++ b/app/api/import/sie/parse/__tests__/route.test.ts @@ -0,0 +1,148 @@ +/** + * Tests for POST /api/import/sie/parse. + * + * Runs through the real withRouteContext wrapper (auth/company deps mocked) + * with the real SIE parser and encoding detection. Focus: the CP1252-mojibake + * tripwire. A file whose text was mis-decoded UPSTREAM (CP437 bytes read as + * windows-1252, then saved as UTF-8) decodes "correctly" here, so byte-level + * detection can never catch it; the artifact scan must add a parse-issue + * warning to the preview WITHOUT blocking the parse. The mojibake literals are + * the subject under test and must stay byte-exact. + * + * The route has no 404 path: it operates on the uploaded file, not a stored + * resource. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { NextResponse } from 'next/server' +import { createQueuedMockSupabase } from '@/tests/helpers' +import type { ParseIssue } from '@/lib/import/types' + +const { supabase, enqueue, reset } = createQueuedMockSupabase() + +const requireAuthMock = vi.fn() +vi.mock('@/lib/auth/require-auth', () => ({ + requireAuth: (...args: unknown[]) => requireAuthMock(...args), +})) + +vi.mock('@/lib/company/context', () => ({ + getActiveCompanyId: vi.fn().mockResolvedValue('company-1'), + requireCompanyId: vi.fn().mockResolvedValue('company-1'), +})) + +const requireWriteMock = vi.fn() +vi.mock('@/lib/auth/require-write', () => ({ + requireWritePermission: (...args: unknown[]) => requireWriteMock(...args), +})) + +// Keep the real preview generator; stub only the DB-touching duplicate checks. +vi.mock('@/lib/import/sie-import', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + checkDuplicateImport: vi.fn().mockResolvedValue(null), + checkDuplicatePeriodImport: vi.fn().mockResolvedValue(null), + } +}) + +import { POST } from '../route' + +const emptyParams = { params: Promise.resolve({}) } + +// A file whose text was mis-decoded upstream and re-saved as UTF-8. The BOM is +// what such a file realistically carries after an editor/tool re-save, and it +// pins detectEncoding to utf8: without intact C3-prefixed Swedish sequences the +// byte heuristic would otherwise read the U+201D/U+201E continuation bytes as +// CP437 and re-mangle the text before the scanner could see the signature. +const MOJIBAKE_SIE = '\uFEFF' + [ + '#FLAGGA 0', + '#SIETYP 4', + '#FNAMN "Migrerad AB"', + '#RAR 0 20240101 20241231', + '#KONTO 1930 "F”retagskonto"', + '#KONTO 4056 "Ink”p tj„nster inom EU"', + '#VER A 1 20240115 "L”neutbetalning"', + '{', + '#TRANS 1930 {} -100.00', + '#TRANS 4056 {} 100.00', + '}', +].join('\n') + +const CLEAN_SIE = MOJIBAKE_SIE + .replace('F”retagskonto', 'Företagskonto') + .replace('Ink”p tj„nster inom EU', 'Inköp tjänster inom EU') + .replace('L”neutbetalning', 'Löneutbetalning') + +function fileRequest(content: string, filename = 'test.se'): Request { + const formData = new FormData() + formData.append('file', new File([content], filename, { type: 'text/plain' })) + return new Request('http://localhost:3000/api/import/sie/parse', { + method: 'POST', + body: formData, + }) +} + +type ParseResponse = { + success: boolean + parsed: { issues: ParseIssue[] } + validation: { valid: boolean; warnings: string[] } +} + +describe('POST /api/import/sie/parse', () => { + beforeEach(() => { + vi.clearAllMocks() + reset() + requireAuthMock.mockResolvedValue({ user: { id: 'user-1' }, supabase }) + requireWriteMock.mockResolvedValue({ ok: true }) + // The stored-mappings lookup (sie_account_mappings select). + enqueue({ data: [] }) + }) + + it('returns 401 when unauthenticated', async () => { + requireAuthMock.mockResolvedValue({ + user: null, + supabase, + error: NextResponse.json({ error: 'Unauthorized' }, { status: 401 }), + }) + + const response = await POST(fileRequest(CLEAN_SIE), emptyParams) + expect(response.status).toBe(401) + }) + + it('returns 400 when no file is attached', async () => { + const formData = new FormData() + const request = new Request('http://localhost:3000/api/import/sie/parse', { + method: 'POST', + body: formData, + }) + + const response = await POST(request, emptyParams) + expect(response.status).toBe(400) + }) + + it('adds a mojibake warning to the preview issues without blocking the parse', async () => { + const response = await POST(fileRequest(MOJIBAKE_SIE), emptyParams) + const body = (await response.json()) as ParseResponse + + expect(response.status).toBe(200) + expect(body.success).toBe(true) + expect(body.validation.valid).toBe(true) + + const issue = body.parsed.issues.find((i) => i.message.includes('felaktigt teckenkodad')) + expect(issue).toBeDefined() + expect(issue!.severity).toBe('warning') + // Points at the first flagged string's line: #KONTO 1930 "F”retagskonto". + expect(issue!.line).toBe(5) + // validateSIEFile folds parse issues into the validation warnings too. + expect(body.validation.warnings.some((w) => w.includes('felaktigt teckenkodad'))).toBe(true) + }) + + it('adds no mojibake warning for a clean file', async () => { + const response = await POST(fileRequest(CLEAN_SIE), emptyParams) + const body = (await response.json()) as ParseResponse + + expect(response.status).toBe(200) + expect(body.success).toBe(true) + expect(body.parsed.issues.some((i) => i.message.includes('felaktigt teckenkodad'))).toBe(false) + expect(body.validation.warnings.some((w) => w.includes('felaktigt teckenkodad'))).toBe(false) + }) +}) diff --git a/app/api/import/sie/parse/route.ts b/app/api/import/sie/parse/route.ts index 8d50aebd..2cbb6fba 100644 --- a/app/api/import/sie/parse/route.ts +++ b/app/api/import/sie/parse/route.ts @@ -7,6 +7,7 @@ import { calculateFileHash, } from '@/lib/import/sie-parser' import { suggestMappings, getMappingStats, isSystemAccount } from '@/lib/import/account-mapper' +import { scanSieForCp1252Artifacts, formatSieArtifactWarning } from '@/lib/import/sie-artifact-scan' import { generateImportPreview, checkDuplicateImport, checkDuplicatePeriodImport } from '@/lib/import/sie-import' import { BAS_REFERENCE } from '@/lib/bookkeeping/bas-data' import { withRouteContext } from '@/lib/api/with-route-context' @@ -70,6 +71,29 @@ export const POST = withRouteContext( const parsed = parseSIEFile(content) + // Mojibake tripwire (warn, never block): CP437 bytes decoded as + // windows-1252 somewhere upstream leave C1 specials mid-word in account + // names and voucher texts. Surface it as a parse-issue warning, the + // preview's existing warnings card, so the user can abort before import. + const artifactScan = scanSieForCp1252Artifacts(parsed) + if (artifactScan.flagged) { + const contentLines = content.split(/\r?\n/) + const firstSample = artifactScan.samples[0] + const sampleLine = firstSample + ? contentLines.findIndex((l) => l.includes(firstSample)) + : -1 + parsed.issues.push({ + severity: 'warning', + line: sampleLine >= 0 ? sampleLine + 1 : 1, + message: formatSieArtifactWarning(artifactScan), + }) + opLog.warn('sie parse: CP1252 mojibake artifacts in decoded content', { + encoding, + artifactCount: artifactScan.artifactCount, + samples: artifactScan.samples, + }) + } + if (parsed.stats.fiscalYearStart && parsed.stats.fiscalYearEnd) { const periodDuplicate = await checkDuplicatePeriodImport( supabase, diff --git a/extensions/general/arcim-migration/__tests__/import-sie-mojibake-tripwire.test.ts b/extensions/general/arcim-migration/__tests__/import-sie-mojibake-tripwire.test.ts new file mode 100644 index 00000000..94497c2e --- /dev/null +++ b/extensions/general/arcim-migration/__tests__/import-sie-mojibake-tripwire.test.ts @@ -0,0 +1,146 @@ +/** + * Tests for the CP1252-mojibake tripwire on POST /import-sie. + * + * This handler receives SIE as an ALREADY-DECODED string (rawContent in JSON), + * so an upstream that decoded CP437 bytes as windows-1252 has baked the + * corruption in before the repo's own encoding detection could run: exactly + * what the retired Arcim Sync gateway did on 2026-03-17, landing mojibake + * ("L”neutbetalning", "BANKTJŽNSTER") in posted entries with no warning. + * + * The tripwire must WARN and NEVER BLOCK: the import proceeds untouched and + * the warning rides on result.warnings, which the migration workspace UI + * already renders. The mojibake literals below are the subject under test and + * must stay byte-exact. + */ +import { describe, it, expect, beforeEach, vi, type Mock } from 'vitest' +import { createMockRequest, parseJsonResponse } from '@/tests/helpers' +import type { ExtensionContext } from '@/lib/extensions/types' + +vi.mock('@/lib/import/sie-import', () => ({ + loadMappings: vi.fn(), + generateImportPreview: vi.fn(), + executeSIEImport: vi.fn(), +})) + +import { arcimMigrationExtension } from '../index' +import { executeSIEImport } from '@/lib/import/sie-import' + +const importSieRoute = (arcimMigrationExtension.apiRoutes ?? []).find( + (r) => r.method === 'POST' && r.path === '/import-sie', +)! + +type RouteHandler = (request: Request, ctx?: ExtensionContext) => Promise +const handler = importSieRoute.handler as RouteHandler + +const MOJIBAKE_SIE = [ + '#FLAGGA 0', + '#SIETYP 4', + '#FNAMN "Migrerad AB"', + '#RAR 0 20240101 20241231', + '#KONTO 1930 "F”retagskonto"', + '#KONTO 4056 "Ink”p tj„nster inom EU"', + '#VER A 1 20240115 "L”neutbetalning"', + '{', + '#TRANS 1930 {} -100.00', + '#TRANS 4056 {} 100.00', + '}', +].join('\n') + +const CLEAN_SIE = MOJIBAKE_SIE + .replace('F”retagskonto', 'Företagskonto') + .replace('Ink”p tj„nster inom EU', 'Inköp tjänster inom EU') + .replace('L”neutbetalning', 'Löneutbetalning') + +const MAPPINGS = [ + { sourceAccount: '1930', sourceName: 'Företagskonto', targetAccount: '1930' }, + { sourceAccount: '4056', sourceName: 'Inköp tjänster inom EU', targetAccount: '4056' }, +] + +function buildCtx(userId: string | null) { + const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() } + const supabase = { + auth: { + getUser: vi.fn().mockResolvedValue({ data: { user: userId ? { id: userId } : null } }), + }, + } + const ctx = { supabase, companyId: 'company-1', log } as unknown as ExtensionContext + return { ctx, log } +} + +function importRequest(body: Record) { + return createMockRequest('http://localhost/api/extensions/ext/arcim-migration/import-sie', { + method: 'POST', + body, + }) +} + +describe('POST /import-sie: mojibake tripwire', () => { + beforeEach(() => { + vi.clearAllMocks() + // Fresh result object per call: the handler pushes onto result.warnings. + ;(executeSIEImport as Mock).mockImplementation(async () => ({ + success: true, + journalEntriesCreated: 1, + errors: [], + warnings: [], + })) + }) + + it('returns 401 when unauthenticated', async () => { + const { ctx } = buildCtx(null) + const response = await handler( + importRequest({ rawContent: CLEAN_SIE, mappings: MAPPINGS, options: {} }), + ctx, + ) + expect(response.status).toBe(401) + expect(executeSIEImport).not.toHaveBeenCalled() + }) + + it('returns 400 when rawContent is missing', async () => { + const { ctx } = buildCtx('user-1') + const response = await handler(importRequest({ mappings: MAPPINGS, options: {} }), ctx) + expect(response.status).toBe(400) + expect(executeSIEImport).not.toHaveBeenCalled() + }) + + it('surfaces a Swedish warning on mojibaked content WITHOUT blocking the import', async () => { + const { ctx, log } = buildCtx('user-1') + const response = await handler( + importRequest({ rawContent: MOJIBAKE_SIE, mappings: MAPPINGS, options: {} }), + ctx, + ) + const { status, body } = await parseJsonResponse<{ warnings: string[] }>(response) + + expect(status).toBe(200) + // Warn, never block: the import ran despite the flagged content. + expect(executeSIEImport).toHaveBeenCalledTimes(1) + + const tripwire = body.warnings.find((w) => w.includes('felaktigt teckenkodad')) + expect(tripwire).toBeDefined() + // The first flagged string (accounts are scanned first) is the example. + expect(tripwire).toContain('F”retagskonto') + // Must dodge the workspace UI's structured-card filters, or the warning + // silently disappears from the "Remaining warnings" card. + expect(tripwire).not.toContain('hoppades över') + expect(tripwire).not.toContain('förts om till eget kapital') + + expect(log.warn).toHaveBeenCalledWith( + 'import-sie: CP1252 mojibake artifacts in SIE text', + expect.objectContaining({ artifactCount: 3 }), + ) + }) + + it('adds no warning for clean content', async () => { + const { ctx, log } = buildCtx('user-1') + const response = await handler( + importRequest({ rawContent: CLEAN_SIE, mappings: MAPPINGS, options: {} }), + ctx, + ) + const { status, body } = await parseJsonResponse<{ warnings: string[] }>(response) + + expect(status).toBe(200) + expect(executeSIEImport).toHaveBeenCalledTimes(1) + expect(body.warnings.some((w) => w.includes('felaktigt teckenkodad'))).toBe(false) + expect(log.warn).not.toHaveBeenCalled() + }) +}) diff --git a/extensions/general/arcim-migration/index.ts b/extensions/general/arcim-migration/index.ts index 3a953b4e..d1faaaf4 100644 --- a/extensions/general/arcim-migration/index.ts +++ b/extensions/general/arcim-migration/index.ts @@ -28,6 +28,7 @@ import { reconcileSupplierInvoiceVouchers } from '@/lib/invoices/bulk-reconcile- import type { ArcimProvider } from './types' import { ARCIM_PROVIDERS } from './types' import { parseSIEFile, validateSIEFile } from '@/lib/import/sie-parser' +import { scanSieForCp1252Artifacts, formatSieArtifactWarning } from '@/lib/import/sie-artifact-scan' import { suggestMappings, getMappingStats, isSystemAccount } from '@/lib/import/account-mapper' import { loadMappings, generateImportPreview, executeSIEImport } from '@/lib/import/sie-import' import { BAS_REFERENCE } from '@/lib/bookkeeping/bas-reference' @@ -995,6 +996,20 @@ export const arcimMigrationExtension: Extension = { try { const parsed = parseSIEFile(rawContent) + // Mojibake tripwire (warn, never block). This handler receives SIE + // as an ALREADY-DECODED string, so an upstream that decoded CP437 + // bytes as windows-1252 has baked the corruption in before we ever + // see it (the retired Arcim Sync gateway did exactly that on + // 2026-03-17). Flag the signature so it can never again land + // silently in posted entries; the import itself proceeds untouched. + const artifactScan = scanSieForCp1252Artifacts(parsed) + if (artifactScan.flagged) { + log.warn('import-sie: CP1252 mojibake artifacts in SIE text', { + artifactCount: artifactScan.artifactCount, + samples: artifactScan.samples, + }) + } + // Validate all accounts are mapped (same as manual upload) const unmapped = mappings.filter((m: import('@/lib/import/types').AccountMapping) => !m.targetAccount) if (unmapped.length > 0) { @@ -1035,6 +1050,12 @@ export const arcimMigrationExtension: Extension = { onExistingPeriod: 'replace', }) + // Surface the tripwire on the result the workspace UI already + // renders (its "Remaining warnings" card shows result.warnings). + if (artifactScan.flagged) { + result.warnings.push(formatSieArtifactWarning(artifactScan)) + } + log.info('SIE import completed:', { success: result.success, journalEntriesCreated: result.journalEntriesCreated, diff --git a/extensions/general/arcim-migration/lib/arcim-client.ts b/extensions/general/arcim-migration/lib/arcim-client.ts index 7b130263..32794027 100644 --- a/extensions/general/arcim-migration/lib/arcim-client.ts +++ b/extensions/general/arcim-migration/lib/arcim-client.ts @@ -233,24 +233,8 @@ export async function fetchSupplierInvoices( return fetchAllPages(consentId, 'supplierinvoices', params) } -// ── SIE export ──────────────────────────────────────────────────── - -export interface SIEExportFile { - fiscalYear: number - sieType: number - rawContent: string - accountCount: number - transactionCount: number -} - -export async function fetchSIEExport( - consentId: string, - sieType?: number -): Promise<{ files: SIEExportFile[] }> { - const params = new URLSearchParams() - if (sieType) params.set('sieType', String(sieType)) - const qs = params.toString() - return request<{ files: SIEExportFile[] }>( - `/api/v1/consents/${consentId}/sie/export${qs ? `?${qs}` : ''}` - ) -} +// The gateway SIE export path (fetchSIEExport/SIEExportFile) was deliberately +// removed: it returned SIE as a pre-decoded string, and the gateway's decode of +// CP437 bytes as windows-1252 caused the 2026-03-17 mojibake incident. Provider +// SIE now travels as raw bytes through lib/sie-fetcher.ts and the repo's own +// encoding detection. Do not re-add a string-typed SIE fetch here. diff --git a/lib/bookkeeping/__tests__/charset-repair.test.ts b/lib/bookkeeping/__tests__/charset-repair.test.ts index 5280c2dc..056e15f5 100644 --- a/lib/bookkeeping/__tests__/charset-repair.test.ts +++ b/lib/bookkeeping/__tests__/charset-repair.test.ts @@ -69,6 +69,27 @@ describe('reverseCp437Mojibake (CP437 read as CP1252)', () => { expect(reverseCp437Mojibake('Kassa')).toBeNull() expect(reverseCp437Mojibake('Företagskonto')).toBeNull() // ö (0xF6→÷) not a CP437 letter byte }) + + it('recovers the 2026-03-17 gateway-migration strings (reported prod fixtures)', () => { + // The retired Arcim Sync gateway decoded Bokio's CP437 SIE bytes as + // windows-1252 before JSON-encoding them: ö 0x94 → U+201D ”, ä 0x84 → + // U+201E „, Ä 0x8E → U+017D Ž. These four strings are the ones reported + // from the affected company's journal; all reverse losslessly. + expect(reverseCp437Mojibake('Ink”p tj„nster inom EU')).toBe('Inköp tjänster inom EU') + expect(reverseCp437Mojibake('L”neutbetalning')).toBe('Löneutbetalning') + expect(reverseCp437Mojibake('BANKTJŽNSTER')).toBe('BANKTJÄNSTER') + expect(reverseCp437Mojibake('UTLŽGG')).toBe('UTLÄGG') + }) + + it('never routes legitimate space-padded typography into the CP437 reversal (false-positive guard)', () => { + // A curly quote or ellipsis that is NOT letter-adjacent is punctuation, + // not a mangled diacritic: the artifact detector must stay quiet so + // resolveCorrectName leaves the text untouched. + expect(hasCp1252Artifact('Betalning enligt avtal ” 2024')).toBe(false) + expect(resolveCorrectName('Betalning enligt avtal ” 2024', [])).toBeNull() + expect(hasCp1252Artifact('Avvaktar underlag …')).toBe(false) + expect(resolveCorrectName('Avvaktar underlag …', [])).toBeNull() + }) }) describe('resolveCorrectName: CP437 branch', () => { diff --git a/lib/import/__tests__/sie-artifact-scan.test.ts b/lib/import/__tests__/sie-artifact-scan.test.ts new file mode 100644 index 00000000..33c21cac --- /dev/null +++ b/lib/import/__tests__/sie-artifact-scan.test.ts @@ -0,0 +1,161 @@ +/** + * Tests for the CP1252-artifact tripwire (lib/import/sie-artifact-scan.ts). + * + * The corruption under test is CP437 SIE bytes decoded as windows-1252 by an + * upstream system BEFORE the string reached this repo (the retired Arcim Sync + * gateway, 2026-03-17): o-umlaut 0x94 -> U+201D, a-umlaut 0x84 -> U+201E, + * A-umlaut 0x8E -> U+017D. The mojibake literals below are the subject being + * tested and must stay byte-exact. + */ +import { describe, it, expect } from 'vitest' +import { + scanSieForCp1252Artifacts, + formatSieArtifactWarning, + SIE_ARTIFACT_THRESHOLD, +} from '../sie-artifact-scan' +import { parseSIEFile } from '../sie-parser' +import type { SIEAccount, SIEVoucher } from '../types' + +function account(number: string, name: string): SIEAccount { + return { number, name } +} + +function voucher( + description: string, + lineDescriptions: (string | undefined)[] = [], +): SIEVoucher { + return { + series: 'A', + number: 1, + date: new Date('2024-01-15'), + description, + lines: lineDescriptions.map((d, i) => ({ + account: '1930', + amount: i % 2 === 0 ? 100 : -100, + description: d, + })), + } +} + +describe('scanSieForCp1252Artifacts', () => { + it('flags a file with artifacts in account names and voucher descriptions', () => { + const result = scanSieForCp1252Artifacts({ + accounts: [ + account('1930', 'F”retagskonto'), // ö -> U+201D + account('4056', 'Ink”p tj„nster inom EU'), // ö/ä -> U+201D/U+201E + ], + vouchers: [voucher('L”neutbetalning')], + }) + + expect(result.flagged).toBe(true) + expect(result.artifactCount).toBe(3) + expect(result.samples).toContain('Ink”p tj„nster inom EU') + expect(result.samples).toContain('L”neutbetalning') + }) + + it('counts transaction line descriptions (BANKTJŽNSTER/UTLŽGG signature)', () => { + const result = scanSieForCp1252Artifacts({ + accounts: [account('1930', 'Bank')], + vouchers: [voucher('Banktransaktion', ['BANKTJŽNSTER', 'UTLŽGG'])], + }) + + expect(result.flagged).toBe(true) + expect(result.artifactCount).toBe(2) + expect(result.samples).toEqual(['BANKTJŽNSTER', 'UTLŽGG']) + }) + + it('does not flag clean Swedish text, including legitimate space-padded typography', () => { + const result = scanSieForCp1252Artifacts({ + accounts: [ + account('1930', 'Företagskonto'), + account('1513', 'Kundfordringar – delad faktura'), // legit space-padded en dash + ], + vouchers: [voucher('Löneutbetalning', ['Inköp tjänster inom EU', 'Avvaktar underlag …'])], + }) + + expect(result.flagged).toBe(false) + expect(result.artifactCount).toBe(0) + expect(result.samples).toEqual([]) + }) + + it('stays below the threshold on a single artifact (no false alarm on one odd string)', () => { + const result = scanSieForCp1252Artifacts({ + accounts: [account('1930', 'Företagskonto')], + // One typographic apostrophe adjacent to letters is indistinguishable + // from mojibake at the single-string level; the >= 2 threshold is what + // keeps a lone occurrence from flagging the whole file. + vouchers: [voucher('Betalning McDonald’s')], + }) + + expect(result.flagged).toBe(false) + expect(result.artifactCount).toBe(1) + expect(SIE_ARTIFACT_THRESHOLD).toBe(2) + }) + + it('caps samples at three distinct strings while counting every hit', () => { + const result = scanSieForCp1252Artifacts({ + accounts: [ + account('4010', 'Ink”p material'), + account('4056', 'Ink”p tj„nster inom EU'), + account('7210', 'L”ner tj„nstem„n'), + account('7510', 'Arbetsgivaravgifter l”n'), + ], + vouchers: [voucher('L”neutbetalning'), voucher('L”neutbetalning')], + }) + + expect(result.flagged).toBe(true) + expect(result.artifactCount).toBe(6) + expect(result.samples).toHaveLength(3) + // Distinct: the duplicated voucher description appears once at most. + expect(new Set(result.samples).size).toBe(3) + }) + + it('flags real parseSIEFile output for a gateway-mojibaked SIE string', () => { + const mojibakeSie = [ + '#FLAGGA 0', + '#SIETYP 4', + '#FNAMN "Migrerad AB"', + '#RAR 0 20240101 20241231', + '#KONTO 1930 "F”retagskonto"', + '#KONTO 4056 "Ink”p tj„nster inom EU"', + '#VER A 1 20240115 "L”neutbetalning"', + '{', + '#TRANS 1930 {} -100.00', + '#TRANS 4056 {} 100.00', + '}', + ].join('\n') + + const flagged = scanSieForCp1252Artifacts(parseSIEFile(mojibakeSie)) + expect(flagged.flagged).toBe(true) + expect(flagged.artifactCount).toBe(3) + + const cleanSie = mojibakeSie + .replace('F”retagskonto', 'Företagskonto') + .replace('Ink”p tj„nster inom EU', 'Inköp tjänster inom EU') + .replace('L”neutbetalning', 'Löneutbetalning') + const clean = scanSieForCp1252Artifacts(parseSIEFile(cleanSie)) + expect(clean.flagged).toBe(false) + expect(clean.artifactCount).toBe(0) + }) +}) + +describe('formatSieArtifactWarning', () => { + it('includes the count and the first sample in Swedish', () => { + const message = formatSieArtifactWarning({ + flagged: true, + artifactCount: 3, + samples: ['Ink”p tj„nster inom EU', 'L”neutbetalning'], + }) + + expect(message).toContain('felaktigt teckenkodad') + expect(message).toContain('3 textfält') + expect(message).toContain('"Ink”p tj„nster inom EU"') + expect(message).toContain('blockeras inte') + }) + + it('omits the example clause when no sample is available', () => { + const message = formatSieArtifactWarning({ flagged: true, artifactCount: 2, samples: [] }) + expect(message).not.toContain('till exempel') + expect(message).toContain('2 textfält') + }) +}) diff --git a/lib/import/sie-artifact-scan.ts b/lib/import/sie-artifact-scan.ts new file mode 100644 index 00000000..0817c5ea --- /dev/null +++ b/lib/import/sie-artifact-scan.ts @@ -0,0 +1,92 @@ +/** + * CP1252-artifact tripwire for SIE imports. + * + * Every in-repo SIE byte path decodes correctly (detectEncoding/decodeBuffer in + * sie-parser.ts), but some import surfaces receive an ALREADY-DECODED string + * (the provider-migration /import-sie handler takes rawContent as JSON). If an + * upstream system decoded CP437 bytes as windows-1252 before handing us the + * string, the damage is already baked in: CP437 diacritics become C1 specials + * (o-umlaut 0x94 -> U+201D, a-umlaut 0x84 -> U+201E, A-umlaut 0x8E -> U+017D), + * producing text like "Ink[U+201D]p tj[U+201E]nster" and "BANKTJ[U+017D]NSTER". + * That exact corruption shipped through the retired Arcim Sync gateway on + * 2026-03-17 and sat undetected in posted entries. + * + * This scanner is the tripwire: it flags parsed SIE text carrying that + * signature so the import can WARN (never block) and the user can abort or + * review. Detection reuses hasCp1252Artifact (lib/bookkeeping/charset-repair), + * whose letter-adjacency heuristic ignores legitimate typography such as a + * space-padded en dash. A minimum of two flagged fields is required before the + * file as a whole is flagged, so a single legitimate curly quote in one + * description cannot trigger the warning. + */ + +import { hasCp1252Artifact } from '@/lib/bookkeeping/charset-repair' +import type { SIEAccount, SIEVoucher } from './types' + +/** Minimum number of artifact-carrying text fields before the file is flagged. */ +export const SIE_ARTIFACT_THRESHOLD = 2 + +/** Cap on distinct example strings carried in the result (for logs/messages). */ +const MAX_SAMPLES = 3 + +export interface SieArtifactScanResult { + /** True when artifactCount >= SIE_ARTIFACT_THRESHOLD. */ + flagged: boolean + /** Number of text fields (account names, voucher/line descriptions) with artifacts. */ + artifactCount: number + /** Up to MAX_SAMPLES distinct flagged strings, in encounter order. */ + samples: string[] +} + +/** + * Scan parsed SIE content (account names, voucher descriptions, transaction + * line descriptions) for the CP437-decoded-as-CP1252 mojibake signature. + * Pure function: no I/O, no side effects. + */ +export function scanSieForCp1252Artifacts(parsed: { + accounts: SIEAccount[] + vouchers: SIEVoucher[] +}): SieArtifactScanResult { + let artifactCount = 0 + const samples: string[] = [] + + const check = (text: string | undefined): void => { + if (!text || !hasCp1252Artifact(text)) return + artifactCount++ + if (samples.length < MAX_SAMPLES && !samples.includes(text)) { + samples.push(text) + } + } + + for (const account of parsed.accounts) { + check(account.name) + } + for (const voucher of parsed.vouchers) { + check(voucher.description) + for (const line of voucher.lines) { + check(line.description) + } + } + + return { + flagged: artifactCount >= SIE_ARTIFACT_THRESHOLD, + artifactCount, + samples, + } +} + +/** + * User-facing Swedish warning for a flagged scan. SIE import warnings are a + * Swedish-only surface (SIE is Swedish by spec); this follows the existing + * warnings-array convention in sie-parser/sie-import rather than i18n keys. + * The C1 special characters in the message are the mojibake signature itself, + * quoted literally so the user can recognize them in their data. + */ +export function formatSieArtifactWarning(scan: SieArtifactScanResult): string { + const example = scan.samples[0] ? ` (till exempel "${scan.samples[0]}")` : '' + return ( + `Filens text verkar vara felaktigt teckenkodad: ${scan.artifactCount} textfält innehåller ` + + `tecken som ”, „ eller Ž i stället för å, ä eller ö${example}. ` + + `Importen blockeras inte, men granska kontonamn och verifikationstexter.` + ) +}