fix(import): guard against CP437-as-CP1252 mojibake entering via pre-decoded SIE text (#1569)

* refactor(arcim-migration): remove the dead gateway SIE export path fetchSIEExport and SIEExportFile have had zero callers since the direct provider clients replaced the Arcim Sync gateway (#181, #718). The path returned SIE as a pre-decoded string, and the gateway's decode of CP437 bytes as windows-1252 is what wrote the 2026-03-17 mojibake into posted entries. Deleting it makes the string-typed SIE fetch impossible to re-wire; a comment marks the grave. The consent lifecycle and entity accessors stay untouched. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(import): warn when SIE text carries CP437-as-CP1252 mojibake The 2026-03-17 migration wrote mojibake ("L"neutbetalning"-style C1 specials) into posted entries because the retired gateway handed the /import-sie handler an already-decoded string: byte-level encoding detection never saw it, and nothing downstream checked. The live bug is gone; this is the tripwire so the signature can never land silently again. - lib/import/sie-artifact-scan.ts: pure scanner over parsed SIE account names and voucher/line descriptions, reusing hasCp1252Artifact from charset-repair; flags at >= 2 hits so a lone legitimate curly quote or apostrophe cannot false-positive a whole file. - arcim-migration /import-sie: warn-never-block; the Swedish warning rides on result.warnings, which the workspace UI already renders, plus a server-side log.warn. - wizard parse route: same scan, surfaced through the existing parse-issue warnings card in the preview, pointing at the first affected line. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * test(bookkeeping): pin the reported gateway mojibake strings Adds the four strings reported from the affected company's journal as reverse_cp437 cases (all reverse losslessly) plus a false-positive guard: space-padded typography must never route into the CP437 reversal. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-13 15:14:23 +02:00
parent c4adc8eb7d
commit 78a581bca1
8 changed files with 618 additions and 21 deletions
@@ -0,0 +1,146 @@
+/**
+ * Tests for the CP1252-mojibake tripwire on POST /import-sie.
+ *
+ * This handler receives SIE as an ALREADY-DECODED string (rawContent in JSON),
+ * so an upstream that decoded CP437 bytes as windows-1252 has baked the
+ * corruption in before the repo's own encoding detection could run: exactly
+ * what the retired Arcim Sync gateway did on 2026-03-17, landing mojibake
+ * ("L”neutbetalning", "BANKTJŽNSTER") in posted entries with no warning.
+ *
+ * The tripwire must WARN and NEVER BLOCK: the import proceeds untouched and
+ * the warning rides on result.warnings, which the migration workspace UI
+ * already renders. The mojibake literals below are the subject under test and
+ * must stay byte-exact.
+ */
+import { describe, it, expect, beforeEach, vi, type Mock } from 'vitest'
+import { createMockRequest, parseJsonResponse } from '@/tests/helpers'
+import type { ExtensionContext } from '@/lib/extensions/types'
+
+vi.mock('@/lib/import/sie-import', () => ({
+  loadMappings: vi.fn(),
+  generateImportPreview: vi.fn(),
+  executeSIEImport: vi.fn(),
+}))
+
+import { arcimMigrationExtension } from '../index'
+import { executeSIEImport } from '@/lib/import/sie-import'
+
+const importSieRoute = (arcimMigrationExtension.apiRoutes ?? []).find(
+  (r) => r.method === 'POST' && r.path === '/import-sie',
+)!
+
+type RouteHandler = (request: Request, ctx?: ExtensionContext) => Promise<Response>
+const handler = importSieRoute.handler as RouteHandler
+
+const MOJIBAKE_SIE = [
+  '#FLAGGA 0',
+  '#SIETYP 4',
+  '#FNAMN "Migrerad AB"',
+  '#RAR 0 20240101 20241231',
+  '#KONTO 1930 "F”retagskonto"',
+  '#KONTO 4056 "Ink”p tj„nster inom EU"',
+  '#VER A 1 20240115 "L”neutbetalning"',
+  '{',
+  '#TRANS 1930 {} -100.00',
+  '#TRANS 4056 {} 100.00',
+  '}',
+].join('\n')
+
+const CLEAN_SIE = MOJIBAKE_SIE
+  .replace('F”retagskonto', 'Företagskonto')
+  .replace('Ink”p tj„nster inom EU', 'Inköp tjänster inom EU')
+  .replace('L”neutbetalning', 'Löneutbetalning')
+
+const MAPPINGS = [
+  { sourceAccount: '1930', sourceName: 'Företagskonto', targetAccount: '1930' },
+  { sourceAccount: '4056', sourceName: 'Inköp tjänster inom EU', targetAccount: '4056' },
+]
+
+function buildCtx(userId: string | null) {
+  const log = { info: vi.fn(), warn: vi.fn(), error: vi.fn() }
+  const supabase = {
+    auth: {
+      getUser: vi.fn().mockResolvedValue({ data: { user: userId ? { id: userId } : null } }),
+    },
+  }
+  const ctx = { supabase, companyId: 'company-1', log } as unknown as ExtensionContext
+  return { ctx, log }
+}
+
+function importRequest(body: Record<string, unknown>) {
+  return createMockRequest('http://localhost/api/extensions/ext/arcim-migration/import-sie', {
+    method: 'POST',
+    body,
+  })
+}
+
+describe('POST /import-sie: mojibake tripwire', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    // Fresh result object per call: the handler pushes onto result.warnings.
+    ;(executeSIEImport as Mock).mockImplementation(async () => ({
+      success: true,
+      journalEntriesCreated: 1,
+      errors: [],
+      warnings: [],
+    }))
+  })
+
+  it('returns 401 when unauthenticated', async () => {
+    const { ctx } = buildCtx(null)
+    const response = await handler(
+      importRequest({ rawContent: CLEAN_SIE, mappings: MAPPINGS, options: {} }),
+      ctx,
+    )
+    expect(response.status).toBe(401)
+    expect(executeSIEImport).not.toHaveBeenCalled()
+  })
+
+  it('returns 400 when rawContent is missing', async () => {
+    const { ctx } = buildCtx('user-1')
+    const response = await handler(importRequest({ mappings: MAPPINGS, options: {} }), ctx)
+    expect(response.status).toBe(400)
+    expect(executeSIEImport).not.toHaveBeenCalled()
+  })
+
+  it('surfaces a Swedish warning on mojibaked content WITHOUT blocking the import', async () => {
+    const { ctx, log } = buildCtx('user-1')
+    const response = await handler(
+      importRequest({ rawContent: MOJIBAKE_SIE, mappings: MAPPINGS, options: {} }),
+      ctx,
+    )
+    const { status, body } = await parseJsonResponse<{ warnings: string[] }>(response)
+
+    expect(status).toBe(200)
+    // Warn, never block: the import ran despite the flagged content.
+    expect(executeSIEImport).toHaveBeenCalledTimes(1)
+
+    const tripwire = body.warnings.find((w) => w.includes('felaktigt teckenkodad'))
+    expect(tripwire).toBeDefined()
+    // The first flagged string (accounts are scanned first) is the example.
+    expect(tripwire).toContain('F”retagskonto')
+    // Must dodge the workspace UI's structured-card filters, or the warning
+    // silently disappears from the "Remaining warnings" card.
+    expect(tripwire).not.toContain('hoppades över')
+    expect(tripwire).not.toContain('förts om till eget kapital')
+
+    expect(log.warn).toHaveBeenCalledWith(
+      'import-sie: CP1252 mojibake artifacts in SIE text',
+      expect.objectContaining({ artifactCount: 3 }),
+    )
+  })
+
+  it('adds no warning for clean content', async () => {
+    const { ctx, log } = buildCtx('user-1')
+    const response = await handler(
+      importRequest({ rawContent: CLEAN_SIE, mappings: MAPPINGS, options: {} }),
+      ctx,
+    )
+    const { status, body } = await parseJsonResponse<{ warnings: string[] }>(response)
+
+    expect(status).toBe(200)
+    expect(executeSIEImport).toHaveBeenCalledTimes(1)
+    expect(body.warnings.some((w) => w.includes('felaktigt teckenkodad'))).toBe(false)
+    expect(log.warn).not.toHaveBeenCalled()
+  })
+})
@@ -28,6 +28,7 @@ import { reconcileSupplierInvoiceVouchers } from '@/lib/invoices/bulk-reconcile-
 import type { ArcimProvider } from './types'
 import { ARCIM_PROVIDERS } from './types'
 import { parseSIEFile, validateSIEFile } from '@/lib/import/sie-parser'
+import { scanSieForCp1252Artifacts, formatSieArtifactWarning } from '@/lib/import/sie-artifact-scan'
 import { suggestMappings, getMappingStats, isSystemAccount } from '@/lib/import/account-mapper'
 import { loadMappings, generateImportPreview, executeSIEImport } from '@/lib/import/sie-import'
 import { BAS_REFERENCE } from '@/lib/bookkeeping/bas-reference'
@@ -995,6 +996,20 @@ export const arcimMigrationExtension: Extension = {
        try {
          const parsed = parseSIEFile(rawContent)

+          // Mojibake tripwire (warn, never block). This handler receives SIE
+          // as an ALREADY-DECODED string, so an upstream that decoded CP437
+          // bytes as windows-1252 has baked the corruption in before we ever
+          // see it (the retired Arcim Sync gateway did exactly that on
+          // 2026-03-17). Flag the signature so it can never again land
+          // silently in posted entries; the import itself proceeds untouched.
+          const artifactScan = scanSieForCp1252Artifacts(parsed)
+          if (artifactScan.flagged) {
+            log.warn('import-sie: CP1252 mojibake artifacts in SIE text', {
+              artifactCount: artifactScan.artifactCount,
+              samples: artifactScan.samples,
+            })
+          }
+
          // Validate all accounts are mapped (same as manual upload)
          const unmapped = mappings.filter((m: import('@/lib/import/types').AccountMapping) => !m.targetAccount)
          if (unmapped.length > 0) {
@@ -1035,6 +1050,12 @@ export const arcimMigrationExtension: Extension = {
            onExistingPeriod: 'replace',
          })

+          // Surface the tripwire on the result the workspace UI already
+          // renders (its "Remaining warnings" card shows result.warnings).
+          if (artifactScan.flagged) {
+            result.warnings.push(formatSieArtifactWarning(artifactScan))
+          }
+
          log.info('SIE import completed:', {
            success: result.success,
            journalEntriesCreated: result.journalEntriesCreated,
@@ -233,24 +233,8 @@ export async function fetchSupplierInvoices(
  return fetchAllPages<SupplierInvoiceDto>(consentId, 'supplierinvoices', params)
 }

-// ── SIE export ────────────────────────────────────────────────────
-
-export interface SIEExportFile {
-  fiscalYear: number
-  sieType: number
-  rawContent: string
-  accountCount: number
-  transactionCount: number
-}
-
-export async function fetchSIEExport(
-  consentId: string,
-  sieType?: number
-): Promise<{ files: SIEExportFile[] }> {
-  const params = new URLSearchParams()
-  if (sieType) params.set('sieType', String(sieType))
-  const qs = params.toString()
-  return request<{ files: SIEExportFile[] }>(
-    `/api/v1/consents/${consentId}/sie/export${qs ? `?${qs}` : ''}`
-  )
-}
+// The gateway SIE export path (fetchSIEExport/SIEExportFile) was deliberately
+// removed: it returned SIE as a pre-decoded string, and the gateway's decode of
+// CP437 bytes as windows-1252 caused the 2026-03-17 mojibake incident. Provider
+// SIE now travels as raw bytes through lib/sie-fetcher.ts and the repo's own
+// encoding detection. Do not re-add a string-typed SIE fetch here.