fix(import): guard against CP437-as-CP1252 mojibake entering via pre-decoded SIE text (#1569)
* refactor(arcim-migration): remove the dead gateway SIE export path fetchSIEExport and SIEExportFile have had zero callers since the direct provider clients replaced the Arcim Sync gateway (#181, #718). The path returned SIE as a pre-decoded string, and the gateway's decode of CP437 bytes as windows-1252 is what wrote the 2026-03-17 mojibake into posted entries. Deleting it makes the string-typed SIE fetch impossible to re-wire; a comment marks the grave. The consent lifecycle and entity accessors stay untouched. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(import): warn when SIE text carries CP437-as-CP1252 mojibake The 2026-03-17 migration wrote mojibake ("L"neutbetalning"-style C1 specials) into posted entries because the retired gateway handed the /import-sie handler an already-decoded string: byte-level encoding detection never saw it, and nothing downstream checked. The live bug is gone; this is the tripwire so the signature can never land silently again. - lib/import/sie-artifact-scan.ts: pure scanner over parsed SIE account names and voucher/line descriptions, reusing hasCp1252Artifact from charset-repair; flags at >= 2 hits so a lone legitimate curly quote or apostrophe cannot false-positive a whole file. - arcim-migration /import-sie: warn-never-block; the Swedish warning rides on result.warnings, which the workspace UI already renders, plus a server-side log.warn. - wizard parse route: same scan, surfaced through the existing parse-issue warnings card in the preview, pointing at the first affected line. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * test(bookkeeping): pin the reported gateway mojibake strings Adds the four strings reported from the affected company's journal as reverse_cp437 cases (all reverse losslessly) plus a false-positive guard: space-padded typography must never route into the CP437 reversal. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Jakob Wennberg <311770904+jakobwennberg-oss@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jakob Wennberg
co-authored by
Claude Fable 5
Jakob Wennberg
Claude Fable 5
Jakob Wennberg
parent
c4adc8eb7d
commit
78a581bca1
@@ -69,6 +69,27 @@ describe('reverseCp437Mojibake (CP437 read as CP1252)', () => {
|
||||
expect(reverseCp437Mojibake('Kassa')).toBeNull()
|
||||
expect(reverseCp437Mojibake('Företagskonto')).toBeNull() // ö (0xF6→÷) not a CP437 letter byte
|
||||
})
|
||||
|
||||
it('recovers the 2026-03-17 gateway-migration strings (reported prod fixtures)', () => {
|
||||
// The retired Arcim Sync gateway decoded Bokio's CP437 SIE bytes as
|
||||
// windows-1252 before JSON-encoding them: ö 0x94 → U+201D ”, ä 0x84 →
|
||||
// U+201E „, Ä 0x8E → U+017D Ž. These four strings are the ones reported
|
||||
// from the affected company's journal; all reverse losslessly.
|
||||
expect(reverseCp437Mojibake('Ink”p tj„nster inom EU')).toBe('Inköp tjänster inom EU')
|
||||
expect(reverseCp437Mojibake('L”neutbetalning')).toBe('Löneutbetalning')
|
||||
expect(reverseCp437Mojibake('BANKTJŽNSTER')).toBe('BANKTJÄNSTER')
|
||||
expect(reverseCp437Mojibake('UTLŽGG')).toBe('UTLÄGG')
|
||||
})
|
||||
|
||||
it('never routes legitimate space-padded typography into the CP437 reversal (false-positive guard)', () => {
|
||||
// A curly quote or ellipsis that is NOT letter-adjacent is punctuation,
|
||||
// not a mangled diacritic: the artifact detector must stay quiet so
|
||||
// resolveCorrectName leaves the text untouched.
|
||||
expect(hasCp1252Artifact('Betalning enligt avtal ” 2024')).toBe(false)
|
||||
expect(resolveCorrectName('Betalning enligt avtal ” 2024', [])).toBeNull()
|
||||
expect(hasCp1252Artifact('Avvaktar underlag …')).toBe(false)
|
||||
expect(resolveCorrectName('Avvaktar underlag …', [])).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveCorrectName: CP437 branch', () => {
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* Tests for the CP1252-artifact tripwire (lib/import/sie-artifact-scan.ts).
|
||||
*
|
||||
* The corruption under test is CP437 SIE bytes decoded as windows-1252 by an
|
||||
* upstream system BEFORE the string reached this repo (the retired Arcim Sync
|
||||
* gateway, 2026-03-17): o-umlaut 0x94 -> U+201D, a-umlaut 0x84 -> U+201E,
|
||||
* A-umlaut 0x8E -> U+017D. The mojibake literals below are the subject being
|
||||
* tested and must stay byte-exact.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import {
|
||||
scanSieForCp1252Artifacts,
|
||||
formatSieArtifactWarning,
|
||||
SIE_ARTIFACT_THRESHOLD,
|
||||
} from '../sie-artifact-scan'
|
||||
import { parseSIEFile } from '../sie-parser'
|
||||
import type { SIEAccount, SIEVoucher } from '../types'
|
||||
|
||||
function account(number: string, name: string): SIEAccount {
|
||||
return { number, name }
|
||||
}
|
||||
|
||||
function voucher(
|
||||
description: string,
|
||||
lineDescriptions: (string | undefined)[] = [],
|
||||
): SIEVoucher {
|
||||
return {
|
||||
series: 'A',
|
||||
number: 1,
|
||||
date: new Date('2024-01-15'),
|
||||
description,
|
||||
lines: lineDescriptions.map((d, i) => ({
|
||||
account: '1930',
|
||||
amount: i % 2 === 0 ? 100 : -100,
|
||||
description: d,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
describe('scanSieForCp1252Artifacts', () => {
|
||||
it('flags a file with artifacts in account names and voucher descriptions', () => {
|
||||
const result = scanSieForCp1252Artifacts({
|
||||
accounts: [
|
||||
account('1930', 'F”retagskonto'), // ö -> U+201D
|
||||
account('4056', 'Ink”p tj„nster inom EU'), // ö/ä -> U+201D/U+201E
|
||||
],
|
||||
vouchers: [voucher('L”neutbetalning')],
|
||||
})
|
||||
|
||||
expect(result.flagged).toBe(true)
|
||||
expect(result.artifactCount).toBe(3)
|
||||
expect(result.samples).toContain('Ink”p tj„nster inom EU')
|
||||
expect(result.samples).toContain('L”neutbetalning')
|
||||
})
|
||||
|
||||
it('counts transaction line descriptions (BANKTJŽNSTER/UTLŽGG signature)', () => {
|
||||
const result = scanSieForCp1252Artifacts({
|
||||
accounts: [account('1930', 'Bank')],
|
||||
vouchers: [voucher('Banktransaktion', ['BANKTJŽNSTER', 'UTLŽGG'])],
|
||||
})
|
||||
|
||||
expect(result.flagged).toBe(true)
|
||||
expect(result.artifactCount).toBe(2)
|
||||
expect(result.samples).toEqual(['BANKTJŽNSTER', 'UTLŽGG'])
|
||||
})
|
||||
|
||||
it('does not flag clean Swedish text, including legitimate space-padded typography', () => {
|
||||
const result = scanSieForCp1252Artifacts({
|
||||
accounts: [
|
||||
account('1930', 'Företagskonto'),
|
||||
account('1513', 'Kundfordringar – delad faktura'), // legit space-padded en dash
|
||||
],
|
||||
vouchers: [voucher('Löneutbetalning', ['Inköp tjänster inom EU', 'Avvaktar underlag …'])],
|
||||
})
|
||||
|
||||
expect(result.flagged).toBe(false)
|
||||
expect(result.artifactCount).toBe(0)
|
||||
expect(result.samples).toEqual([])
|
||||
})
|
||||
|
||||
it('stays below the threshold on a single artifact (no false alarm on one odd string)', () => {
|
||||
const result = scanSieForCp1252Artifacts({
|
||||
accounts: [account('1930', 'Företagskonto')],
|
||||
// One typographic apostrophe adjacent to letters is indistinguishable
|
||||
// from mojibake at the single-string level; the >= 2 threshold is what
|
||||
// keeps a lone occurrence from flagging the whole file.
|
||||
vouchers: [voucher('Betalning McDonald’s')],
|
||||
})
|
||||
|
||||
expect(result.flagged).toBe(false)
|
||||
expect(result.artifactCount).toBe(1)
|
||||
expect(SIE_ARTIFACT_THRESHOLD).toBe(2)
|
||||
})
|
||||
|
||||
it('caps samples at three distinct strings while counting every hit', () => {
|
||||
const result = scanSieForCp1252Artifacts({
|
||||
accounts: [
|
||||
account('4010', 'Ink”p material'),
|
||||
account('4056', 'Ink”p tj„nster inom EU'),
|
||||
account('7210', 'L”ner tj„nstem„n'),
|
||||
account('7510', 'Arbetsgivaravgifter l”n'),
|
||||
],
|
||||
vouchers: [voucher('L”neutbetalning'), voucher('L”neutbetalning')],
|
||||
})
|
||||
|
||||
expect(result.flagged).toBe(true)
|
||||
expect(result.artifactCount).toBe(6)
|
||||
expect(result.samples).toHaveLength(3)
|
||||
// Distinct: the duplicated voucher description appears once at most.
|
||||
expect(new Set(result.samples).size).toBe(3)
|
||||
})
|
||||
|
||||
it('flags real parseSIEFile output for a gateway-mojibaked SIE string', () => {
|
||||
const mojibakeSie = [
|
||||
'#FLAGGA 0',
|
||||
'#SIETYP 4',
|
||||
'#FNAMN "Migrerad AB"',
|
||||
'#RAR 0 20240101 20241231',
|
||||
'#KONTO 1930 "F”retagskonto"',
|
||||
'#KONTO 4056 "Ink”p tj„nster inom EU"',
|
||||
'#VER A 1 20240115 "L”neutbetalning"',
|
||||
'{',
|
||||
'#TRANS 1930 {} -100.00',
|
||||
'#TRANS 4056 {} 100.00',
|
||||
'}',
|
||||
].join('\n')
|
||||
|
||||
const flagged = scanSieForCp1252Artifacts(parseSIEFile(mojibakeSie))
|
||||
expect(flagged.flagged).toBe(true)
|
||||
expect(flagged.artifactCount).toBe(3)
|
||||
|
||||
const cleanSie = mojibakeSie
|
||||
.replace('F”retagskonto', 'Företagskonto')
|
||||
.replace('Ink”p tj„nster inom EU', 'Inköp tjänster inom EU')
|
||||
.replace('L”neutbetalning', 'Löneutbetalning')
|
||||
const clean = scanSieForCp1252Artifacts(parseSIEFile(cleanSie))
|
||||
expect(clean.flagged).toBe(false)
|
||||
expect(clean.artifactCount).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('formatSieArtifactWarning', () => {
|
||||
it('includes the count and the first sample in Swedish', () => {
|
||||
const message = formatSieArtifactWarning({
|
||||
flagged: true,
|
||||
artifactCount: 3,
|
||||
samples: ['Ink”p tj„nster inom EU', 'L”neutbetalning'],
|
||||
})
|
||||
|
||||
expect(message).toContain('felaktigt teckenkodad')
|
||||
expect(message).toContain('3 textfält')
|
||||
expect(message).toContain('"Ink”p tj„nster inom EU"')
|
||||
expect(message).toContain('blockeras inte')
|
||||
})
|
||||
|
||||
it('omits the example clause when no sample is available', () => {
|
||||
const message = formatSieArtifactWarning({ flagged: true, artifactCount: 2, samples: [] })
|
||||
expect(message).not.toContain('till exempel')
|
||||
expect(message).toContain('2 textfält')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* CP1252-artifact tripwire for SIE imports.
|
||||
*
|
||||
* Every in-repo SIE byte path decodes correctly (detectEncoding/decodeBuffer in
|
||||
* sie-parser.ts), but some import surfaces receive an ALREADY-DECODED string
|
||||
* (the provider-migration /import-sie handler takes rawContent as JSON). If an
|
||||
* upstream system decoded CP437 bytes as windows-1252 before handing us the
|
||||
* string, the damage is already baked in: CP437 diacritics become C1 specials
|
||||
* (o-umlaut 0x94 -> U+201D, a-umlaut 0x84 -> U+201E, A-umlaut 0x8E -> U+017D),
|
||||
* producing text like "Ink[U+201D]p tj[U+201E]nster" and "BANKTJ[U+017D]NSTER".
|
||||
* That exact corruption shipped through the retired Arcim Sync gateway on
|
||||
* 2026-03-17 and sat undetected in posted entries.
|
||||
*
|
||||
* This scanner is the tripwire: it flags parsed SIE text carrying that
|
||||
* signature so the import can WARN (never block) and the user can abort or
|
||||
* review. Detection reuses hasCp1252Artifact (lib/bookkeeping/charset-repair),
|
||||
* whose letter-adjacency heuristic ignores legitimate typography such as a
|
||||
* space-padded en dash. A minimum of two flagged fields is required before the
|
||||
* file as a whole is flagged, so a single legitimate curly quote in one
|
||||
* description cannot trigger the warning.
|
||||
*/
|
||||
|
||||
import { hasCp1252Artifact } from '@/lib/bookkeeping/charset-repair'
|
||||
import type { SIEAccount, SIEVoucher } from './types'
|
||||
|
||||
/** Minimum number of artifact-carrying text fields before the file is flagged. */
|
||||
export const SIE_ARTIFACT_THRESHOLD = 2
|
||||
|
||||
/** Cap on distinct example strings carried in the result (for logs/messages). */
|
||||
const MAX_SAMPLES = 3
|
||||
|
||||
export interface SieArtifactScanResult {
|
||||
/** True when artifactCount >= SIE_ARTIFACT_THRESHOLD. */
|
||||
flagged: boolean
|
||||
/** Number of text fields (account names, voucher/line descriptions) with artifacts. */
|
||||
artifactCount: number
|
||||
/** Up to MAX_SAMPLES distinct flagged strings, in encounter order. */
|
||||
samples: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan parsed SIE content (account names, voucher descriptions, transaction
|
||||
* line descriptions) for the CP437-decoded-as-CP1252 mojibake signature.
|
||||
* Pure function: no I/O, no side effects.
|
||||
*/
|
||||
export function scanSieForCp1252Artifacts(parsed: {
|
||||
accounts: SIEAccount[]
|
||||
vouchers: SIEVoucher[]
|
||||
}): SieArtifactScanResult {
|
||||
let artifactCount = 0
|
||||
const samples: string[] = []
|
||||
|
||||
const check = (text: string | undefined): void => {
|
||||
if (!text || !hasCp1252Artifact(text)) return
|
||||
artifactCount++
|
||||
if (samples.length < MAX_SAMPLES && !samples.includes(text)) {
|
||||
samples.push(text)
|
||||
}
|
||||
}
|
||||
|
||||
for (const account of parsed.accounts) {
|
||||
check(account.name)
|
||||
}
|
||||
for (const voucher of parsed.vouchers) {
|
||||
check(voucher.description)
|
||||
for (const line of voucher.lines) {
|
||||
check(line.description)
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
flagged: artifactCount >= SIE_ARTIFACT_THRESHOLD,
|
||||
artifactCount,
|
||||
samples,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* User-facing Swedish warning for a flagged scan. SIE import warnings are a
|
||||
* Swedish-only surface (SIE is Swedish by spec); this follows the existing
|
||||
* warnings-array convention in sie-parser/sie-import rather than i18n keys.
|
||||
* The C1 special characters in the message are the mojibake signature itself,
|
||||
* quoted literally so the user can recognize them in their data.
|
||||
*/
|
||||
export function formatSieArtifactWarning(scan: SieArtifactScanResult): string {
|
||||
const example = scan.samples[0] ? ` (till exempel "${scan.samples[0]}")` : ''
|
||||
return (
|
||||
`Filens text verkar vara felaktigt teckenkodad: ${scan.artifactCount} textfält innehåller ` +
|
||||
`tecken som ”, „ eller Ž i stället för å, ä eller ö${example}. ` +
|
||||
`Importen blockeras inte, men granska kontonamn och verifikationstexter.`
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user