From dec920682f050b563a4ff3de6d77caab34c4b9c5 Mon Sep 17 00:00:00 2001 From: Mattsson <111893710+mattssonn@users.noreply.github.com> Date: Mon, 11 May 2026 23:18:17 +0200 Subject: [PATCH] Fix/UI changes (#439) * feat(bookkeeping): add preview for next voucher number in JournalEntryForm * feat(encoding): implement U+FFFD recovery for Swedish text in encoding functions --- app/(dashboard)/page.tsx | 9 +- .../voucher-sequences/next/route.ts | 41 +- components/bookkeeping/JournalEntryForm.tsx | 43 ++- lib/import/__tests__/sie-parser.test.ts | 47 +++ lib/import/shared/__tests__/encoding.test.ts | 74 ++++ lib/import/shared/encoding.ts | 153 ++++++++ lib/import/sie-parser.ts | 32 +- scripts/fix-replacement-chars.ts | 352 ++++++++++++++++++ 8 files changed, 728 insertions(+), 23 deletions(-) create mode 100644 scripts/fix-replacement-chars.ts diff --git a/app/(dashboard)/page.tsx b/app/(dashboard)/page.tsx index 919dc621..6b2ceded 100644 --- a/app/(dashboard)/page.tsx +++ b/app/(dashboard)/page.tsx @@ -3,6 +3,7 @@ import { redirect } from 'next/navigation' import { cookies } from 'next/headers' import DashboardContent from '@/components/dashboard/DashboardContent' import { getActiveCompanyId } from '@/lib/company/context' +import { getDisplayTotal } from '@/lib/invoices/rounding' import type { Deadline, ReceiptQueueSummary, OnboardingProgress } from '@/types' export const dynamic = 'force-dynamic' @@ -159,8 +160,14 @@ export default async function DashboardPage() { .filter((t) => t.amount < 0) .reduce((sum, t) => sum + Math.abs(Number(t.amount_sek || t.amount)), 0) + // Mirror the per-invoice öresavrundning rule used on the invoice list/detail + // pages: sum the displayed (rounded) SEK amount per invoice so the dashboard + // total matches what the user sees on the invoice list when the setting is on. const unpaidTotal = (unpaidInvoices || []).reduce( - (sum, inv) => sum + Number(inv.total_sek || inv.total), + (sum, inv) => sum + getDisplayTotal( + { total: Number(inv.total_sek || inv.total), currency: 'SEK' }, + settings, + ).displayed, 0 ) diff --git a/app/api/bookkeeping/voucher-sequences/next/route.ts b/app/api/bookkeeping/voucher-sequences/next/route.ts index 76d174ab..36361246 100644 --- a/app/api/bookkeeping/voucher-sequences/next/route.ts +++ b/app/api/bookkeeping/voucher-sequences/next/route.ts @@ -4,25 +4,38 @@ import { errorResponse } from '@/lib/errors/get-structured-error' export const GET = withRouteContext( 'voucher_sequence.next', - async (_request, ctx) => { + async (request, ctx) => { const { supabase, companyId, log, requestId } = ctx + const url = new URL(request.url) + const overridePeriodId = url.searchParams.get('period_id') + const overrideSeries = url.searchParams.get('series') + const today = new Date().toISOString().split('T')[0] const [{ data: period, error: periodError }, { data: settings, error: settingsError }] = await Promise.all([ - supabase - .from('fiscal_periods') - .select('id') - .eq('company_id', companyId) - .lte('period_start', today) - .gte('period_end', today) - .maybeSingle(), - supabase - .from('company_settings') - .select('default_voucher_series') - .eq('company_id', companyId) - .maybeSingle(), + overridePeriodId + ? supabase + .from('fiscal_periods') + .select('id') + .eq('company_id', companyId) + .eq('id', overridePeriodId) + .maybeSingle() + : supabase + .from('fiscal_periods') + .select('id') + .eq('company_id', companyId) + .lte('period_start', today) + .gte('period_end', today) + .maybeSingle(), + overrideSeries + ? Promise.resolve({ data: null, error: null }) + : supabase + .from('company_settings') + .select('default_voucher_series') + .eq('company_id', companyId) + .maybeSingle(), ]) if (periodError) { @@ -34,7 +47,7 @@ export const GET = withRouteContext( return errorResponse(settingsError, log, { requestId }) } - const series = settings?.default_voucher_series || 'A' + const series = overrideSeries || settings?.default_voucher_series || 'A' if (!period) { return NextResponse.json({ data: { next: null, series, fiscal_period_id: null } }) diff --git a/components/bookkeeping/JournalEntryForm.tsx b/components/bookkeeping/JournalEntryForm.tsx index 5519ed98..e8ad2cbf 100644 --- a/components/bookkeeping/JournalEntryForm.tsx +++ b/components/bookkeeping/JournalEntryForm.tsx @@ -86,6 +86,7 @@ export default function JournalEntryForm({ initialLines ?? [{ ...BLANK_LINE }, { ...BLANK_LINE }] ) const [voucherSeries, setVoucherSeries] = useState('A') + const [nextVoucherNumber, setNextVoucherNumber] = useState(null) const [isSubmitting, setIsSubmitting] = useState(false) const [showReview, setShowReview] = useState(false) const [showNoDocWarning, setShowNoDocWarning] = useState(false) @@ -157,6 +158,31 @@ export default function JournalEntryForm({ } }, [entryDate, periods]) + // Preview the upcoming voucher number for the selected period + series. + // Read-only hint; the actual number is reserved atomically at commit time, + // so this may shift by one if another entry lands first. + useEffect(() => { + if (embedded || !selectedPeriod || !voucherSeries) { + setNextVoucherNumber(null) + return + } + let cancelled = false + const qs = new URLSearchParams({ period_id: selectedPeriod, series: voucherSeries }) + fetch(`/api/bookkeeping/voucher-sequences/next?${qs}`) + .then((r) => (r.ok ? r.json() : null)) + .then((body) => { + if (cancelled) return + const next = body?.data?.next + setNextVoucherNumber(typeof next === 'number' ? next : null) + }) + .catch(() => { + if (!cancelled) setNextVoucherNumber(null) + }) + return () => { + cancelled = true + } + }, [embedded, selectedPeriod, voucherSeries]) + // Fetch exchange rate from Riksbanken when currency changes const fetchRate = useCallback(async (currency: Currency) => { if (currency === 'SEK') return @@ -447,8 +473,15 @@ export default function JournalEntryForm({ { - const v = e.target.value.toUpperCase().replace(/[^A-Z]/g, '').slice(0, 1) - setVoucherSeries(v || 'A') + const v = e.target.value.toUpperCase().replace(/[^A-Z]/g, '').slice(-1) + setVoucherSeries(v) + }} + onFocus={(e) => { + const target = e.target + setTimeout(() => target.select(), 0) + }} + onBlur={() => { + if (!voucherSeries) setVoucherSeries('A') }} className="mt-1 text-center font-mono" maxLength={1} @@ -792,7 +825,11 @@ export default function JournalEntryForm({ onOpenChange={setShowReview} onConfirm={handleConfirm} isSubmitting={isSubmitting} - title="Granska verifikation" + title={ + !embedded && nextVoucherNumber != null + ? `Granska verifikation (${voucherSeries}${nextVoucherNumber})` + : 'Granska verifikation' + } warningText={embedded ? '' : 'En verifikation skapas och kan inte ändras efteråt. Korrigeringar görs genom storno.'} > { }) }) +// --- Defensive encoding: handle files where the detector picks the wrong encoding --- + +describe('detectEncoding — full-buffer scan', () => { + it('detects Win-1252 even when Swedish chars appear past the legacy 4KB sample boundary', () => { + // Build a buffer where the first 8000 bytes are pure ASCII header + filler, + // and the Swedish Win-1252 byte appears only at byte 8000+. The old + // implementation sampled the first 4000 bytes and would default to UTF-8. + const filler = new Uint8Array(8000).fill(0x20) // spaces + const tail = new Uint8Array([ + 0x46, 0x4f, 0x52, 0x45, 0x4e, 0x49, 0x4e, 0x47, // FORENING + 0xd6, // Ö in Win-1252 (0xD6) — invalid lone UTF-8 byte + ]) + const buf = new Uint8Array(filler.length + tail.length) + buf.set(filler, 0) + buf.set(tail, filler.length) + const encoding = detectEncoding(buf.buffer) + expect(encoding).toBe('windows1252') + }) +}) + +describe('decodeBuffer — fallback on U+FFFD', () => { + it('falls back from utf8 to windows1252 when the result has replacement characters', () => { + // "F" "Ö" "RENING" in Windows-1252 — Ö is lone byte 0xD6, not valid UTF-8 + const buf = new Uint8Array([0x46, 0xd6, 0x52, 0x45, 0x4e, 0x49, 0x4e, 0x47]) + const result = decodeBuffer(buf.buffer, 'utf8') + expect(result).toBe('FÖRENING') + expect(result.includes('\uFFFD')).toBe(false) + }) + + it('falls back from utf8 to cp437 when both windows1252 also fails', () => { + // 0x94 is "ö" in CP437; in Win-1252 it's an unprintable "" but textually different; + // in UTF-8 it's invalid → U+FFFD. Verify CP437 path is reachable when chosen wrong. + const buf = new Uint8Array([0x66, 0x94, 0x72]) // f + ö-cp437 + r + const result = decodeBuffer(buf.buffer, 'utf8') + // Either windows1252 or cp437 fallback produces a non-FFFD result; both are + // acceptable here since the byte 0x94 is interpretable in both — what matters + // is no U+FFFD leaks through. + expect(result.includes('\uFFFD')).toBe(false) + }) + + it('returns primary decode unchanged when it contains no U+FFFD', () => { + const buf = new TextEncoder().encode('Företag').buffer + const result = decodeBuffer(buf, 'utf8') + expect(result).toBe('Företag') + }) +}) + // --- Fix 3: Invalid date rejection --- describe('parseSIEFile — invalid date handling', () => { diff --git a/lib/import/shared/__tests__/encoding.test.ts b/lib/import/shared/__tests__/encoding.test.ts index 13e5a479..1c5babeb 100644 --- a/lib/import/shared/__tests__/encoding.test.ts +++ b/lib/import/shared/__tests__/encoding.test.ts @@ -3,6 +3,8 @@ import { decodeFileContent, decodeStringContent, hasEncodingIssues, + recoverStringWithFFFD, + recoverWordWithFFFD, } from '../encoding' describe('decodeStringContent', () => { @@ -77,3 +79,75 @@ describe('decodeFileContent', () => { expect(decodeFileContent(cp1252)).toBe('GÖTEBORG') }) }) + +// --- U+FFFD heuristic recovery --- + +describe('recoverWordWithFFFD', () => { + it('recovers uppercase Ö in common Swedish stems', () => { + expect(recoverWordWithFFFD('F\uFFFDRENING')).toBe('FÖRENING') + expect(recoverWordWithFFFD('F\uFFFDRETAG')).toBe('FÖRETAG') + expect(recoverWordWithFFFD('G\uFFFDTEBORG')).toBe('GÖTEBORG') + expect(recoverWordWithFFFD('LINK\uFFFDPING')).toBe('LINKÖPING') + }) + + it('recovers lowercase ö in common Swedish stems', () => { + expect(recoverWordWithFFFD('f\uFFFDrening')).toBe('förening') + expect(recoverWordWithFFFD('malm\uFFFD')).toBe('malmö') + expect(recoverWordWithFFFD('k\uFFFDp')).toBe('köp') + }) + + it('recovers compound words via substring match', () => { + expect(recoverWordWithFFFD('BOSTADSR\uFFFDTTSF\uFFFDRENING')).toBe( + 'BOSTADSRÄTTSFÖRENING' + ) + expect(recoverWordWithFFFD('Idrottsf\uFFFDrening')).toBe('Idrottsförening') + }) + + it('is a no-op when the input has no U+FFFD', () => { + expect(recoverWordWithFFFD('FÖRENING')).toBe('FÖRENING') + expect(recoverWordWithFFFD('hello')).toBe('hello') + }) + + it('returns null for ambiguous words not in the dictionary', () => { + // Random 4-letter word with U+FFFD; no Swedish stem hits. + expect(recoverWordWithFFFD('Z\uFFFDXQ')).toBeNull() + }) + + it('returns null for words with too many U+FFFDs to disambiguate', () => { + expect( + recoverWordWithFFFD('\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD') + ).toBeNull() + }) +}) + +describe('recoverStringWithFFFD', () => { + it('repairs the canonical "Levbet FÖRENING" case', () => { + expect(recoverStringWithFFFD('Levbet F\uFFFDRENING')).toBe('Levbet FÖRENING') + }) + + it('repairs city + business-name combos', () => { + expect(recoverStringWithFFFD('Sjöberg AB, Malm\uFFFD')).toBe('Sjöberg AB, Malmö') + expect(recoverStringWithFFFD('Faktura fr\uFFFDn G\uFFFDTEBORG AB')).toBe( + 'Faktura från GÖTEBORG AB' + ) + }) + + it('preserves punctuation, whitespace, and digits', () => { + expect(recoverStringWithFFFD('K\uFFFDp 1 234,56 SEK')).toBe('Köp 1 234,56 SEK') + }) + + it('is a no-op on clean strings', () => { + expect(recoverStringWithFFFD('Hello World')).toBe('Hello World') + expect(recoverStringWithFFFD('FÖRENING')).toBe('FÖRENING') + }) + + it('returns null when any word in the string is ambiguous', () => { + expect(recoverStringWithFFFD('FÖRENING Z\uFFFDXQ')).toBeNull() + }) + + it('is idempotent on recovered output', () => { + const once = recoverStringWithFFFD('F\uFFFDRENING') + expect(once).toBe('FÖRENING') + expect(recoverStringWithFFFD(once!)).toBe('FÖRENING') + }) +}) diff --git a/lib/import/shared/encoding.ts b/lib/import/shared/encoding.ts index a05d6bae..3b8cd082 100644 --- a/lib/import/shared/encoding.ts +++ b/lib/import/shared/encoding.ts @@ -87,3 +87,156 @@ export function stripBOM(content: string): string { export function prepareContent(content: string): string { return normalizeLineEndings(stripBOM(decodeStringContent(content))) } + +/** + * --- U+FFFD heuristic recovery --- + * + * When Windows-1252 / Latin-1 bytes are decoded as UTF-8 with `fatal: false`, + * invalid sequences silently become U+FFFD. The original byte is lost — but + * for Swedish text we can guess from context: the missing letter is almost + * always one of Å/Ä/Ö (uppercase context) or å/ä/ö (lowercase context). + * + * This recovery tries each Swedish vowel substitution and scores the resulting + * word against a small dictionary of Swedish stems. If exactly one candidate + * scores above the threshold, it's applied. Ambiguous cases return null and + * must be reviewed manually. + */ + +const REPLACEMENT = '\uFFFD' +const SWEDISH_VOWELS_UPPER = ['Å', 'Ä', 'Ö'] as const +const SWEDISH_VOWELS_LOWER = ['å', 'ä', 'ö'] as const + +/** + * Stems of common Swedish words containing åäö that appear in business names, + * place names, addresses, and accounting descriptions. + */ +const SWEDISH_STEMS = new Set([ + // -för- prefix (extremely common) + 'för', 'före', 'förening', 'företag', 'försäkring', 'försäljning', + 'försäljnings', 'förskola', 'församling', 'förvaltning', 'förbund', + 'förlag', 'föräldra', 'försök', 'förbrukning', 'förbättring', 'förskott', + 'försening', 'förhandling', 'förbättrings', + // domain terms + 'bostadsrätt', 'rätt', 'samfällighet', 'idrott', 'fastighet', 'utbildning', + 'näring', 'växel', 'värme', 'köp', 'köpa', 'inköp', 'sälja', 'säljs', + // accounting + 'kostnad', 'kostnader', 'intäkt', 'intäkter', 'avskrivning', 'avsättning', + 'lön', 'lönekostnad', 'pension', 'utgående', 'ingående', 'momspliktig', + 'redovisning', 'företagskonto', 'bankkonto', 'överavskrivning', + 'överskott', 'underskott', 'överföring', 'överlåtelse', 'återbetalning', + 'utlägg', 'utgift', + // common short prepositions and adverbs + 'från', 'för', 'över', 'är', 'när', 'där', 'även', 'någon', 'något', + 'många', 'själv', 'små', 'väg', 'gång', 'tjänst', 'tjänster', 'räkning', + 'räntor', 'år', + // common cities + 'göteborg', 'malmö', 'örebro', 'östersund', 'jönköping', 'linköping', + 'norrköping', 'lidköping', 'köping', 'helsingborg', 'umeå', 'skellefteå', + 'piteå', 'luleå', 'borås', 'växjö', 'östhammar', 'södertälje', 'västerås', + 'härnösand', 'värnamo', 'mölndal', 'mörrum', 'mönsterås', 'färjestaden', + 'eskilstuna', + // directions / common geo terms + 'östra', 'västra', 'södra', 'norra', 'öster', 'väster', 'söder', + // legal forms + 'aktiebolag', 'handelsbolag', 'ekonomisk', 'allmännyttig', + // misc + 'företagsledare', 'koncernbidrag', 'utländsk', 'utländska', 'främmande', + 'vägen', 'gatan', 'allén', 'gränden', 'torget', + // surnames containing åäö + 'lindström', 'sjöberg', 'söderberg', 'öberg', 'åström', 'åkerlund', + 'östlund', 'lindgren', 'sjögren', 'hägglund', 'bäckström', +]) + +/** + * Score a candidate word. + * - 1000 if the entire word matches a known stem (highest confidence). + * - Otherwise the count of distinct stems that appear as substrings. + * Counting (not boolean-returning) is required: when the same word has + * multiple U+FFFD positions, the correct combination must outscore wrong + * combinations that still happen to contain *one* stem each. + */ +function scoreCandidate(word: string): number { + const lower = word.toLowerCase() + if (SWEDISH_STEMS.has(lower)) return 1000 + let score = 0 + for (const stem of SWEDISH_STEMS) { + if (lower.includes(stem)) score++ + } + return score +} + +function chooseCase(word: string): 'upper' | 'lower' { + let upper = 0 + let lower = 0 + for (const ch of word) { + if (ch === REPLACEMENT) continue + if (ch >= 'A' && ch <= 'Z') upper++ + else if (ch >= 'a' && ch <= 'z') lower++ + } + return upper > lower ? 'upper' : 'lower' +} + +/** + * Try every Swedish-vowel substitution for the U+FFFDs in `word`, then return + * the highest-scoring candidate. Returns null if no candidate matches a + * dictionary stem (i.e. ambiguous — operator must review). + */ +export function recoverWordWithFFFD(word: string): string | null { + if (!word.includes(REPLACEMENT)) return word + + const vowels = chooseCase(word) === 'upper' ? SWEDISH_VOWELS_UPPER : SWEDISH_VOWELS_LOWER + const positions: number[] = [] + for (let i = 0; i < word.length; i++) { + if (word[i] === REPLACEMENT) positions.push(i) + } + + // Words with more than ~6 lost bytes blow up the combinatorial space — + // bail out rather than spend cycles on something that's likely garbage anyway. + const totalCombos = Math.pow(vowels.length, positions.length) + if (totalCombos > 729) return null + + let best: { word: string; score: number } | null = null + for (let combo = 0; combo < totalCombos; combo++) { + const chars = word.split('') + let c = combo + for (const pos of positions) { + chars[pos] = vowels[c % vowels.length] + c = Math.floor(c / vowels.length) + } + const candidate = chars.join('') + const score = scoreCandidate(candidate) + if (!best || score > best.score) { + best = { word: candidate, score } + } + } + + if (!best || best.score === 0) return null + return best.word +} + +/** + * Repair every U+FFFD-containing word in `text` via dictionary-backed + * substitution. Returns the repaired string when *every* corrupted word + * resolved to a confident candidate; returns null if any word remains + * ambiguous (operator must review the whole string by hand). + * + * Idempotent on clean input. + */ +export function recoverStringWithFFFD(text: string): string | null { + if (!text.includes(REPLACEMENT)) return text + + // Split on runs of non-letter/non-digit characters so punctuation, + // whitespace, and structural characters are preserved as-is. + const tokens = text.split(/([^\p{L}\p{N}\uFFFD]+)/u) + const out: string[] = [] + for (const token of tokens) { + if (!token.includes(REPLACEMENT)) { + out.push(token) + continue + } + const recovered = recoverWordWithFFFD(token) + if (recovered === null) return null + out.push(recovered) + } + return out.join('') +} diff --git a/lib/import/sie-parser.ts b/lib/import/sie-parser.ts index a7b417e2..4f3677cd 100644 --- a/lib/import/sie-parser.ts +++ b/lib/import/sie-parser.ts @@ -85,6 +85,10 @@ const WIN1252_SWEDISH_BYTES = new Set([ * so presence in one range rules out the other. * 4. UTF-8 multi-byte sequences (0xC3 + continuation) are detected with proper * skipping of continuation bytes to avoid false CP437 counts. + * + * Scans the entire buffer (not a sample): SIE files are capped at 50 MB and + * Swedish characters often only appear deep in voucher descriptions, well past + * any small header sample. */ export function detectEncoding(buffer: ArrayBuffer): SIEEncoding { const bytes = new Uint8Array(buffer) @@ -99,19 +103,17 @@ export function detectEncoding(buffer: ArrayBuffer): SIEEncoding { // (Fortnox, Bokio, Dooer etc. export UTF-8 with #FORMAT PC8). // Instead, we detect encoding from actual byte patterns. - // Scan sample for encoding-specific byte ranges - const sampleSize = Math.min(bytes.length, 4000) let cp437Count = 0 // Swedish chars in 0x80-0x9F (CP437 range) let utf8Count = 0 // Valid UTF-8 multi-byte Swedish sequences let win1252Count = 0 // Swedish chars in 0xC0-0xFF (Win-1252 range) - for (let i = 0; i < sampleSize; i++) { + for (let i = 0; i < bytes.length; i++) { const byte = bytes[i] // Check for UTF-8 multi-byte sequences for Swedish chars FIRST // to avoid false CP437/Win-1252 counts from continuation bytes. // Ä = C3 84, Å = C3 85, Ö = C3 96, ä = C3 A4, å = C3 A5, ö = C3 B6, é = C3 A9 - if (byte === 0xc3 && i + 1 < sampleSize) { + if (byte === 0xc3 && i + 1 < bytes.length) { const nextByte = bytes[i + 1] if ([0x84, 0x85, 0x96, 0xa4, 0xa5, 0xb6, 0xa9].includes(nextByte)) { utf8Count++ @@ -140,9 +142,29 @@ export function detectEncoding(buffer: ArrayBuffer): SIEEncoding { } /** - * Decode a buffer to string using the specified encoding + * Decode a buffer to string using the specified encoding. + * + * After decoding, validates the result for U+FFFD replacement characters + * (which signal that the chosen encoding was wrong). When found, retries + * with each alternate encoding and returns the first result without U+FFFD. + * This guards against `detectEncoding` heuristic misses on files where + * Swedish characters are rare or absent in the bytes the detector looked at. */ export function decodeBuffer(buffer: ArrayBuffer, encoding: SIEEncoding): string { + const primary = decodeBufferRaw(buffer, encoding) + if (!primary.includes('\uFFFD')) return primary + + const alternates: SIEEncoding[] = (['utf8', 'windows1252', 'cp437'] as const).filter( + (e) => e !== encoding + ) + for (const alt of alternates) { + const candidate = decodeBufferRaw(buffer, alt) + if (!candidate.includes('\uFFFD')) return candidate + } + return primary +} + +function decodeBufferRaw(buffer: ArrayBuffer, encoding: SIEEncoding): string { if (encoding === 'utf8') { const decoder = new TextDecoder('utf-8') return decoder.decode(buffer) diff --git a/scripts/fix-replacement-chars.ts b/scripts/fix-replacement-chars.ts new file mode 100644 index 00000000..6cf81f84 --- /dev/null +++ b/scripts/fix-replacement-chars.ts @@ -0,0 +1,352 @@ +#!/usr/bin/env npx tsx +/** + * Find and repair U+FFFD (replacement character) corruption in user-facing + * text columns across the production database. + * + * Cause: prior to the fix in lib/import/sie-parser.ts:decodeBuffer, the SIE + * importer's encoding detector sampled only the first 4 KB of each file. When + * Swedish characters appeared only deeper in the file, the detector defaulted + * to UTF-8. Decoding a Windows-1252 byte like 0xD6 (Ö) as UTF-8 produces the + * replacement character U+FFFD — silently, because TextDecoder defaults to + * `fatal: false`. The mangled strings ended up in customers, suppliers, + * journal_entries, etc. + * + * Recovery is not deterministic — the original byte is lost. This script does + * heuristic substitution against a small Swedish-word dictionary: + * + * 1. Find rows where any text column contains U+FFFD. + * 2. For each U+FFFD-containing word, try substituting Å/Ä/Ö (uppercase + * context) or å/ä/ö (lowercase). Use the surrounding word casing to + * pick the case. + * 3. If exactly one substitution matches a known Swedish stem, apply it. + * 4. Otherwise log the row for manual review and leave it untouched. + * + * Idempotent: re-running matches zero rows once successful repairs are applied. + * + * Usage: + * # Preview every company + * npx tsx scripts/fix-replacement-chars.ts + * + * # Preview a single company + * npx tsx scripts/fix-replacement-chars.ts --company-id + * + * # Apply + * npx tsx scripts/fix-replacement-chars.ts --commit + * npx tsx scripts/fix-replacement-chars.ts --company-id --commit + */ + +import { config } from 'dotenv' +config({ path: '.env.local' }) +import { createClient, type SupabaseClient } from '@supabase/supabase-js' +import { recoverStringWithFFFD } from '../lib/import/shared/encoding' + +function arg(name: string): string | undefined { + const i = process.argv.indexOf(`--${name}`) + return i >= 0 ? process.argv[i + 1] : undefined +} + +const COMPANY_ID = arg('company-id') ?? null +const COMMIT = process.argv.includes('--commit') + +const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL +const serviceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY + +if (!supabaseUrl || !serviceRoleKey) { + console.error('Missing NEXT_PUBLIC_SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY in .env.local') + process.exit(1) +} + +const supabase = createClient(supabaseUrl, serviceRoleKey) as SupabaseClient + +const REPLACEMENT = '\uFFFD' + +interface ColumnSpec { + table: string + /** Column for tenant filtering. Most have company_id; legacy tables may use user_id. */ + tenantColumn: 'company_id' | 'user_id' | null + /** Free-text columns to scan for U+FFFD. */ + columns: readonly string[] + /** Optional row filter — used to skip immutable/posted rows. */ + filter?: (row: Record) => boolean + /** Extra SELECTs needed by the filter — pulled but not scanned for U+FFFD. */ + extraSelect?: readonly string[] +} + +const TARGETS: readonly ColumnSpec[] = [ + { + table: 'customers', + tenantColumn: 'company_id', + columns: ['name', 'address_line1', 'address_line2', 'city', 'country', 'notes'], + }, + { + table: 'suppliers', + tenantColumn: 'company_id', + columns: ['name', 'address_line1', 'address_line2', 'city', 'country', 'category', 'notes'], + }, + { + table: 'transactions', + tenantColumn: 'company_id', + columns: ['description', 'merchant_name', 'notes'], + }, + { + table: 'chart_of_accounts', + tenantColumn: 'company_id', + columns: ['account_name', 'description'], + }, + { + table: 'journal_entries', + tenantColumn: 'company_id', + columns: ['description'], + extraSelect: ['status'], + // Skip posted entries — they're legally immutable per BFL. + filter: (row) => row.status !== 'posted', + }, + // Note: journal_entry_lines.line_description is intentionally NOT scanned. + // The table has no direct company_id column, and posted lines are immutable + // per the engine's enforcement triggers. If line descriptions need repair, + // run a targeted SQL query against drafts only. + { + table: 'voucher_gap_explanations', + tenantColumn: 'company_id', + columns: ['explanation'], + }, + { + table: 'cost_centers', + tenantColumn: 'company_id', + columns: ['name'], + }, + { + table: 'projects', + tenantColumn: 'company_id', + columns: ['name'], + }, + { + table: 'receipts', + tenantColumn: 'company_id', + columns: ['merchant_name', 'representation_purpose'], + }, + { + table: 'receipt_line_items', + tenantColumn: 'company_id', + columns: ['description'], + }, + { + table: 'employees', + tenantColumn: 'company_id', + columns: ['first_name', 'last_name', 'address_line1', 'postal_code', 'city'], + }, + { + table: 'invoices', + tenantColumn: 'company_id', + columns: ['your_reference', 'our_reference', 'notes', 'reverse_charge_text'], + }, + { + table: 'invoice_items', + tenantColumn: 'company_id', + columns: ['description'], + }, + { + table: 'supplier_invoices', + tenantColumn: 'company_id', + columns: ['notes'], + extraSelect: ['status'], + filter: (row) => !['paid', 'credited', 'reversed'].includes(String(row.status)), + }, + { + table: 'supplier_invoice_items', + tenantColumn: 'company_id', + columns: ['description'], + }, + { + table: 'categorization_templates', + tenantColumn: 'company_id', + columns: ['counterparty_name'], + }, + { + table: 'companies', + tenantColumn: null, // top-level — filter directly on id when --company-id is given + columns: ['name', 'address_line1', 'address_line2', 'city'], + }, +] + +const PAGE_SIZE = 500 + +async function fetchAll(spec: ColumnSpec): Promise[]> { + const selectCols = ['id', ...spec.columns, ...(spec.extraSelect ?? [])] + if (spec.tenantColumn) selectCols.push(spec.tenantColumn) + + const all: Record[] = [] + let from = 0 + for (;;) { + let query = supabase + .from(spec.table) + .select(selectCols.join(',')) + .order('id', { ascending: true }) + .range(from, from + PAGE_SIZE - 1) + + if (COMPANY_ID) { + if (spec.tenantColumn === 'company_id') { + query = query.eq('company_id', COMPANY_ID) + } else if (spec.table === 'companies') { + query = query.eq('id', COMPANY_ID) + } + } + + const { data, error } = await query + if (error) throw new Error(`Failed to read ${spec.table}: ${error.message}`) + const rows = (data ?? []) as unknown as Record[] + all.push(...rows) + if (rows.length < PAGE_SIZE) break + from += PAGE_SIZE + } + return all +} + +interface Report { + table: string + scanned: number + affected: number + recovered: number + ambiguous: number + applied: number + failed: number +} + +async function scanTable(spec: ColumnSpec): Promise { + const rows = await fetchAll(spec) + const filtered = spec.filter ? rows.filter(spec.filter) : rows + let affected = 0 + let recovered = 0 + let ambiguous = 0 + let applied = 0 + let failed = 0 + + for (const row of filtered) { + const updates: Record = {} + const ambiguousFields: { field: string; value: string }[] = [] + let rowHasFFFD = false + + for (const col of spec.columns) { + const value = row[col] + if (typeof value !== 'string' || !value.includes(REPLACEMENT)) continue + rowHasFFFD = true + + const fixed = recoverStringWithFFFD(value) + if (fixed !== null && fixed !== value) { + updates[col] = fixed + } else { + ambiguousFields.push({ field: col, value }) + } + } + + if (!rowHasFFFD) continue + affected++ + + const id = row.id as string + const tenantId = spec.tenantColumn ? row[spec.tenantColumn] : null + + if (Object.keys(updates).length > 0) { + recovered++ + console.log(`\n · ${spec.table}.${id}${tenantId ? ` (${spec.tenantColumn}=${tenantId})` : ''}`) + for (const [field, fixed] of Object.entries(updates)) { + const before = row[field] as string + console.log(` ${field}:`) + console.log(` before: ${JSON.stringify(before)}`) + console.log(` after : ${JSON.stringify(fixed)}`) + } + + if (COMMIT) { + const { error } = await supabase.from(spec.table).update(updates).eq('id', id) + if (error) { + console.error(` FAILED: ${error.message}`) + failed++ + } else { + applied++ + } + } + } + + if (ambiguousFields.length > 0) { + ambiguous++ + console.log( + `\n · ${spec.table}.${id}${tenantId ? ` (${spec.tenantColumn}=${tenantId})` : ''} — AMBIGUOUS (manual review):` + ) + for (const { field, value } of ambiguousFields) { + console.log(` ${field}: ${JSON.stringify(value)}`) + } + } + } + + return { + table: spec.table, + scanned: filtered.length, + affected, + recovered, + ambiguous, + applied, + failed, + } +} + +async function main() { + console.log('─────────────────────────────────────────────────────────') + console.log('U+FFFD repair across user-facing text columns') + console.log('─────────────────────────────────────────────────────────') + console.log('Supabase URL :', supabaseUrl) + console.log('Company :', COMPANY_ID ?? '(all)') + console.log('Mode :', COMMIT ? 'COMMIT (writes)' : 'DRY RUN (no writes)') + console.log('─────────────────────────────────────────────────────────\n') + + const reports: Report[] = [] + for (const spec of TARGETS) { + console.log(`[${spec.table}]`) + try { + const report = await scanTable(spec) + reports.push(report) + console.log( + ` · Scanned ${report.scanned} rows; ${report.affected} affected (${report.recovered} recovered, ${report.ambiguous} ambiguous)` + ) + } catch (err) { + console.error(` · FAILED: ${err instanceof Error ? err.message : err}`) + } + } + + console.log('\n─────────────────────────────────────────────────────────') + console.log('Summary') + console.log('─────────────────────────────────────────────────────────') + const totals = reports.reduce( + (acc, r) => ({ + scanned: acc.scanned + r.scanned, + affected: acc.affected + r.affected, + recovered: acc.recovered + r.recovered, + ambiguous: acc.ambiguous + r.ambiguous, + applied: acc.applied + r.applied, + failed: acc.failed + r.failed, + }), + { scanned: 0, affected: 0, recovered: 0, ambiguous: 0, applied: 0, failed: 0 } + ) + for (const r of reports) { + if (r.affected === 0) continue + console.log( + `${r.table.padEnd(28)}: ${r.affected.toString().padStart(4)} affected, ${r.recovered.toString().padStart(4)} recovered, ${r.ambiguous.toString().padStart(4)} ambiguous` + ) + } + console.log('─────────────────────────────────────────────────────────') + console.log( + `TOTAL : ${totals.affected.toString().padStart(4)} affected, ${totals.recovered.toString().padStart(4)} recovered, ${totals.ambiguous.toString().padStart(4)} ambiguous` + ) + if (COMMIT) { + console.log(`Applied : ${totals.applied}`) + console.log(`Failed : ${totals.failed}`) + } else { + console.log('\nDry run — no changes written. Re-run with --commit to apply.') + console.log( + 'Ambiguous rows are NOT touched: review them by hand and update manually if needed.' + ) + } +} + +main().catch((err) => { + console.error('\nFATAL:', err instanceof Error ? err.message : err) + process.exit(1) +})