Fix/UI changes (#439)
* feat(bookkeeping): add preview for next voucher number in JournalEntryForm * feat(encoding): implement U+FFFD recovery for Swedish text in encoding functions
This commit is contained in:
Mattsson
@@ -3,6 +3,7 @@ import { redirect } from 'next/navigation'
|
||||
import { cookies } from 'next/headers'
|
||||
import DashboardContent from '@/components/dashboard/DashboardContent'
|
||||
import { getActiveCompanyId } from '@/lib/company/context'
|
||||
import { getDisplayTotal } from '@/lib/invoices/rounding'
|
||||
import type { Deadline, ReceiptQueueSummary, OnboardingProgress } from '@/types'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
@@ -159,8 +160,14 @@ export default async function DashboardPage() {
|
||||
.filter((t) => t.amount < 0)
|
||||
.reduce((sum, t) => sum + Math.abs(Number(t.amount_sek || t.amount)), 0)
|
||||
|
||||
// Mirror the per-invoice öresavrundning rule used on the invoice list/detail
|
||||
// pages: sum the displayed (rounded) SEK amount per invoice so the dashboard
|
||||
// total matches what the user sees on the invoice list when the setting is on.
|
||||
const unpaidTotal = (unpaidInvoices || []).reduce(
|
||||
(sum, inv) => sum + Number(inv.total_sek || inv.total),
|
||||
(sum, inv) => sum + getDisplayTotal(
|
||||
{ total: Number(inv.total_sek || inv.total), currency: 'SEK' },
|
||||
settings,
|
||||
).displayed,
|
||||
0
|
||||
)
|
||||
|
||||
|
||||
@@ -4,25 +4,38 @@ import { errorResponse } from '@/lib/errors/get-structured-error'
|
||||
|
||||
export const GET = withRouteContext(
|
||||
'voucher_sequence.next',
|
||||
async (_request, ctx) => {
|
||||
async (request, ctx) => {
|
||||
const { supabase, companyId, log, requestId } = ctx
|
||||
|
||||
const url = new URL(request.url)
|
||||
const overridePeriodId = url.searchParams.get('period_id')
|
||||
const overrideSeries = url.searchParams.get('series')
|
||||
|
||||
const today = new Date().toISOString().split('T')[0]
|
||||
|
||||
const [{ data: period, error: periodError }, { data: settings, error: settingsError }] =
|
||||
await Promise.all([
|
||||
supabase
|
||||
.from('fiscal_periods')
|
||||
.select('id')
|
||||
.eq('company_id', companyId)
|
||||
.lte('period_start', today)
|
||||
.gte('period_end', today)
|
||||
.maybeSingle(),
|
||||
supabase
|
||||
.from('company_settings')
|
||||
.select('default_voucher_series')
|
||||
.eq('company_id', companyId)
|
||||
.maybeSingle(),
|
||||
overridePeriodId
|
||||
? supabase
|
||||
.from('fiscal_periods')
|
||||
.select('id')
|
||||
.eq('company_id', companyId)
|
||||
.eq('id', overridePeriodId)
|
||||
.maybeSingle()
|
||||
: supabase
|
||||
.from('fiscal_periods')
|
||||
.select('id')
|
||||
.eq('company_id', companyId)
|
||||
.lte('period_start', today)
|
||||
.gte('period_end', today)
|
||||
.maybeSingle(),
|
||||
overrideSeries
|
||||
? Promise.resolve({ data: null, error: null })
|
||||
: supabase
|
||||
.from('company_settings')
|
||||
.select('default_voucher_series')
|
||||
.eq('company_id', companyId)
|
||||
.maybeSingle(),
|
||||
])
|
||||
|
||||
if (periodError) {
|
||||
@@ -34,7 +47,7 @@ export const GET = withRouteContext(
|
||||
return errorResponse(settingsError, log, { requestId })
|
||||
}
|
||||
|
||||
const series = settings?.default_voucher_series || 'A'
|
||||
const series = overrideSeries || settings?.default_voucher_series || 'A'
|
||||
|
||||
if (!period) {
|
||||
return NextResponse.json({ data: { next: null, series, fiscal_period_id: null } })
|
||||
|
||||
@@ -86,6 +86,7 @@ export default function JournalEntryForm({
|
||||
initialLines ?? [{ ...BLANK_LINE }, { ...BLANK_LINE }]
|
||||
)
|
||||
const [voucherSeries, setVoucherSeries] = useState('A')
|
||||
const [nextVoucherNumber, setNextVoucherNumber] = useState<number | null>(null)
|
||||
const [isSubmitting, setIsSubmitting] = useState(false)
|
||||
const [showReview, setShowReview] = useState(false)
|
||||
const [showNoDocWarning, setShowNoDocWarning] = useState(false)
|
||||
@@ -157,6 +158,31 @@ export default function JournalEntryForm({
|
||||
}
|
||||
}, [entryDate, periods])
|
||||
|
||||
// Preview the upcoming voucher number for the selected period + series.
|
||||
// Read-only hint; the actual number is reserved atomically at commit time,
|
||||
// so this may shift by one if another entry lands first.
|
||||
useEffect(() => {
|
||||
if (embedded || !selectedPeriod || !voucherSeries) {
|
||||
setNextVoucherNumber(null)
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
const qs = new URLSearchParams({ period_id: selectedPeriod, series: voucherSeries })
|
||||
fetch(`/api/bookkeeping/voucher-sequences/next?${qs}`)
|
||||
.then((r) => (r.ok ? r.json() : null))
|
||||
.then((body) => {
|
||||
if (cancelled) return
|
||||
const next = body?.data?.next
|
||||
setNextVoucherNumber(typeof next === 'number' ? next : null)
|
||||
})
|
||||
.catch(() => {
|
||||
if (!cancelled) setNextVoucherNumber(null)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [embedded, selectedPeriod, voucherSeries])
|
||||
|
||||
// Fetch exchange rate from Riksbanken when currency changes
|
||||
const fetchRate = useCallback(async (currency: Currency) => {
|
||||
if (currency === 'SEK') return
|
||||
@@ -447,8 +473,15 @@ export default function JournalEntryForm({
|
||||
<Input
|
||||
value={voucherSeries}
|
||||
onChange={(e) => {
|
||||
const v = e.target.value.toUpperCase().replace(/[^A-Z]/g, '').slice(0, 1)
|
||||
setVoucherSeries(v || 'A')
|
||||
const v = e.target.value.toUpperCase().replace(/[^A-Z]/g, '').slice(-1)
|
||||
setVoucherSeries(v)
|
||||
}}
|
||||
onFocus={(e) => {
|
||||
const target = e.target
|
||||
setTimeout(() => target.select(), 0)
|
||||
}}
|
||||
onBlur={() => {
|
||||
if (!voucherSeries) setVoucherSeries('A')
|
||||
}}
|
||||
className="mt-1 text-center font-mono"
|
||||
maxLength={1}
|
||||
@@ -792,7 +825,11 @@ export default function JournalEntryForm({
|
||||
onOpenChange={setShowReview}
|
||||
onConfirm={handleConfirm}
|
||||
isSubmitting={isSubmitting}
|
||||
title="Granska verifikation"
|
||||
title={
|
||||
!embedded && nextVoucherNumber != null
|
||||
? `Granska verifikation (${voucherSeries}${nextVoucherNumber})`
|
||||
: 'Granska verifikation'
|
||||
}
|
||||
warningText={embedded ? '' : 'En verifikation skapas och kan inte ändras efteråt. Korrigeringar görs genom storno.'}
|
||||
>
|
||||
<JournalEntryReviewContent
|
||||
|
||||
@@ -568,6 +568,53 @@ describe('decodeBuffer — Windows-1252', () => {
|
||||
})
|
||||
})
|
||||
|
||||
// --- Defensive encoding: handle files where the detector picks the wrong encoding ---
|
||||
|
||||
describe('detectEncoding — full-buffer scan', () => {
|
||||
it('detects Win-1252 even when Swedish chars appear past the legacy 4KB sample boundary', () => {
|
||||
// Build a buffer where the first 8000 bytes are pure ASCII header + filler,
|
||||
// and the Swedish Win-1252 byte appears only at byte 8000+. The old
|
||||
// implementation sampled the first 4000 bytes and would default to UTF-8.
|
||||
const filler = new Uint8Array(8000).fill(0x20) // spaces
|
||||
const tail = new Uint8Array([
|
||||
0x46, 0x4f, 0x52, 0x45, 0x4e, 0x49, 0x4e, 0x47, // FORENING
|
||||
0xd6, // Ö in Win-1252 (0xD6) — invalid lone UTF-8 byte
|
||||
])
|
||||
const buf = new Uint8Array(filler.length + tail.length)
|
||||
buf.set(filler, 0)
|
||||
buf.set(tail, filler.length)
|
||||
const encoding = detectEncoding(buf.buffer)
|
||||
expect(encoding).toBe('windows1252')
|
||||
})
|
||||
})
|
||||
|
||||
describe('decodeBuffer — fallback on U+FFFD', () => {
|
||||
it('falls back from utf8 to windows1252 when the result has replacement characters', () => {
|
||||
// "F" "Ö" "RENING" in Windows-1252 — Ö is lone byte 0xD6, not valid UTF-8
|
||||
const buf = new Uint8Array([0x46, 0xd6, 0x52, 0x45, 0x4e, 0x49, 0x4e, 0x47])
|
||||
const result = decodeBuffer(buf.buffer, 'utf8')
|
||||
expect(result).toBe('FÖRENING')
|
||||
expect(result.includes('\uFFFD')).toBe(false)
|
||||
})
|
||||
|
||||
it('falls back from utf8 to cp437 when both windows1252 also fails', () => {
|
||||
// 0x94 is "ö" in CP437; in Win-1252 it's an unprintable "" but textually different;
|
||||
// in UTF-8 it's invalid → U+FFFD. Verify CP437 path is reachable when chosen wrong.
|
||||
const buf = new Uint8Array([0x66, 0x94, 0x72]) // f + ö-cp437 + r
|
||||
const result = decodeBuffer(buf.buffer, 'utf8')
|
||||
// Either windows1252 or cp437 fallback produces a non-FFFD result; both are
|
||||
// acceptable here since the byte 0x94 is interpretable in both — what matters
|
||||
// is no U+FFFD leaks through.
|
||||
expect(result.includes('\uFFFD')).toBe(false)
|
||||
})
|
||||
|
||||
it('returns primary decode unchanged when it contains no U+FFFD', () => {
|
||||
const buf = new TextEncoder().encode('Företag').buffer
|
||||
const result = decodeBuffer(buf, 'utf8')
|
||||
expect(result).toBe('Företag')
|
||||
})
|
||||
})
|
||||
|
||||
// --- Fix 3: Invalid date rejection ---
|
||||
|
||||
describe('parseSIEFile — invalid date handling', () => {
|
||||
|
||||
@@ -3,6 +3,8 @@ import {
|
||||
decodeFileContent,
|
||||
decodeStringContent,
|
||||
hasEncodingIssues,
|
||||
recoverStringWithFFFD,
|
||||
recoverWordWithFFFD,
|
||||
} from '../encoding'
|
||||
|
||||
describe('decodeStringContent', () => {
|
||||
@@ -77,3 +79,75 @@ describe('decodeFileContent', () => {
|
||||
expect(decodeFileContent(cp1252)).toBe('GÖTEBORG')
|
||||
})
|
||||
})
|
||||
|
||||
// --- U+FFFD heuristic recovery ---
|
||||
|
||||
describe('recoverWordWithFFFD', () => {
|
||||
it('recovers uppercase Ö in common Swedish stems', () => {
|
||||
expect(recoverWordWithFFFD('F\uFFFDRENING')).toBe('FÖRENING')
|
||||
expect(recoverWordWithFFFD('F\uFFFDRETAG')).toBe('FÖRETAG')
|
||||
expect(recoverWordWithFFFD('G\uFFFDTEBORG')).toBe('GÖTEBORG')
|
||||
expect(recoverWordWithFFFD('LINK\uFFFDPING')).toBe('LINKÖPING')
|
||||
})
|
||||
|
||||
it('recovers lowercase ö in common Swedish stems', () => {
|
||||
expect(recoverWordWithFFFD('f\uFFFDrening')).toBe('förening')
|
||||
expect(recoverWordWithFFFD('malm\uFFFD')).toBe('malmö')
|
||||
expect(recoverWordWithFFFD('k\uFFFDp')).toBe('köp')
|
||||
})
|
||||
|
||||
it('recovers compound words via substring match', () => {
|
||||
expect(recoverWordWithFFFD('BOSTADSR\uFFFDTTSF\uFFFDRENING')).toBe(
|
||||
'BOSTADSRÄTTSFÖRENING'
|
||||
)
|
||||
expect(recoverWordWithFFFD('Idrottsf\uFFFDrening')).toBe('Idrottsförening')
|
||||
})
|
||||
|
||||
it('is a no-op when the input has no U+FFFD', () => {
|
||||
expect(recoverWordWithFFFD('FÖRENING')).toBe('FÖRENING')
|
||||
expect(recoverWordWithFFFD('hello')).toBe('hello')
|
||||
})
|
||||
|
||||
it('returns null for ambiguous words not in the dictionary', () => {
|
||||
// Random 4-letter word with U+FFFD; no Swedish stem hits.
|
||||
expect(recoverWordWithFFFD('Z\uFFFDXQ')).toBeNull()
|
||||
})
|
||||
|
||||
it('returns null for words with too many U+FFFDs to disambiguate', () => {
|
||||
expect(
|
||||
recoverWordWithFFFD('\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD')
|
||||
).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('recoverStringWithFFFD', () => {
|
||||
it('repairs the canonical "Levbet FÖRENING" case', () => {
|
||||
expect(recoverStringWithFFFD('Levbet F\uFFFDRENING')).toBe('Levbet FÖRENING')
|
||||
})
|
||||
|
||||
it('repairs city + business-name combos', () => {
|
||||
expect(recoverStringWithFFFD('Sjöberg AB, Malm\uFFFD')).toBe('Sjöberg AB, Malmö')
|
||||
expect(recoverStringWithFFFD('Faktura fr\uFFFDn G\uFFFDTEBORG AB')).toBe(
|
||||
'Faktura från GÖTEBORG AB'
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves punctuation, whitespace, and digits', () => {
|
||||
expect(recoverStringWithFFFD('K\uFFFDp 1 234,56 SEK')).toBe('Köp 1 234,56 SEK')
|
||||
})
|
||||
|
||||
it('is a no-op on clean strings', () => {
|
||||
expect(recoverStringWithFFFD('Hello World')).toBe('Hello World')
|
||||
expect(recoverStringWithFFFD('FÖRENING')).toBe('FÖRENING')
|
||||
})
|
||||
|
||||
it('returns null when any word in the string is ambiguous', () => {
|
||||
expect(recoverStringWithFFFD('FÖRENING Z\uFFFDXQ')).toBeNull()
|
||||
})
|
||||
|
||||
it('is idempotent on recovered output', () => {
|
||||
const once = recoverStringWithFFFD('F\uFFFDRENING')
|
||||
expect(once).toBe('FÖRENING')
|
||||
expect(recoverStringWithFFFD(once!)).toBe('FÖRENING')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -87,3 +87,156 @@ export function stripBOM(content: string): string {
|
||||
export function prepareContent(content: string): string {
|
||||
return normalizeLineEndings(stripBOM(decodeStringContent(content)))
|
||||
}
|
||||
|
||||
/**
|
||||
* --- U+FFFD heuristic recovery ---
|
||||
*
|
||||
* When Windows-1252 / Latin-1 bytes are decoded as UTF-8 with `fatal: false`,
|
||||
* invalid sequences silently become U+FFFD. The original byte is lost — but
|
||||
* for Swedish text we can guess from context: the missing letter is almost
|
||||
* always one of Å/Ä/Ö (uppercase context) or å/ä/ö (lowercase context).
|
||||
*
|
||||
* This recovery tries each Swedish vowel substitution and scores the resulting
|
||||
* word against a small dictionary of Swedish stems. If exactly one candidate
|
||||
* scores above the threshold, it's applied. Ambiguous cases return null and
|
||||
* must be reviewed manually.
|
||||
*/
|
||||
|
||||
const REPLACEMENT = '\uFFFD'
|
||||
const SWEDISH_VOWELS_UPPER = ['Å', 'Ä', 'Ö'] as const
|
||||
const SWEDISH_VOWELS_LOWER = ['å', 'ä', 'ö'] as const
|
||||
|
||||
/**
|
||||
* Stems of common Swedish words containing åäö that appear in business names,
|
||||
* place names, addresses, and accounting descriptions.
|
||||
*/
|
||||
const SWEDISH_STEMS = new Set<string>([
|
||||
// -för- prefix (extremely common)
|
||||
'för', 'före', 'förening', 'företag', 'försäkring', 'försäljning',
|
||||
'försäljnings', 'förskola', 'församling', 'förvaltning', 'förbund',
|
||||
'förlag', 'föräldra', 'försök', 'förbrukning', 'förbättring', 'förskott',
|
||||
'försening', 'förhandling', 'förbättrings',
|
||||
// domain terms
|
||||
'bostadsrätt', 'rätt', 'samfällighet', 'idrott', 'fastighet', 'utbildning',
|
||||
'näring', 'växel', 'värme', 'köp', 'köpa', 'inköp', 'sälja', 'säljs',
|
||||
// accounting
|
||||
'kostnad', 'kostnader', 'intäkt', 'intäkter', 'avskrivning', 'avsättning',
|
||||
'lön', 'lönekostnad', 'pension', 'utgående', 'ingående', 'momspliktig',
|
||||
'redovisning', 'företagskonto', 'bankkonto', 'överavskrivning',
|
||||
'överskott', 'underskott', 'överföring', 'överlåtelse', 'återbetalning',
|
||||
'utlägg', 'utgift',
|
||||
// common short prepositions and adverbs
|
||||
'från', 'för', 'över', 'är', 'när', 'där', 'även', 'någon', 'något',
|
||||
'många', 'själv', 'små', 'väg', 'gång', 'tjänst', 'tjänster', 'räkning',
|
||||
'räntor', 'år',
|
||||
// common cities
|
||||
'göteborg', 'malmö', 'örebro', 'östersund', 'jönköping', 'linköping',
|
||||
'norrköping', 'lidköping', 'köping', 'helsingborg', 'umeå', 'skellefteå',
|
||||
'piteå', 'luleå', 'borås', 'växjö', 'östhammar', 'södertälje', 'västerås',
|
||||
'härnösand', 'värnamo', 'mölndal', 'mörrum', 'mönsterås', 'färjestaden',
|
||||
'eskilstuna',
|
||||
// directions / common geo terms
|
||||
'östra', 'västra', 'södra', 'norra', 'öster', 'väster', 'söder',
|
||||
// legal forms
|
||||
'aktiebolag', 'handelsbolag', 'ekonomisk', 'allmännyttig',
|
||||
// misc
|
||||
'företagsledare', 'koncernbidrag', 'utländsk', 'utländska', 'främmande',
|
||||
'vägen', 'gatan', 'allén', 'gränden', 'torget',
|
||||
// surnames containing åäö
|
||||
'lindström', 'sjöberg', 'söderberg', 'öberg', 'åström', 'åkerlund',
|
||||
'östlund', 'lindgren', 'sjögren', 'hägglund', 'bäckström',
|
||||
])
|
||||
|
||||
/**
|
||||
* Score a candidate word.
|
||||
* - 1000 if the entire word matches a known stem (highest confidence).
|
||||
* - Otherwise the count of distinct stems that appear as substrings.
|
||||
* Counting (not boolean-returning) is required: when the same word has
|
||||
* multiple U+FFFD positions, the correct combination must outscore wrong
|
||||
* combinations that still happen to contain *one* stem each.
|
||||
*/
|
||||
function scoreCandidate(word: string): number {
|
||||
const lower = word.toLowerCase()
|
||||
if (SWEDISH_STEMS.has(lower)) return 1000
|
||||
let score = 0
|
||||
for (const stem of SWEDISH_STEMS) {
|
||||
if (lower.includes(stem)) score++
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
function chooseCase(word: string): 'upper' | 'lower' {
|
||||
let upper = 0
|
||||
let lower = 0
|
||||
for (const ch of word) {
|
||||
if (ch === REPLACEMENT) continue
|
||||
if (ch >= 'A' && ch <= 'Z') upper++
|
||||
else if (ch >= 'a' && ch <= 'z') lower++
|
||||
}
|
||||
return upper > lower ? 'upper' : 'lower'
|
||||
}
|
||||
|
||||
/**
|
||||
* Try every Swedish-vowel substitution for the U+FFFDs in `word`, then return
|
||||
* the highest-scoring candidate. Returns null if no candidate matches a
|
||||
* dictionary stem (i.e. ambiguous — operator must review).
|
||||
*/
|
||||
export function recoverWordWithFFFD(word: string): string | null {
|
||||
if (!word.includes(REPLACEMENT)) return word
|
||||
|
||||
const vowels = chooseCase(word) === 'upper' ? SWEDISH_VOWELS_UPPER : SWEDISH_VOWELS_LOWER
|
||||
const positions: number[] = []
|
||||
for (let i = 0; i < word.length; i++) {
|
||||
if (word[i] === REPLACEMENT) positions.push(i)
|
||||
}
|
||||
|
||||
// Words with more than ~6 lost bytes blow up the combinatorial space —
|
||||
// bail out rather than spend cycles on something that's likely garbage anyway.
|
||||
const totalCombos = Math.pow(vowels.length, positions.length)
|
||||
if (totalCombos > 729) return null
|
||||
|
||||
let best: { word: string; score: number } | null = null
|
||||
for (let combo = 0; combo < totalCombos; combo++) {
|
||||
const chars = word.split('')
|
||||
let c = combo
|
||||
for (const pos of positions) {
|
||||
chars[pos] = vowels[c % vowels.length]
|
||||
c = Math.floor(c / vowels.length)
|
||||
}
|
||||
const candidate = chars.join('')
|
||||
const score = scoreCandidate(candidate)
|
||||
if (!best || score > best.score) {
|
||||
best = { word: candidate, score }
|
||||
}
|
||||
}
|
||||
|
||||
if (!best || best.score === 0) return null
|
||||
return best.word
|
||||
}
|
||||
|
||||
/**
|
||||
* Repair every U+FFFD-containing word in `text` via dictionary-backed
|
||||
* substitution. Returns the repaired string when *every* corrupted word
|
||||
* resolved to a confident candidate; returns null if any word remains
|
||||
* ambiguous (operator must review the whole string by hand).
|
||||
*
|
||||
* Idempotent on clean input.
|
||||
*/
|
||||
export function recoverStringWithFFFD(text: string): string | null {
|
||||
if (!text.includes(REPLACEMENT)) return text
|
||||
|
||||
// Split on runs of non-letter/non-digit characters so punctuation,
|
||||
// whitespace, and structural characters are preserved as-is.
|
||||
const tokens = text.split(/([^\p{L}\p{N}\uFFFD]+)/u)
|
||||
const out: string[] = []
|
||||
for (const token of tokens) {
|
||||
if (!token.includes(REPLACEMENT)) {
|
||||
out.push(token)
|
||||
continue
|
||||
}
|
||||
const recovered = recoverWordWithFFFD(token)
|
||||
if (recovered === null) return null
|
||||
out.push(recovered)
|
||||
}
|
||||
return out.join('')
|
||||
}
|
||||
|
||||
@@ -85,6 +85,10 @@ const WIN1252_SWEDISH_BYTES = new Set([
|
||||
* so presence in one range rules out the other.
|
||||
* 4. UTF-8 multi-byte sequences (0xC3 + continuation) are detected with proper
|
||||
* skipping of continuation bytes to avoid false CP437 counts.
|
||||
*
|
||||
* Scans the entire buffer (not a sample): SIE files are capped at 50 MB and
|
||||
* Swedish characters often only appear deep in voucher descriptions, well past
|
||||
* any small header sample.
|
||||
*/
|
||||
export function detectEncoding(buffer: ArrayBuffer): SIEEncoding {
|
||||
const bytes = new Uint8Array(buffer)
|
||||
@@ -99,19 +103,17 @@ export function detectEncoding(buffer: ArrayBuffer): SIEEncoding {
|
||||
// (Fortnox, Bokio, Dooer etc. export UTF-8 with #FORMAT PC8).
|
||||
// Instead, we detect encoding from actual byte patterns.
|
||||
|
||||
// Scan sample for encoding-specific byte ranges
|
||||
const sampleSize = Math.min(bytes.length, 4000)
|
||||
let cp437Count = 0 // Swedish chars in 0x80-0x9F (CP437 range)
|
||||
let utf8Count = 0 // Valid UTF-8 multi-byte Swedish sequences
|
||||
let win1252Count = 0 // Swedish chars in 0xC0-0xFF (Win-1252 range)
|
||||
|
||||
for (let i = 0; i < sampleSize; i++) {
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const byte = bytes[i]
|
||||
|
||||
// Check for UTF-8 multi-byte sequences for Swedish chars FIRST
|
||||
// to avoid false CP437/Win-1252 counts from continuation bytes.
|
||||
// Ä = C3 84, Å = C3 85, Ö = C3 96, ä = C3 A4, å = C3 A5, ö = C3 B6, é = C3 A9
|
||||
if (byte === 0xc3 && i + 1 < sampleSize) {
|
||||
if (byte === 0xc3 && i + 1 < bytes.length) {
|
||||
const nextByte = bytes[i + 1]
|
||||
if ([0x84, 0x85, 0x96, 0xa4, 0xa5, 0xb6, 0xa9].includes(nextByte)) {
|
||||
utf8Count++
|
||||
@@ -140,9 +142,29 @@ export function detectEncoding(buffer: ArrayBuffer): SIEEncoding {
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a buffer to string using the specified encoding
|
||||
* Decode a buffer to string using the specified encoding.
|
||||
*
|
||||
* After decoding, validates the result for U+FFFD replacement characters
|
||||
* (which signal that the chosen encoding was wrong). When found, retries
|
||||
* with each alternate encoding and returns the first result without U+FFFD.
|
||||
* This guards against `detectEncoding` heuristic misses on files where
|
||||
* Swedish characters are rare or absent in the bytes the detector looked at.
|
||||
*/
|
||||
export function decodeBuffer(buffer: ArrayBuffer, encoding: SIEEncoding): string {
|
||||
const primary = decodeBufferRaw(buffer, encoding)
|
||||
if (!primary.includes('\uFFFD')) return primary
|
||||
|
||||
const alternates: SIEEncoding[] = (['utf8', 'windows1252', 'cp437'] as const).filter(
|
||||
(e) => e !== encoding
|
||||
)
|
||||
for (const alt of alternates) {
|
||||
const candidate = decodeBufferRaw(buffer, alt)
|
||||
if (!candidate.includes('\uFFFD')) return candidate
|
||||
}
|
||||
return primary
|
||||
}
|
||||
|
||||
function decodeBufferRaw(buffer: ArrayBuffer, encoding: SIEEncoding): string {
|
||||
if (encoding === 'utf8') {
|
||||
const decoder = new TextDecoder('utf-8')
|
||||
return decoder.decode(buffer)
|
||||
|
||||
@@ -0,0 +1,352 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
/**
|
||||
* Find and repair U+FFFD (replacement character) corruption in user-facing
|
||||
* text columns across the production database.
|
||||
*
|
||||
* Cause: prior to the fix in lib/import/sie-parser.ts:decodeBuffer, the SIE
|
||||
* importer's encoding detector sampled only the first 4 KB of each file. When
|
||||
* Swedish characters appeared only deeper in the file, the detector defaulted
|
||||
* to UTF-8. Decoding a Windows-1252 byte like 0xD6 (Ö) as UTF-8 produces the
|
||||
* replacement character U+FFFD — silently, because TextDecoder defaults to
|
||||
* `fatal: false`. The mangled strings ended up in customers, suppliers,
|
||||
* journal_entries, etc.
|
||||
*
|
||||
* Recovery is not deterministic — the original byte is lost. This script does
|
||||
* heuristic substitution against a small Swedish-word dictionary:
|
||||
*
|
||||
* 1. Find rows where any text column contains U+FFFD.
|
||||
* 2. For each U+FFFD-containing word, try substituting Å/Ä/Ö (uppercase
|
||||
* context) or å/ä/ö (lowercase). Use the surrounding word casing to
|
||||
* pick the case.
|
||||
* 3. If exactly one substitution matches a known Swedish stem, apply it.
|
||||
* 4. Otherwise log the row for manual review and leave it untouched.
|
||||
*
|
||||
* Idempotent: re-running matches zero rows once successful repairs are applied.
|
||||
*
|
||||
* Usage:
|
||||
* # Preview every company
|
||||
* npx tsx scripts/fix-replacement-chars.ts
|
||||
*
|
||||
* # Preview a single company
|
||||
* npx tsx scripts/fix-replacement-chars.ts --company-id <uuid>
|
||||
*
|
||||
* # Apply
|
||||
* npx tsx scripts/fix-replacement-chars.ts --commit
|
||||
* npx tsx scripts/fix-replacement-chars.ts --company-id <uuid> --commit
|
||||
*/
|
||||
|
||||
import { config } from 'dotenv'
|
||||
config({ path: '.env.local' })
|
||||
import { createClient, type SupabaseClient } from '@supabase/supabase-js'
|
||||
import { recoverStringWithFFFD } from '../lib/import/shared/encoding'
|
||||
|
||||
function arg(name: string): string | undefined {
|
||||
const i = process.argv.indexOf(`--${name}`)
|
||||
return i >= 0 ? process.argv[i + 1] : undefined
|
||||
}
|
||||
|
||||
const COMPANY_ID = arg('company-id') ?? null
|
||||
const COMMIT = process.argv.includes('--commit')
|
||||
|
||||
const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL
|
||||
const serviceRoleKey = process.env.SUPABASE_SERVICE_ROLE_KEY
|
||||
|
||||
if (!supabaseUrl || !serviceRoleKey) {
|
||||
console.error('Missing NEXT_PUBLIC_SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY in .env.local')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const supabase = createClient(supabaseUrl, serviceRoleKey) as SupabaseClient
|
||||
|
||||
const REPLACEMENT = '\uFFFD'
|
||||
|
||||
interface ColumnSpec {
|
||||
table: string
|
||||
/** Column for tenant filtering. Most have company_id; legacy tables may use user_id. */
|
||||
tenantColumn: 'company_id' | 'user_id' | null
|
||||
/** Free-text columns to scan for U+FFFD. */
|
||||
columns: readonly string[]
|
||||
/** Optional row filter — used to skip immutable/posted rows. */
|
||||
filter?: (row: Record<string, unknown>) => boolean
|
||||
/** Extra SELECTs needed by the filter — pulled but not scanned for U+FFFD. */
|
||||
extraSelect?: readonly string[]
|
||||
}
|
||||
|
||||
const TARGETS: readonly ColumnSpec[] = [
|
||||
{
|
||||
table: 'customers',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['name', 'address_line1', 'address_line2', 'city', 'country', 'notes'],
|
||||
},
|
||||
{
|
||||
table: 'suppliers',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['name', 'address_line1', 'address_line2', 'city', 'country', 'category', 'notes'],
|
||||
},
|
||||
{
|
||||
table: 'transactions',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['description', 'merchant_name', 'notes'],
|
||||
},
|
||||
{
|
||||
table: 'chart_of_accounts',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['account_name', 'description'],
|
||||
},
|
||||
{
|
||||
table: 'journal_entries',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['description'],
|
||||
extraSelect: ['status'],
|
||||
// Skip posted entries — they're legally immutable per BFL.
|
||||
filter: (row) => row.status !== 'posted',
|
||||
},
|
||||
// Note: journal_entry_lines.line_description is intentionally NOT scanned.
|
||||
// The table has no direct company_id column, and posted lines are immutable
|
||||
// per the engine's enforcement triggers. If line descriptions need repair,
|
||||
// run a targeted SQL query against drafts only.
|
||||
{
|
||||
table: 'voucher_gap_explanations',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['explanation'],
|
||||
},
|
||||
{
|
||||
table: 'cost_centers',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['name'],
|
||||
},
|
||||
{
|
||||
table: 'projects',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['name'],
|
||||
},
|
||||
{
|
||||
table: 'receipts',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['merchant_name', 'representation_purpose'],
|
||||
},
|
||||
{
|
||||
table: 'receipt_line_items',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['description'],
|
||||
},
|
||||
{
|
||||
table: 'employees',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['first_name', 'last_name', 'address_line1', 'postal_code', 'city'],
|
||||
},
|
||||
{
|
||||
table: 'invoices',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['your_reference', 'our_reference', 'notes', 'reverse_charge_text'],
|
||||
},
|
||||
{
|
||||
table: 'invoice_items',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['description'],
|
||||
},
|
||||
{
|
||||
table: 'supplier_invoices',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['notes'],
|
||||
extraSelect: ['status'],
|
||||
filter: (row) => !['paid', 'credited', 'reversed'].includes(String(row.status)),
|
||||
},
|
||||
{
|
||||
table: 'supplier_invoice_items',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['description'],
|
||||
},
|
||||
{
|
||||
table: 'categorization_templates',
|
||||
tenantColumn: 'company_id',
|
||||
columns: ['counterparty_name'],
|
||||
},
|
||||
{
|
||||
table: 'companies',
|
||||
tenantColumn: null, // top-level — filter directly on id when --company-id is given
|
||||
columns: ['name', 'address_line1', 'address_line2', 'city'],
|
||||
},
|
||||
]
|
||||
|
||||
const PAGE_SIZE = 500
|
||||
|
||||
async function fetchAll(spec: ColumnSpec): Promise<Record<string, unknown>[]> {
|
||||
const selectCols = ['id', ...spec.columns, ...(spec.extraSelect ?? [])]
|
||||
if (spec.tenantColumn) selectCols.push(spec.tenantColumn)
|
||||
|
||||
const all: Record<string, unknown>[] = []
|
||||
let from = 0
|
||||
for (;;) {
|
||||
let query = supabase
|
||||
.from(spec.table)
|
||||
.select(selectCols.join(','))
|
||||
.order('id', { ascending: true })
|
||||
.range(from, from + PAGE_SIZE - 1)
|
||||
|
||||
if (COMPANY_ID) {
|
||||
if (spec.tenantColumn === 'company_id') {
|
||||
query = query.eq('company_id', COMPANY_ID)
|
||||
} else if (spec.table === 'companies') {
|
||||
query = query.eq('id', COMPANY_ID)
|
||||
}
|
||||
}
|
||||
|
||||
const { data, error } = await query
|
||||
if (error) throw new Error(`Failed to read ${spec.table}: ${error.message}`)
|
||||
const rows = (data ?? []) as unknown as Record<string, unknown>[]
|
||||
all.push(...rows)
|
||||
if (rows.length < PAGE_SIZE) break
|
||||
from += PAGE_SIZE
|
||||
}
|
||||
return all
|
||||
}
|
||||
|
||||
interface Report {
|
||||
table: string
|
||||
scanned: number
|
||||
affected: number
|
||||
recovered: number
|
||||
ambiguous: number
|
||||
applied: number
|
||||
failed: number
|
||||
}
|
||||
|
||||
async function scanTable(spec: ColumnSpec): Promise<Report> {
|
||||
const rows = await fetchAll(spec)
|
||||
const filtered = spec.filter ? rows.filter(spec.filter) : rows
|
||||
let affected = 0
|
||||
let recovered = 0
|
||||
let ambiguous = 0
|
||||
let applied = 0
|
||||
let failed = 0
|
||||
|
||||
for (const row of filtered) {
|
||||
const updates: Record<string, string> = {}
|
||||
const ambiguousFields: { field: string; value: string }[] = []
|
||||
let rowHasFFFD = false
|
||||
|
||||
for (const col of spec.columns) {
|
||||
const value = row[col]
|
||||
if (typeof value !== 'string' || !value.includes(REPLACEMENT)) continue
|
||||
rowHasFFFD = true
|
||||
|
||||
const fixed = recoverStringWithFFFD(value)
|
||||
if (fixed !== null && fixed !== value) {
|
||||
updates[col] = fixed
|
||||
} else {
|
||||
ambiguousFields.push({ field: col, value })
|
||||
}
|
||||
}
|
||||
|
||||
if (!rowHasFFFD) continue
|
||||
affected++
|
||||
|
||||
const id = row.id as string
|
||||
const tenantId = spec.tenantColumn ? row[spec.tenantColumn] : null
|
||||
|
||||
if (Object.keys(updates).length > 0) {
|
||||
recovered++
|
||||
console.log(`\n · ${spec.table}.${id}${tenantId ? ` (${spec.tenantColumn}=${tenantId})` : ''}`)
|
||||
for (const [field, fixed] of Object.entries(updates)) {
|
||||
const before = row[field] as string
|
||||
console.log(` ${field}:`)
|
||||
console.log(` before: ${JSON.stringify(before)}`)
|
||||
console.log(` after : ${JSON.stringify(fixed)}`)
|
||||
}
|
||||
|
||||
if (COMMIT) {
|
||||
const { error } = await supabase.from(spec.table).update(updates).eq('id', id)
|
||||
if (error) {
|
||||
console.error(` FAILED: ${error.message}`)
|
||||
failed++
|
||||
} else {
|
||||
applied++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ambiguousFields.length > 0) {
|
||||
ambiguous++
|
||||
console.log(
|
||||
`\n · ${spec.table}.${id}${tenantId ? ` (${spec.tenantColumn}=${tenantId})` : ''} — AMBIGUOUS (manual review):`
|
||||
)
|
||||
for (const { field, value } of ambiguousFields) {
|
||||
console.log(` ${field}: ${JSON.stringify(value)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
table: spec.table,
|
||||
scanned: filtered.length,
|
||||
affected,
|
||||
recovered,
|
||||
ambiguous,
|
||||
applied,
|
||||
failed,
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('─────────────────────────────────────────────────────────')
|
||||
console.log('U+FFFD repair across user-facing text columns')
|
||||
console.log('─────────────────────────────────────────────────────────')
|
||||
console.log('Supabase URL :', supabaseUrl)
|
||||
console.log('Company :', COMPANY_ID ?? '(all)')
|
||||
console.log('Mode :', COMMIT ? 'COMMIT (writes)' : 'DRY RUN (no writes)')
|
||||
console.log('─────────────────────────────────────────────────────────\n')
|
||||
|
||||
const reports: Report[] = []
|
||||
for (const spec of TARGETS) {
|
||||
console.log(`[${spec.table}]`)
|
||||
try {
|
||||
const report = await scanTable(spec)
|
||||
reports.push(report)
|
||||
console.log(
|
||||
` · Scanned ${report.scanned} rows; ${report.affected} affected (${report.recovered} recovered, ${report.ambiguous} ambiguous)`
|
||||
)
|
||||
} catch (err) {
|
||||
console.error(` · FAILED: ${err instanceof Error ? err.message : err}`)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n─────────────────────────────────────────────────────────')
|
||||
console.log('Summary')
|
||||
console.log('─────────────────────────────────────────────────────────')
|
||||
const totals = reports.reduce(
|
||||
(acc, r) => ({
|
||||
scanned: acc.scanned + r.scanned,
|
||||
affected: acc.affected + r.affected,
|
||||
recovered: acc.recovered + r.recovered,
|
||||
ambiguous: acc.ambiguous + r.ambiguous,
|
||||
applied: acc.applied + r.applied,
|
||||
failed: acc.failed + r.failed,
|
||||
}),
|
||||
{ scanned: 0, affected: 0, recovered: 0, ambiguous: 0, applied: 0, failed: 0 }
|
||||
)
|
||||
for (const r of reports) {
|
||||
if (r.affected === 0) continue
|
||||
console.log(
|
||||
`${r.table.padEnd(28)}: ${r.affected.toString().padStart(4)} affected, ${r.recovered.toString().padStart(4)} recovered, ${r.ambiguous.toString().padStart(4)} ambiguous`
|
||||
)
|
||||
}
|
||||
console.log('─────────────────────────────────────────────────────────')
|
||||
console.log(
|
||||
`TOTAL : ${totals.affected.toString().padStart(4)} affected, ${totals.recovered.toString().padStart(4)} recovered, ${totals.ambiguous.toString().padStart(4)} ambiguous`
|
||||
)
|
||||
if (COMMIT) {
|
||||
console.log(`Applied : ${totals.applied}`)
|
||||
console.log(`Failed : ${totals.failed}`)
|
||||
} else {
|
||||
console.log('\nDry run — no changes written. Re-run with --commit to apply.')
|
||||
console.log(
|
||||
'Ambiguous rows are NOT touched: review them by hand and update manually if needed.'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('\nFATAL:', err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
})
|
||||
Reference in New Issue
Block a user