accounted/lib/import/shared/__tests__/encoding.test.ts

import { describe, it, expect } from 'vitest'
import {
  decodeFileContent,
  decodeStringContent,
  hasEncodingIssues,
  recoverStringWithFFFD,
  recoverWordWithFFFD,
  stripBOM,
} from '../encoding'

describe('decodeStringContent', () => {
  it('recovers UTF-8-as-Latin-1 mojibake for lowercase Swedish chars', () => {
    expect(decodeStringContent('Malmö')).toBe('Malmö')
    expect(decodeStringContent('Ã¥re')).toBe('Åre'.toLowerCase())
    expect(decodeStringContent('Linköping')).toBe('Linköping')
  })

  it('recovers UTF-8-as-Latin-1 mojibake for uppercase Swedish chars', () => {
    // The middle char is U+0096 (control), invisible in most renderings → "GÃTEBORG"
    expect(decodeStringContent('GÃ\u0096TEBORG')).toBe('GÖTEBORG')
    expect(decodeStringContent('HISINGS KÃ\u0084RRA')).toBe('HISINGS KÄRRA')
    expect(decodeStringContent('Ã\u0085NGE')).toBe('ÅNGE')
  })

  it('is a no-op on already-correct Swedish strings', () => {
    expect(decodeStringContent('GÖTEBORG')).toBe('GÖTEBORG')
    expect(decodeStringContent('Malmö')).toBe('Malmö')
    expect(decodeStringContent('STOCKHOLM')).toBe('STOCKHOLM')
    expect(decodeStringContent('')).toBe('')
  })

  it('is idempotent (running twice equals running once)', () => {
    const once = decodeStringContent('Malmö')
    const twice = decodeStringContent(once)
    expect(twice).toBe(once)
    expect(twice).toBe('Malmö')
  })

  it('preserves non-Swedish strings unchanged', () => {
    expect(decodeStringContent('Café')).toBe('Café')
    expect(decodeStringContent('München')).toBe('München')
    expect(decodeStringContent('123 Main St')).toBe('123 Main St')
  })
})

describe('hasEncodingIssues', () => {
  it('detects U+FFFD replacement characters', () => {
    expect(hasEncodingIssues('Foo\uFFFDbar')).toBe(true)
  })

  it('detects all six Swedish mojibake patterns', () => {
    expect(hasEncodingIssues('Malmö')).toBe(true) // ö
    expect(hasEncodingIssues('Ã¥re')).toBe(true) // å
    expect(hasEncodingIssues('älg')).toBe(true) // ä
    expect(hasEncodingIssues('GÃ\u0096TEBORG')).toBe(true) // Ö
    expect(hasEncodingIssues('Ã\u0085NGE')).toBe(true) // Å
    expect(hasEncodingIssues('Ã\u0084RRA')).toBe(true) // Ä
  })

  it('returns false for clean strings', () => {
    expect(hasEncodingIssues('Stockholm')).toBe(false)
    expect(hasEncodingIssues('Malmö')).toBe(false)
    expect(hasEncodingIssues('Café')).toBe(false)
  })
})

describe('decodeFileContent', () => {
  function buf(bytes: number[]): ArrayBuffer {
    return new Uint8Array(bytes).buffer
  }

  it('decodes UTF-8 bytes correctly', () => {
    const utf8 = new TextEncoder().encode('GÖTEBORG').buffer
    expect(decodeFileContent(utf8)).toBe('GÖTEBORG')
  })

  it('falls back to Windows-1252 when UTF-8 decode is invalid', () => {
    // 0xD6 = Ö in Windows-1252; lone 0xD6 is not valid UTF-8 start byte
    const cp1252 = buf([0x47, 0xd6, 0x54, 0x45, 0x42, 0x4f, 0x52, 0x47])
    expect(decodeFileContent(cp1252)).toBe('GÖTEBORG')
  })

  it('strips a UTF-8 BOM (EF BB BF) and decodes the remainder as UTF-8', () => {
    const payload = Array.from(new TextEncoder().encode('Bokföringsdag;Belopp'))
    const text = decodeFileContent(buf([0xef, 0xbb, 0xbf, ...payload]))
    expect(text).toBe('Bokföringsdag;Belopp')
    expect(text.charCodeAt(0)).not.toBe(0xfeff)
  })

  it('never produces a mojibake BOM prefix when a BOM-ed file needs the Windows-1252 fallback', () => {
    // 'Datum;' + 0xD6 (Ö in Windows-1252, invalid as a lone UTF-8 byte),
    // behind a UTF-8 BOM. The BOM bytes must never re-enter the fallback
    // decode, so the result starts with 'Datum', not the literal mojibake.
    const bytes = buf([0xef, 0xbb, 0xbf, 0x44, 0x61, 0x74, 0x75, 0x6d, 0x3b, 0xd6])
    const text = decodeFileContent(bytes)
    expect(text.startsWith('')).toBe(false)
    expect(text).toBe('Datum;Ö')
  })

  it('decodes UTF-16LE content behind a FF FE BOM', () => {
    // 'Öre' in UTF-16LE: d6 00, 72 00, 65 00
    const bytes = buf([0xff, 0xfe, 0xd6, 0x00, 0x72, 0x00, 0x65, 0x00])
    expect(decodeFileContent(bytes)).toBe('Öre')
  })

  it('decodes UTF-16BE content behind a FE FF BOM', () => {
    const bytes = buf([0xfe, 0xff, 0x00, 0xd6, 0x00, 0x72, 0x00, 0x65])
    expect(decodeFileContent(bytes)).toBe('Öre')
  })
})

describe('stripBOM', () => {
  it('strips a leading U+FEFF', () => {
    expect(stripBOM('\uFEFF' + 'Datum;Belopp')).toBe('Datum;Belopp')
  })

  it('strips a leading literal mojibake BOM ()', () => {
    expect(stripBOM('Datum;Belopp')).toBe('Datum;Belopp')
  })

  it('is a no-op on clean content and never strips mid-string', () => {
    expect(stripBOM('Datum;Belopp')).toBe('Datum;Belopp')
    expect(stripBOM('Datum;Belopp')).toBe('Datum;Belopp')
  })
})

// --- U+FFFD heuristic recovery ---

describe('recoverWordWithFFFD', () => {
  it('recovers uppercase Ö in common Swedish stems', () => {
    expect(recoverWordWithFFFD('F\uFFFDRENING')).toBe('FÖRENING')
    expect(recoverWordWithFFFD('F\uFFFDRETAG')).toBe('FÖRETAG')
    expect(recoverWordWithFFFD('G\uFFFDTEBORG')).toBe('GÖTEBORG')
    expect(recoverWordWithFFFD('LINK\uFFFDPING')).toBe('LINKÖPING')
  })

  it('recovers lowercase ö in common Swedish stems', () => {
    expect(recoverWordWithFFFD('f\uFFFDrening')).toBe('förening')
    expect(recoverWordWithFFFD('malm\uFFFD')).toBe('malmö')
    expect(recoverWordWithFFFD('k\uFFFDp')).toBe('köp')
  })

  it('recovers compound words via substring match', () => {
    expect(recoverWordWithFFFD('BOSTADSR\uFFFDTTSF\uFFFDRENING')).toBe(
      'BOSTADSRÄTTSFÖRENING'
    )
    expect(recoverWordWithFFFD('Idrottsf\uFFFDrening')).toBe('Idrottsförening')
  })

  it('is a no-op when the input has no U+FFFD', () => {
    expect(recoverWordWithFFFD('FÖRENING')).toBe('FÖRENING')
    expect(recoverWordWithFFFD('hello')).toBe('hello')
  })

  it('returns null for ambiguous words not in the dictionary', () => {
    // Random 4-letter word with U+FFFD; no Swedish stem hits.
    expect(recoverWordWithFFFD('Z\uFFFDXQ')).toBeNull()
  })

  it('returns null for words with too many U+FFFDs to disambiguate', () => {
    expect(
      recoverWordWithFFFD('\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD')
    ).toBeNull()
  })
})

describe('recoverStringWithFFFD', () => {
  it('repairs the canonical "Levbet FÖRENING" case', () => {
    expect(recoverStringWithFFFD('Levbet F\uFFFDRENING')).toBe('Levbet FÖRENING')
  })

  it('repairs city + business-name combos', () => {
    expect(recoverStringWithFFFD('Sjöberg AB, Malm\uFFFD')).toBe('Sjöberg AB, Malmö')
    expect(recoverStringWithFFFD('Faktura fr\uFFFDn G\uFFFDTEBORG AB')).toBe(
      'Faktura från GÖTEBORG AB'
    )
  })

  it('preserves punctuation, whitespace, and digits', () => {
    expect(recoverStringWithFFFD('K\uFFFDp 1 234,56 SEK')).toBe('Köp 1 234,56 SEK')
  })

  it('is a no-op on clean strings', () => {
    expect(recoverStringWithFFFD('Hello World')).toBe('Hello World')
    expect(recoverStringWithFFFD('FÖRENING')).toBe('FÖRENING')
  })

  it('returns null when any word in the string is ambiguous', () => {
    expect(recoverStringWithFFFD('FÖRENING Z\uFFFDXQ')).toBeNull()
  })

  it('is idempotent on recovered output', () => {
    const once = recoverStringWithFFFD('F\uFFFDRENING')
    expect(once).toBe('FÖRENING')
    expect(recoverStringWithFFFD(once!)).toBe('FÖRENING')
  })
})