accounted/lib/import/shared/encoding.ts

/**
 * Encoding detection and conversion for Swedish import files.
 *
 * Used by bank file, supplier, customer, and opening-balance parsers.
 * Swedish data exports use either UTF-8 or Windows-1252 (ISO-8859-1).
 * We detect encoding by checking for valid Swedish characters.
 */

/**
 * Decode file content, handling BOMs plus UTF-8 and Windows-1252 encodings.
 *
 * Strategy: inspect the leading BYTES first.
 * - FF FE / FE FF: UTF-16LE/BE (e.g. Excel "Unicode text" re-saves). Decode
 *   as such; TextDecoder consumes the BOM itself.
 * - EF BB BF (UTF-8 BOM): strip the three bytes and decode the REMAINDER.
 *   If the remainder still contains U+FFFD (invalid UTF-8 somewhere in the
 *   file), decode the remainder as Windows-1252. The BOM bytes are never
 *   re-included, so the fallback can no longer produce a literal mojibake
 *   "" prefix that breaks exact-match header detection.
 * - No BOM: try UTF-8; fall back to Windows-1252 on replacement characters
 *   or garbled Swedish chars (unchanged behavior).
 */
export function decodeFileContent(buffer: ArrayBuffer): string {
  const bytes = new Uint8Array(buffer)

  if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xfe) {
    return new TextDecoder('utf-16le', { fatal: false }).decode(buffer)
  }
  if (bytes.length >= 2 && bytes[0] === 0xfe && bytes[1] === 0xff) {
    return new TextDecoder('utf-16be', { fatal: false }).decode(buffer)
  }

  if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) {
    const rest = bytes.subarray(3)
    const utf8Rest = new TextDecoder('utf-8', { fatal: false }).decode(rest)
    if (!utf8Rest.includes('�')) {
      return utf8Rest
    }
    return new TextDecoder('windows-1252', { fatal: false }).decode(rest)
  }

  const utf8Decoder = new TextDecoder('utf-8', { fatal: false })
  const utf8Result = utf8Decoder.decode(buffer)

  if (!hasEncodingIssues(utf8Result)) {
    return utf8Result
  }

  const latin1Decoder = new TextDecoder('windows-1252', { fatal: false })
  return latin1Decoder.decode(buffer)
}

/**
 * Re-decode a string that suffered the canonical "UTF-8 bytes read as Latin-1"
 * mojibake (e.g. "Malmö" → "Malmö", "GÖTEBORG" → "GÖTEBORG").
 *
 * Mechanism: each char in the input is a codepoint that was originally a UTF-8
 * byte misinterpreted as a Latin-1/Windows-1252 character. We pack those chars
 * back into a byte sequence and decode the bytes as UTF-8 to recover the
 * original text.
 *
 * No-op when the string is already clean (no garbled patterns).
 */
export function decodeStringContent(content: string): string {
  if (!hasEncodingIssues(content)) {
    return content
  }

  try {
    const bytes = new Uint8Array(content.length)
    for (let i = 0; i < content.length; i++) {
      bytes[i] = content.charCodeAt(i) & 0xff
    }
    const decoder = new TextDecoder('utf-8', { fatal: false })
    return decoder.decode(bytes)
  } catch {
    return content
  }
}

/**
 * Check if a string has encoding issues (garbled Swedish characters).
 */
export function hasEncodingIssues(text: string): boolean {
  if (text.includes('\uFFFD')) return true

  // Common garbled patterns when Windows-1252 is read as UTF-8:
  // Ã¥ = å, ä = ä, ö = ö, Ã… = Å, Ä = Ä, Ö = Ö
  const garbledPatterns = ['Ã¥', 'ä', 'ö', 'Ã\u0085', 'Ã\u0084', 'Ã\u0096']
  return garbledPatterns.some((pattern) => text.includes(pattern))
}

/**
 * Normalize line endings to \n
 */
export function normalizeLineEndings(content: string): string {
  return content.replace(/\r\n/g, '\n').replace(/\r/g, '\n')
}

/**
 * Strip BOM (Byte Order Mark) from start of content.
 *
 * Also strips the literal mojibake form "" (U+00EF U+00BB U+00BF): a
 * UTF-8 BOM whose bytes were decoded as Windows-1252/Latin-1 upstream.
 * Covers string-entry paths that were pre-decoded outside decodeFileContent.
 */
export function stripBOM(content: string): string {
  if (content.charCodeAt(0) === 0xfeff) {
    return content.slice(1)
  }
  if (content.startsWith('')) {
    return content.slice(3)
  }
  return content
}

/**
 * Prepare file content for parsing: strip BOM, normalize line endings, handle encoding
 */
export function prepareContent(content: string): string {
  return normalizeLineEndings(stripBOM(decodeStringContent(content)))
}

/**
 * --- U+FFFD heuristic recovery ---
 *
 * When Windows-1252 / Latin-1 bytes are decoded as UTF-8 with `fatal: false`,
 * invalid sequences silently become U+FFFD. The original byte is lost, but
 * for Swedish text we can guess from context: the missing letter is almost
 * always one of Å/Ä/Ö (uppercase context) or å/ä/ö (lowercase context).
 *
 * This recovery tries each Swedish vowel substitution and scores the resulting
 * word against a small dictionary of Swedish stems. If exactly one candidate
 * scores above the threshold, it's applied. Ambiguous cases return null and
 * must be reviewed manually.
 */

const REPLACEMENT = '\uFFFD'
const SWEDISH_VOWELS_UPPER = ['Å', 'Ä', 'Ö'] as const
const SWEDISH_VOWELS_LOWER = ['å', 'ä', 'ö'] as const

/**
 * Stems of common Swedish words containing åäö that appear in business names,
 * place names, addresses, and accounting descriptions.
 */
const SWEDISH_STEMS = new Set<string>([
  // -för- prefix (extremely common)
  'för', 'före', 'förening', 'företag', 'försäkring', 'försäljning',
  'försäljnings', 'förskola', 'församling', 'förvaltning', 'förbund',
  'förlag', 'föräldra', 'försök', 'förbrukning', 'förbättring', 'förskott',
  'försening', 'förhandling', 'förbättrings',
  // för-* derivatives that often appear in account names
  'förmån', 'förmåner', 'förmedlad', 'förmedlade', 'förmedling', 'förmedlings',
  'förvaltar', 'förmedla',
  // -mark- / marknadsföring
  'marknadsföring', 'marknadsförings', 'marknad',
  // common Swedish words/prefixes
  'bostadsrätt', 'rätt', 'samfällighet', 'idrott', 'fastighet', 'utbildning',
  'näring', 'växel', 'värme', 'köp', 'köpa', 'inköp', 'sälja', 'säljs',
  'tjänst', 'tjänster',
  // accounting terms
  'kostnad', 'kostnader', 'intäkt', 'intäkter', 'avskrivning', 'avsättning',
  'lön', 'löner', 'lönekostnad', 'pension', 'utgående', 'ingående',
  'momspliktig', 'redovisning', 'företagskonto', 'bankkonto',
  'överavskrivning', 'överskott', 'underskott', 'överföring', 'överlåtelse',
  'återbetalning', 'utlägg', 'utgift', 'avdrag',
  // omvänd moms etc.
  'omvänd', 'omvänt', 'omvända',
  // 3+ letter prepositions/adverbs (skip 2-letter ones: too ambiguous)
  'från', 'över', 'när', 'där', 'även', 'någon', 'något', 'många',
  'själv', 'små', 'väg', 'gång', 'räkning', 'räntor', 'är',
  // 'på': short but extremely common; include explicitly
  'på',
  // cities
  'göteborg', 'malmö', 'örebro', 'östersund', 'jönköping', 'linköping',
  'norrköping', 'lidköping', 'köping', 'helsingborg', 'umeå', 'skellefteå',
  'piteå', 'luleå', 'borås', 'växjö', 'östhammar', 'södertälje', 'västerås',
  'härnösand', 'värnamo', 'mölndal', 'mörrum', 'mönsterås', 'färjestaden',
  'eskilstuna',
  // directions / geo
  'östra', 'västra', 'södra', 'norra', 'öster', 'väster', 'söder',
  // legal forms
  'aktiebolag', 'handelsbolag', 'ekonomisk', 'allmännyttig',
  // misc
  'företagsledare', 'koncernbidrag', 'utländsk', 'utländska', 'främmande',
  'vägen', 'gatan', 'allén', 'gränden', 'torget',
  // surnames containing åäö
  'lindström', 'sjöberg', 'söderberg', 'öberg', 'åström', 'åkerlund',
  'östlund', 'lindgren', 'sjögren', 'hägglund', 'bäckström',
])

/**
 * Score a candidate word by *length-weighted* stem matching.
 *
 *  - Exact word match: 1_000_000 (still beats any substring sum).
 *  - Otherwise: sum the lengths of every stem that is a substring of the
 *    candidate. Length-weighting is the critical fix vs. simple counting:
 *    "fårmedlad" hits "år" (2 chars), "förmedlad" hits "för" (3 chars).
 *    Counting would tie them at 1; length-weighting gives 2 vs 3, so the
 *    correct candidate wins.
 */
function scoreCandidate(word: string): number {
  const lower = word.toLowerCase()
  if (SWEDISH_STEMS.has(lower)) return 1_000_000
  let score = 0
  for (const stem of SWEDISH_STEMS) {
    if (lower.includes(stem)) score += stem.length
  }
  return score
}

function chooseCase(word: string): 'upper' | 'lower' {
  let upper = 0
  let lower = 0
  for (const ch of word) {
    if (ch === REPLACEMENT) continue
    if (ch >= 'A' && ch <= 'Z') upper++
    else if (ch >= 'a' && ch <= 'z') lower++
  }
  return upper > lower ? 'upper' : 'lower'
}

/**
 * Try every Swedish-vowel substitution for the U+FFFDs in `word`, then return
 * the highest-scoring candidate. Returns null if no candidate matches a
 * dictionary stem (i.e. ambiguous: operator must review).
 */
export function recoverWordWithFFFD(word: string): string | null {
  if (!word.includes(REPLACEMENT)) return word

  const vowels = chooseCase(word) === 'upper' ? SWEDISH_VOWELS_UPPER : SWEDISH_VOWELS_LOWER
  const positions: number[] = []
  for (let i = 0; i < word.length; i++) {
    if (word[i] === REPLACEMENT) positions.push(i)
  }

  // Words with more than ~6 lost bytes blow up the combinatorial space:   // bail out rather than spend cycles on something that's likely garbage anyway.
  const totalCombos = Math.pow(vowels.length, positions.length)
  if (totalCombos > 729) return null

  let best: { word: string; score: number } | null = null
  for (let combo = 0; combo < totalCombos; combo++) {
    const chars = word.split('')
    let c = combo
    for (const pos of positions) {
      chars[pos] = vowels[c % vowels.length]
      c = Math.floor(c / vowels.length)
    }
    const candidate = chars.join('')
    const score = scoreCandidate(candidate)
    if (!best || score > best.score) {
      best = { word: candidate, score }
    }
  }

  if (!best || best.score === 0) return null
  return best.word
}

/**
 * Repair every U+FFFD-containing word in `text` via dictionary-backed
 * substitution. Returns the repaired string when *every* corrupted word
 * resolved to a confident candidate; returns null if any word remains
 * ambiguous (operator must review the whole string by hand).
 *
 * Idempotent on clean input.
 */
export function recoverStringWithFFFD(text: string): string | null {
  if (!text.includes(REPLACEMENT)) return text

  // Split on runs of non-letter/non-digit characters so punctuation,
  // whitespace, and structural characters are preserved as-is.
  const tokens = text.split(/([^\p{L}\p{N}\uFFFD]+)/u)
  const out: string[] = []
  for (const token of tokens) {
    if (!token.includes(REPLACEMENT)) {
      out.push(token)
      continue
    }
    const recovered = recoverWordWithFFFD(token)
    if (recovered === null) return null
    out.push(recovered)
  }
  return out.join('')
}